From 0f5edbd6c4c89c285796ce8ea1de9e1ddf5bc20f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 15:36:13 +0200 Subject: [PATCH 1/8] Draw humanitarian immigration statuses to cited stocks in the US stage REFUGEE / ASYLEE / PAROLED_ONE_YEAR (per program origin) / TPS (per country) are now selected between the ASEC-UA indicators and the EAD spill, each to a manifest-cited target, so the H.R.1 $71109/$71301/ $71302 and SNAP $10108 eligibility channels stop computing $0 on the release. DEPORTATION_WITHHELD carries an explicit cited zero (no published stock). Residual-pool parole/TPS draws receive EAD SSN cards, keeping the NONE-undocumented invariant; the composition gate now bands each category against its target. Refs #767 Co-Authored-By: Claude Fable 5 --- .../src/microcosm/build/us/source_stages.json | 62 +- .../microcosm/build/us_runtime/__init__.py | 6 + .../microcosm/build/us_runtime/immigration.py | 619 ++++++++++++++++-- 3 files changed, 621 insertions(+), 66 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 5304c4352..af0b376f5 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1879,7 +1879,7 @@ "kind": "administrative_table", "format": "published_estimate", "vintage": "latest_available", - "locator": "Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration Portal undocumented-student estimates" + "locator": "Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration Portal undocumented-student estimates; OHSS refugee and asylee annual flow reports; DHS OAW/U4U parole reports and CBP CHNV releases; CRS RS20844 TPS designations table; EOIR adjudication statistics via CRS IN12501" } ], "operations": [ @@ -1903,6 +1903,64 @@ "undocumented_population_anchor": { "value": 11000000, "source": "https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/" + }, + "humanitarian_status_stocks": { + "paroled_one_year": { + "afghanistan": { + "target": 73566, + "source": "https://www.dhs.gov/sites/default/files/2023-06/PLCY%20-%20AFG%202503%20Update%202_0.pdf" + }, + "ukraine": { + "target": 158000, + "source": "https://www.dhs.gov/sites/default/files/2024-12/2024_1104_dmo_plcy_uniting_for_ukraine_process_overview_and_assessment.pdf" + }, + "nicaragua": { + "target": 93070, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update" + }, + "venezuela": { + "target": 117330, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update" + } + }, + "refugee": { + "target": 160000, + "source": "https://ohss.dhs.gov/sites/default/files/2024-11/2024_1108_ohss_refugee_annual_flow_report_2023.pdf" + }, + "asylee": { + "target": 160000, + "source": "https://ohss.dhs.gov/sites/default/files/2024-10/2024_1002_ohss_asylees_fy2023.pdf" + }, + "deportation_withheld": { + "target": 0, + "source": "https://crsreports.congress.gov/product/pdf/IN/IN12501" + }, + "tps": { + "venezuela": { + "target": 605015, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "el_salvador": { + "target": 170125, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "honduras": { + "target": 51225, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "nicaragua": { + "target": 2910, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "nepal": { + "target": 7160, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "other_designated": { + "target": 21215, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + } + } } } ], @@ -1910,7 +1968,7 @@ "ssn_card_type", "immigration_status_str" ], - "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. The CPS carries no work-authorization variable, so among residual non-citizens, workers and students spill to NON_CITIZEN_VALID_EAD in deterministic seeded order until the remaining undocumented worker count matches Pew's 8.3M and the undocumented student count matches the Higher Ed Immigration Portal's ~408k \u2014 the only forced margins. The total undocumented population is emergent (\u224813M on the 2024 ASEC, within the range of published 2023-24 estimates) and is gated against Pew's 11.0M 2022 anchor with a coarse plausibility band rather than forced; reconciling the level is the calibration lane's job. Immigration-status tags carry only statutory tests the data supports: DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival); other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status \u2014 blanket REFUGEE/TPS labels are deliberately not fabricated because they would mislabel millions and over-grant refugee-class benefit exemptions." + "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually exclusively in decreasing order of target precision \u2014 PAROLED_ONE_YEAR per program origin (OAW Afghans 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS \u2014 refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the trailing two years), ASYLEE (FY2022-FY2024 grants 35,720+54,350+~70,000 OHSS; 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility \u2014 Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS's resolution for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), consistent with Pew's own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student spill then binds the remaining residual pool to Pew's 8.3M workers and the Higher Ed Immigration Portal's ~408k students \u2014 still the only forced margins. The total undocumented population is emergent and gated against Pew's 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are gated against their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals \u2014 the Ukraine parole pool saturates below its admin count); reconciling levels is the calibration lane's job. DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status." }, { "stage": "hours_worked", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index e9e756473..35dc9b7fa 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -371,12 +371,15 @@ with_us_housing_inputs, ) from microcosm.build.us_runtime.immigration import ( + HUMANITARIAN_STATUS_CATEGORIES, IMMIGRATION_STATUS_VALUES, SSN_CARD_TYPE_VALUES, US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS, US_IMMIGRATION_OUTPUT_COLUMNS, US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS, US_IMMIGRATION_STAGE_NAME, + HumanitarianDraw, + ImmigrationControls, UndocumentedControls, derive_us_immigration_status_from_manifest, us_immigration_composition_gate, @@ -1149,12 +1152,15 @@ "US_JCT_TAX_EXPENDITURE_TARGET_SPECS", "US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES", "SOI_VARIABLE_MAP", + "HUMANITARIAN_STATUS_CATEGORIES", "IMMIGRATION_STATUS_VALUES", "SSN_CARD_TYPE_VALUES", "US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS", "US_IMMIGRATION_OUTPUT_COLUMNS", "US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS", "US_IMMIGRATION_STAGE_NAME", + "HumanitarianDraw", + "ImmigrationControls", "UndocumentedControls", "US_HOURS_WORKED_NONCONSTANT_PERSON_COLUMNS", "US_HOURS_WORKED_OUTPUT_COLUMNS", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index 9c5bffbf1..48dc63557 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -41,16 +41,33 @@ emergent total is ≈13M — inside the range of published 2023–24 estimates — with 26.6M non-citizens, 18.0M residual after the indicators, and 4.8M spilled to EAD. -5. **Status tags carry only statutory tests the data can support.** +5. **Humanitarian/temporary-protection statuses are drawn to published + stocks, never blanketed (microcosm #767).** Blanket ``REFUGEE``/``TPS`` + labels for recent arrivals were originally refused because LPR treatment + gave near-identical means-tested eligibility; H.R.1 repealed exactly that + equivalence (Medicaid §71109 effective 2026-10-01, SNAP §10108 effective + 2025-07-01, ACA §71301/§71302 effective 2026/2027 — all keyed to these + enum values in policyengine-us parameters), so an LPR-only file silently + zeroes every one of those channels. Instead of blankets, the stage now + draws ``REFUGEE`` / ``ASYLEE`` / ``DEPORTATION_WITHHELD`` / + ``PAROLED_ONE_YEAR`` / ``TPS`` — in that order, mutually exclusive — + from candidates whose survey signature supports the status (origin + country x arrival window x legal-status indicators), each to a + manifest-cited weighted stock target. Cuba/Haiti-born persons are + excluded from every humanitarian draw (the ``CUBAN_HAITIAN_ENTRANT`` + class is the better statutory label and keeps H.R.1 eligibility), as is + the DACA statutory cohort. Draws from the residual pool (TPS and parole + only) flip ``ssn_card_type`` ``NONE`` to ``NON_CITIZEN_VALID_EAD`` — + both programs grant employment authorization — so the two output columns + never disagree. ``CONDITIONAL_ENTRANT`` is deliberately not emitted: the + INA 203(a)(7) class closed in 1980 and surviving holders are + indistinguishable from LPRs at this granularity. +6. **Status tags carry only statutory tests the data can support.** ``DACA`` applies the statutory cohort test (arrived by 2007 before age 16, aged 15+) to EAD holders; ``CUBAN_HAITIAN_ENTRANT`` applies the nationality-plus-arrival class to documented non-citizens; every other documented non-citizen stays ``LEGAL_PERMANENT_RESIDENT`` (the modal true - status). Blanket ``REFUGEE``/``TPS`` labels for recent arrivals or - leftover EAD holders are deliberately not emitted: they would mislabel - millions (true stocks are under a million each) and over-grant - refugee-class exemptions in benefit rules, while LPR treatment gives - near-identical means-tested eligibility for those populations. + status). Selection draws are seeded blake2b hashes keyed by the person's stable source identity (``source_year``/``source_person_id`` when present), so @@ -89,12 +106,15 @@ from microcosm.frame.units import US_SCHEMA __all__ = [ + "HUMANITARIAN_STATUS_CATEGORIES", "IMMIGRATION_STATUS_VALUES", "SSN_CARD_TYPE_VALUES", "US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS", "US_IMMIGRATION_OUTPUT_COLUMNS", "US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS", "US_IMMIGRATION_STAGE_NAME", + "HumanitarianDraw", + "ImmigrationControls", "UndocumentedControls", "derive_us_immigration_status_from_manifest", "us_immigration_composition_gate", @@ -126,13 +146,18 @@ #: PolicyEngine-US ``ImmigrationStatus`` enum member names this stage emits #: (a deliberate subset of the engine's full enum domain; see module -#: docstring for why blanket REFUGEE/TPS labels are not fabricated). +#: docstring for why ``CONDITIONAL_ENTRANT`` is not fabricated). IMMIGRATION_STATUS_VALUES: tuple[str, ...] = ( "CITIZEN", "LEGAL_PERMANENT_RESIDENT", "CUBAN_HAITIAN_ENTRANT", "DACA", "UNDOCUMENTED", + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "TPS", ) #: Raw CPS ASEC person columns the derivation reads. All of them ship in the @@ -169,11 +194,13 @@ _FORCED_CONTROL_KEYS = ("undocumented_workers", "undocumented_students") _ANCHOR_KEY = "undocumented_population_anchor" +_HUMANITARIAN_KEY = "humanitarian_status_stocks" _DERIVE_IMMIGRATION_STATUS_PARAMETER_KEYS = frozenset( { *_FORCED_CONTROL_KEYS, _ANCHOR_KEY, + _HUMANITARIAN_KEY, "seed_from_build_config", "time_period_from_build_config", } @@ -226,6 +253,118 @@ _DACA_MAX_AGE_AT_ENTRY = 16 _DACA_MIN_CURRENT_AGE = 15 +#: Humanitarian draw order: decreasing target precision (exact per-origin +#: program admissions first, national flow aggregates next, per-country TPS +#: registrations last). Draws are sequential and mutually exclusive. +HUMANITARIAN_STATUS_CATEGORIES: tuple[str, ...] = ( + "paroled_one_year", + "refugee", + "asylee", + "deportation_withheld", + "tps", +) + +_HUMANITARIAN_STATUS_BY_CATEGORY: Mapping[str, str] = { + "paroled_one_year": "PAROLED_ONE_YEAR", + "refugee": "REFUGEE", + "asylee": "ASYLEE", + "deportation_withheld": "DEPORTATION_WITHHELD", + "tps": "TPS", +} + +#: PEINUSYR codes for 2020+ arrivals (2024 ASEC data dictionary: 27 = +#: 2020-2021, 28 = 2022-2024) — the humanitarian-parole and recent-refugee +#: cohorts. The Census codebook rebins this variable across vintages, so +#: these windows are survey-vintage facts, not statutory dates. +_RECENT_ARRIVAL_CODES = (27, 28) +#: Asylum grants lag arrival by filing queues plus court backlog; grantees +#: in 2022-2024 overwhelmingly arrived 2016+ (OHSS asylee flow reports). +_ASYLEE_ARRIVAL_CODES = (25, 26, 27, 28) +#: Withholding of removal follows years of proceedings: settled arrivals only. +_WITHHELD_MAX_ARRIVAL_CODE = 26 + +#: PENATVTY birth-country codes, verified against Census CPS technical +#: documentation (cpsmar24.pdf) Appendix I "Countries and Areas of the +#: World". Cuba (327) and Haiti (332) stay out of every humanitarian draw: +#: the Cuban/Haitian-entrant class is the better statutory label. +#: +#: Parole per-origin keys: Operation Allies Welcome (Afghanistan), Uniting +#: for Ukraine, and the non-Cuban/Haitian CHNV nationalities. +_PAROLE_ORIGIN_CODES: Mapping[str, tuple[int, ...]] = { + "afghanistan": (200,), + "ukraine": (164,), + "nicaragua": (315,), + "venezuela": (373,), +} + +#: Top refugee-resettlement nationalities, OHSS Refugees Annual Flow Report +#: FY2023 Table 3 and the OHSS FY-24 report (DR Congo, Syria, Afghanistan, +#: Burma, Venezuela lead; Congo appears as both 412 Congo and 459 Zaire). +_REFUGEE_ORIGIN_CODES: tuple[int, ...] = ( + 412, # Congo + 459, # Zaire (Democratic Republic of the Congo) + 239, # Syria + 200, # Afghanistan + 205, # Myanmar (Burma) + 373, # Venezuela + 448, # Somalia + 451, # Sudan + 417, # Eritrea + 213, # Iraq + 313, # Guatemala + 164, # Ukraine +) + +#: Top asylum-grant nationalities, OHSS Asylees flow reports FY2022-FY2024 +#: (Afghanistan, China, Venezuela, Russia lead; Central/South America and +#: Egypt/Cameroon/Turkey round out the recurring top grant countries). +_ASYLEE_ORIGIN_CODES: tuple[int, ...] = ( + 200, # Afghanistan + 207, # China + 373, # Venezuela + 163, # Russia + 312, # El Salvador + 313, # Guatemala + 314, # Honduras + 315, # Nicaragua + 210, # India + 414, # Egypt + 407, # Cameroon + 364, # Colombia + 365, # Ecuador + 370, # Peru + 243, # Turkey + 239, # Syria +) + +#: TPS per-country keys: birth codes plus the latest PEINUSYR arrival code +#: compatible with the designation's required continuous-residence date +#: (CRS RS20844 Table 1). Legacy designations bind hard (El Salvador +#: 2001-02-13 -> code 17; Honduras/Nicaragua 1998-12-30 -> code 16; Nepal +#: 2015-06-24 -> code 24); 2021+ designations reach the top code, where the +#: 2022-2024 bin unavoidably includes some post-cutoff arrivals. +#: Ukraine and Afghanistan TPS registrants are carried by the parole draw +#: (same U4U/OAW people); Haiti TPS is carried by CUBAN_HAITIAN_ENTRANT. +_TPS_ORIGIN_CODES: Mapping[str, tuple[tuple[int, ...], int]] = { + "venezuela": ((373,), 28), + "el_salvador": ((312,), 17), + "honduras": ((314,), 16), + "nicaragua": ((315,), 16), + "nepal": ((229,), 24), + # Burma, Syria, Yemen, Lebanon, Cameroon, Ethiopia, Somalia, Sudan + # (South Sudan shares 451 in the CPS codebook). + "other_designated": ((205, 239, 248, 224, 407, 416, 448, 451), 28), +} + +#: Categories whose flat manifest block carries one national target. +_FLAT_HUMANITARIAN_CATEGORIES = ("refugee", "asylee", "deportation_withheld") +#: Categories whose manifest block carries per-origin targets, and the +#: module table each block's keys must match exactly. +_PER_ORIGIN_HUMANITARIAN_CATEGORIES: Mapping[str, tuple[str, ...]] = { + "paroled_one_year": tuple(_PAROLE_ORIGIN_CODES), + "tps": tuple(_TPS_ORIGIN_CODES), +} + #: Weighted share of persons whose SSN card type is not ``CITIZEN`` must land #: in this band (Census counts ~22–27M non-citizens of ~336M residents; a #: share outside it means the imputation collapsed or exploded). @@ -235,6 +374,12 @@ #: collapse or explosion, not a calibration objective; the level belongs to #: the calibration lane. _UNDOCUMENTED_ANCHOR_RELATIVE_BAND = (0.5, 1.6) +#: Emitted humanitarian mass per category relative to its cited target. +#: The draw forces the target when candidates suffice, so the band only +#: bites on pool exhaustion (the ASEC undercovers 2022-24 arrivals — the +#: Ukraine parole pool saturates below its admin count) or collapse. A +#: category with an explicit zero target must emit exactly zero. +_HUMANITARIAN_TARGET_RELATIVE_BAND = (0.35, 1.5) @dataclass(frozen=True) @@ -273,6 +418,69 @@ def __post_init__(self) -> None: ) +@dataclass(frozen=True) +class HumanitarianDraw: + """One seeded humanitarian draw: a cited stock the stage selects toward. + + Attributes: + category: Manifest category key (``paroled_one_year`` … ``tps``). + origin: Per-origin key inside the category, or ``None`` for a + national draw. + status: ``ImmigrationStatus`` enum member name the draw emits. + target: Cited weighted stock. Zero is allowed and means the + category is explicitly not imputed (the citation documents why). + source: Citation URL or reference for the target. + """ + + category: str + origin: str | None + status: str + target: float + source: str + + def __post_init__(self) -> None: + if not np.isfinite(self.target) or self.target < 0: + raise ValueError( + f"Humanitarian target {self.label!r} must be non-negative, " + f"got {self.target!r}." + ) + if not self.source: + raise ValueError( + f"Humanitarian target {self.label!r} requires a source citation." + ) + + @property + def label(self) -> str: + return self.category if self.origin is None else f"{self.category}:{self.origin}" + + @property + def salt(self) -> str: + return f"immigration:{self.label}" + + +@dataclass(frozen=True) +class ImmigrationControls: + """Every manifest-sourced control the immigration stage consumes. + + Attributes: + undocumented: The legacy forced margins and the composition-gate + anchor for the residual pool. + humanitarian: The humanitarian draws in assignment order — the + per-origin draws of a category are adjacent, categories follow + ``HUMANITARIAN_STATUS_CATEGORIES``. + """ + + undocumented: UndocumentedControls + humanitarian: tuple[HumanitarianDraw, ...] + + def humanitarian_target(self, category: str) -> float: + """Summed cited target for one category across its origins.""" + + return float( + sum(draw.target for draw in self.humanitarian if draw.category == category) + ) + + def us_immigration_stage_spec() -> SourceStageSpec: """Load the packaged ``immigration_status`` source-stage manifest entry.""" @@ -346,15 +554,17 @@ def derive_us_immigration_status_from_manifest( raise SourceRuntimeError( "US immigration derivation requires finite non-negative person weights." ) - ssn_codes = _assign_ssn_card_codes( + ssn_codes, humanitarian_marks = _assign_ssn_card_codes( result, weights.to_numpy(dtype=np.float64), seed=int(context.config.seed), controls=controls, + time_period=int(context.config.target_year), ) status = _derive_immigration_status( result, ssn_codes, + humanitarian_marks, time_period=int(context.config.target_year), ) code_to_name = { @@ -368,7 +578,119 @@ def derive_us_immigration_status_from_manifest( return result -def _controls_from_parameters(params: Mapping[str, object]) -> UndocumentedControls: +def _target_and_source( + block: object, + *, + label: str, + value_key: str, + allow_zero: bool, +) -> tuple[float, str]: + if not isinstance(block, Mapping): + raise SourceRuntimeError( + f"US immigration control {label!r} requires an object with " + f"{value_key!r} and 'source'." + ) + unexpected = sorted(set(block) - {value_key, "source"}) + if unexpected: + raise SourceRuntimeError( + f"US immigration control {label!r} has unsupported key(s): {unexpected}." + ) + value = block.get(value_key) + source = block.get("source") + valid_number = isinstance(value, int | float) and not isinstance(value, bool) + if not valid_number or (float(value) <= 0 and not (allow_zero and value == 0)): + requirement = "non-negative" if allow_zero else "positive" + raise SourceRuntimeError( + f"US immigration control {label!r} requires a {requirement} numeric " + f"{value_key!r}." + ) + if not isinstance(source, str) or not source: + raise SourceRuntimeError( + f"US immigration control {label!r} requires a source citation." + ) + return float(value), source + + +def _humanitarian_draws_from_parameters( + params: Mapping[str, object], +) -> tuple[HumanitarianDraw, ...]: + block = params.get(_HUMANITARIAN_KEY) + if not isinstance(block, Mapping): + raise SourceRuntimeError( + f"US immigration derivation requires a {_HUMANITARIAN_KEY!r} object " + f"with one block per category {list(HUMANITARIAN_STATUS_CATEGORIES)}." + ) + unexpected = sorted(set(block) - set(HUMANITARIAN_STATUS_CATEGORIES)) + if unexpected: + raise SourceRuntimeError( + f"US immigration {_HUMANITARIAN_KEY} has unsupported category(ies): " + f"{unexpected}." + ) + missing = sorted(set(HUMANITARIAN_STATUS_CATEGORIES) - set(block)) + if missing: + raise SourceRuntimeError( + f"US immigration {_HUMANITARIAN_KEY} is missing category(ies): " + f"{missing}." + ) + draws: list[HumanitarianDraw] = [] + for category in HUMANITARIAN_STATUS_CATEGORIES: + status = _HUMANITARIAN_STATUS_BY_CATEGORY[category] + category_block = block[category] + if category in _PER_ORIGIN_HUMANITARIAN_CATEGORIES: + origin_keys = _PER_ORIGIN_HUMANITARIAN_CATEGORIES[category] + if not isinstance(category_block, Mapping): + raise SourceRuntimeError( + f"US immigration control {category!r} requires per-origin " + f"blocks {list(origin_keys)}." + ) + unexpected_origins = sorted(set(category_block) - set(origin_keys)) + if unexpected_origins: + raise SourceRuntimeError( + f"US immigration control {category!r} has origin(s) outside " + f"the stage's codebook table: {unexpected_origins}." + ) + missing_origins = sorted(set(origin_keys) - set(category_block)) + if missing_origins: + raise SourceRuntimeError( + f"US immigration control {category!r} is missing origin(s): " + f"{missing_origins}." + ) + for origin in origin_keys: + target, source = _target_and_source( + category_block[origin], + label=f"{category}:{origin}", + value_key="target", + allow_zero=True, + ) + draws.append( + HumanitarianDraw( + category=category, + origin=origin, + status=status, + target=target, + source=source, + ) + ) + else: + target, source = _target_and_source( + category_block, + label=category, + value_key="target", + allow_zero=True, + ) + draws.append( + HumanitarianDraw( + category=category, + origin=None, + status=status, + target=target, + source=source, + ) + ) + return tuple(draws) + + +def _controls_from_parameters(params: Mapping[str, object]) -> ImmigrationControls: values: dict[str, float] = {} sources: dict[str, str] = {} for key, value_key in ( @@ -376,40 +698,21 @@ def _controls_from_parameters(params: Mapping[str, object]) -> UndocumentedContr ("undocumented_students", "target"), (_ANCHOR_KEY, "value"), ): - block = params.get(key) - if not isinstance(block, Mapping): - raise SourceRuntimeError( - f"US immigration derivation requires a {key!r} object with " - f"{value_key!r} and 'source'." - ) - unexpected = sorted(set(block) - {value_key, "source"}) - if unexpected: - raise SourceRuntimeError( - f"US immigration control {key!r} has unsupported key(s): {unexpected}." - ) - value = block.get(value_key) - source = block.get("source") - if ( - not isinstance(value, int | float) - or isinstance(value, bool) - or float(value) <= 0 - ): - raise SourceRuntimeError( - f"US immigration control {key!r} requires a positive numeric " - f"{value_key!r}." - ) - if not isinstance(source, str) or not source: - raise SourceRuntimeError( - f"US immigration control {key!r} requires a source citation." - ) - values[key] = float(value) + value, source = _target_and_source( + params.get(key), label=key, value_key=value_key, allow_zero=False + ) + values[key] = value sources[key] = source - return UndocumentedControls( + undocumented = UndocumentedControls( workers=values["undocumented_workers"], students=values["undocumented_students"], population_anchor=values[_ANCHOR_KEY], sources=sources, ) + return ImmigrationControls( + undocumented=undocumented, + humanitarian=_humanitarian_draws_from_parameters(params), + ) def _integer_column(person: pd.DataFrame, column: str) -> np.ndarray: @@ -500,13 +803,145 @@ def _select_weight_to_target( return selected -def _assign_ssn_card_codes( +def _arrival_profile( + person: pd.DataFrame, *, time_period: int +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """(PEINUSYR code, arrival-year midpoint, age at entry) per person.""" + + arrival_code = _integer_column(person, "PEINUSYR") + arrival_year = np.full(len(person), time_period, dtype=np.int64) + for code, midpoint in _ARRIVAL_YEAR_MIDPOINTS.items(): + arrival_year[arrival_code == code] = midpoint + years_in_us = time_period - arrival_year + age = _integer_column(person, "A_AGE") + age_at_entry = np.maximum(0, age - years_in_us) + return arrival_code, arrival_year, age_at_entry + + +def _daca_statutory_cohort( + arrival_year: np.ndarray, age_at_entry: np.ndarray, age: np.ndarray +) -> np.ndarray: + return ( + (arrival_year <= _DACA_LATEST_ARRIVAL_YEAR) + & (age_at_entry < _DACA_MAX_AGE_AT_ENTRY) + & (age >= _DACA_MIN_CURRENT_AGE) + ) + + +def _humanitarian_draw_candidates( + draw: HumanitarianDraw, + *, + ssn_codes: np.ndarray, + birth_country: np.ndarray, + arrival_code: np.ndarray, +) -> np.ndarray: + """Candidate mask for one draw, before exclusions shared by all draws. + + ``REFUGEE``/``ASYLEE``/``DEPORTATION_WITHHELD`` draw from the + indicator-documented pool only (code 3: those statuses carry immediate + federal benefits access, the same signals the ASEC-UA method reads); + parole and TPS may also draw from the still-unspilled residual pool + (code 0), whose members then receive EAD-backed SSN cards. + """ + + if draw.category == "paroled_one_year": + assert draw.origin is not None + return ( + np.isin(ssn_codes, (0, 3)) + & np.isin(birth_country, _PAROLE_ORIGIN_CODES[draw.origin]) + & np.isin(arrival_code, _RECENT_ARRIVAL_CODES) + ) + if draw.category == "refugee": + return ( + (ssn_codes == 3) + & np.isin(birth_country, _REFUGEE_ORIGIN_CODES) + & np.isin(arrival_code, _RECENT_ARRIVAL_CODES) + ) + if draw.category == "asylee": + return ( + (ssn_codes == 3) + & np.isin(birth_country, _ASYLEE_ORIGIN_CODES) + & np.isin(arrival_code, _ASYLEE_ARRIVAL_CODES) + ) + if draw.category == "deportation_withheld": + return ( + (ssn_codes == 3) + & (arrival_code >= 1) + & (arrival_code <= _WITHHELD_MAX_ARRIVAL_CODE) + ) + if draw.category == "tps": + assert draw.origin is not None + codes, max_arrival_code = _TPS_ORIGIN_CODES[draw.origin] + return ( + np.isin(ssn_codes, (0, 3)) + & np.isin(birth_country, codes) + & (arrival_code >= 1) + & (arrival_code <= max_arrival_code) + ) + raise SourceRuntimeError( + f"US immigration derivation has no candidate rule for draw " + f"{draw.label!r}." + ) + + +def _assign_humanitarian_statuses( person: pd.DataFrame, + ssn_codes: np.ndarray, weights: np.ndarray, *, seed: int, - controls: UndocumentedControls, + controls: ImmigrationControls, + time_period: int, ) -> np.ndarray: + """Mark humanitarian statuses and upgrade residual draws' SSN codes. + + Runs after the legal-status indicators and before the EAD worker/student + spill, so the Pew worker/student controls still bind the remaining + residual pool exactly. Draws are sequential over + ``controls.humanitarian``; a person marked by an earlier draw is out of + every later candidate pool. Residual-pool selections (parole/TPS only) + move to ``NON_CITIZEN_VALID_EAD`` — both programs confer employment + authorization — keeping the ``NONE`` ⇔ ``UNDOCUMENTED`` invariant. + """ + + marks = np.full(len(person), "", dtype="U24") + arrival_code, arrival_year, age_at_entry = _arrival_profile( + person, time_period=time_period + ) + age = _integer_column(person, "A_AGE") + birth_country = _integer_column(person, "PENATVTY") + # Cuba/Haiti-born keep the statutorily-favorable entrant class; the DACA + # cohort keeps its statutory tag (dual TPS/DACA holders stay DACA). + excluded = np.isin(birth_country, _CUBAN_HAITIAN_BIRTH_CODES) + excluded |= _daca_statutory_cohort(arrival_year, age_at_entry, age) + for draw in controls.humanitarian: + if draw.target <= 0: + continue + candidates = ( + _humanitarian_draw_candidates( + draw, + ssn_codes=ssn_codes, + birth_country=birth_country, + arrival_code=arrival_code, + ) + & ~excluded + & (marks == "") + ) + draws = _stable_person_draws(person, seed=seed, salt=draw.salt) + selected = _select_weight_to_target(candidates, weights, draws, draw.target) + marks[selected] = draw.status + ssn_codes[selected & (ssn_codes == 0)] = 2 + return marks + + +def _assign_ssn_card_codes( + person: pd.DataFrame, + weights: np.ndarray, + *, + seed: int, + controls: ImmigrationControls, + time_period: int, +) -> tuple[np.ndarray, np.ndarray]: citizenship = _integer_column(person, "PRCITSHP") unknown = ~np.isin(citizenship, [1, 2, 3, 4, 5]) if unknown.any(): @@ -566,6 +1001,18 @@ def _assign_ssn_card_codes( ) ssn_codes[(ssn_codes == 0) & assumed_documented] = 3 + # Humanitarian draws run between the indicators and the EAD spill: the + # spill controls then bind the *remaining* residual pool, so the final + # undocumented worker/student counts still land on the Pew anchors. + humanitarian_marks = _assign_humanitarian_statuses( + person, + ssn_codes, + weights, + seed=seed, + controls=controls, + time_period=time_period, + ) + is_worker = (_float_column(person, "WSAL_VAL") > 0) | ( _float_column(person, "SEMP_VAL") > 0 ) @@ -579,7 +1026,9 @@ def _assign_ssn_card_codes( # composition gate checks it against a cited anchor, and reconciling the # level is the calibration lane's job, not this label stage's. worker_candidates = (ssn_codes == 0) & noncitizens & is_worker - worker_excess = float(weights[worker_candidates].sum()) - controls.workers + worker_excess = ( + float(weights[worker_candidates].sum()) - controls.undocumented.workers + ) worker_draws = _stable_person_draws( person, seed=seed, salt="immigration:ead_workers" ) @@ -590,7 +1039,9 @@ def _assign_ssn_card_codes( ] = 2 student_candidates = (ssn_codes == 0) & noncitizens & is_student - student_excess = float(weights[student_candidates].sum()) - controls.students + student_excess = ( + float(weights[student_candidates].sum()) - controls.undocumented.students + ) student_draws = _stable_person_draws( person, seed=seed, salt="immigration:ead_students" ) @@ -599,22 +1050,18 @@ def _assign_ssn_card_codes( student_candidates, weights, student_draws, student_excess ) ] = 2 - return ssn_codes + return ssn_codes, humanitarian_marks def _derive_immigration_status( person: pd.DataFrame, ssn_codes: np.ndarray, + humanitarian_marks: np.ndarray, *, time_period: int, ) -> np.ndarray: - arrival_code = _integer_column(person, "PEINUSYR") - arrival_year = np.full(len(person), time_period, dtype=np.int64) - for code, midpoint in _ARRIVAL_YEAR_MIDPOINTS.items(): - arrival_year[arrival_code == code] = midpoint - years_in_us = time_period - arrival_year + _, arrival_year, age_at_entry = _arrival_profile(person, time_period=time_period) age = _integer_column(person, "A_AGE") - age_at_entry = np.maximum(0, age - years_in_us) birth_country = _integer_column(person, "PENATVTY") # Documented non-citizens default to LPR — the modal true status — and @@ -633,13 +1080,14 @@ def _derive_immigration_status( ) status[cuban_haitian] = "CUBAN_HAITIAN_ENTRANT" - daca = ( - (ssn_codes == 2) - & (arrival_year <= _DACA_LATEST_ARRIVAL_YEAR) - & (age_at_entry < _DACA_MAX_AGE_AT_ENTRY) - & (age >= _DACA_MIN_CURRENT_AGE) - ) + daca = (ssn_codes == 2) & _daca_statutory_cohort(arrival_year, age_at_entry, age) status[daca] = "DACA" + + # Humanitarian draws exclude Cuba/Haiti-born and the DACA cohort, so + # the marks are disjoint from both tags; every marked person carries a + # non-NONE SSN code, so the columns still agree about the undocumented. + marked = humanitarian_marks != "" + status[marked] = humanitarian_marks[marked] return status @@ -746,7 +1194,7 @@ def us_immigration_composition_summary(frame: Frame) -> dict[str, object]: def us_immigration_composition_gate( frame: Frame, *, - controls: UndocumentedControls | None = None, + controls: ImmigrationControls | None = None, ) -> GateResult: """Release gate: the SSN/immigration surface exists and is plausible. @@ -754,8 +1202,12 @@ def us_immigration_composition_gate( mode: everyone a citizen with a valid SSN), when a value falls outside the engine enum domain, when the two columns disagree about citizenship or undocumented status, when the weighted non-citizen share leaves its - plausibility band, or when the emergent undocumented population strays - outside a coarse band around its cited published anchor. + plausibility band, when the emergent undocumented population strays + outside a coarse band around its cited published anchor, or when a + humanitarian category's emitted mass leaves the coarse band around its + cited stock target (microcosm #767 — the H.R.1 §71109/§71301/§71302 and + SNAP §10108 eligibility channels all bind through these categories; an + explicit zero target must emit exactly zero). """ if controls is None: @@ -779,13 +1231,20 @@ def us_immigration_composition_gate( details: dict[str, object] = { "summary": us_immigration_composition_summary(frame), "controls": { - "undocumented_workers": controls.workers, - "undocumented_students": controls.students, - "undocumented_population_anchor": controls.population_anchor, - "sources": dict(controls.sources), + "undocumented_workers": controls.undocumented.workers, + "undocumented_students": controls.undocumented.students, + "undocumented_population_anchor": ( + controls.undocumented.population_anchor + ), + "sources": dict(controls.undocumented.sources), + "humanitarian_status_stocks": { + draw.label: {"target": draw.target, "source": draw.source} + for draw in controls.humanitarian + }, }, "non_citizen_share_band": list(_NON_CITIZEN_SHARE_BAND), "undocumented_anchor_relative_band": list(_UNDOCUMENTED_ANCHOR_RELATIVE_BAND), + "humanitarian_target_relative_band": list(_HUMANITARIAN_TARGET_RELATIVE_BAND), } missing = [ @@ -850,16 +1309,48 @@ def us_immigration_composition_gate( ) undocumented = float(weights[ssn_none].sum()) - relative = undocumented / controls.population_anchor + relative = undocumented / controls.undocumented.population_anchor rel_low, rel_high = _UNDOCUMENTED_ANCHOR_RELATIVE_BAND if not (rel_low <= relative <= rel_high): failures.append( f"emergent undocumented population {undocumented:,.0f} is " f"{relative:.2f}x the published anchor " - f"{controls.population_anchor:,.0f} (band [{rel_low}, {rel_high}], " - f"{controls.sources[_ANCHOR_KEY]})." + f"{controls.undocumented.population_anchor:,.0f} " + f"(band [{rel_low}, {rel_high}], " + f"{controls.undocumented.sources[_ANCHOR_KEY]})." ) + status_values = status.to_numpy() + hum_low, hum_high = _HUMANITARIAN_TARGET_RELATIVE_BAND + achieved_by_category: dict[str, dict[str, object]] = {} + for category in HUMANITARIAN_STATUS_CATEGORIES: + status_name = _HUMANITARIAN_STATUS_BY_CATEGORY[category] + target = controls.humanitarian_target(category) + emitted = float(weights[status_values == status_name].sum()) + achieved_by_category[category] = { + "status": status_name, + "target": target, + "population": emitted, + "relative": (emitted / target) if target > 0 else None, + } + if target <= 0: + if emitted > 0: + failures.append( + f"{status_name}: {emitted:,.0f} weighted persons emitted " + "against an explicit zero target — the manifest documents " + "this category as not imputed." + ) + continue + category_relative = emitted / target + if not (hum_low <= category_relative <= hum_high): + failures.append( + f"{status_name}: emitted population {emitted:,.0f} is " + f"{category_relative:.2f}x the cited stock target " + f"{target:,.0f} (band [{hum_low}, {hum_high}]) — the H.R.1 " + "eligibility channels through this category are degenerate." + ) + details["humanitarian_achieved"] = achieved_by_category + return GateResult( name="immigration_composition", passed=not failures, From 0e2cb72161276f724607242f69b29ca473805cee Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 15:39:27 +0200 Subject: [PATCH 2/8] Declare the humanitarian draw sites and regenerate the US spec bundle Thirteen stable blake2b sites (per-origin parole, refugee, asylee, deportation-withheld, per-country TPS) join the seed protocol; the seed-owner ledger binds them to the immigration_status stage; the sources schema gains the required humanitarian_status_stocks block; the bundle regenerates with the new source_stages attestation. Refs #767 Co-Authored-By: Claude Fable 5 --- .../spec_engine/schema/sources.schema.json | 261 ++++++++++++++++++ .../src/microcosm/build/spec_engine/seeds.py | 25 ++ .../src/microcosm/build/us/spec/sources.yaml | 89 +++++- .../src/microcosm/build/us/spec/spine.yaml | 52 ++++ tools/generate_us_bundle_from_constants.py | 2 +- .../identity_contracts.py | 24 +- 6 files changed, 437 insertions(+), 16 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index c6f16826f..ff05c8a21 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -1594,9 +1594,270 @@ "source", "target" ] + }, + "humanitarian_status_stocks": { + "type": "object", + "additionalProperties": false, + "properties": { + "asylee": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "deportation_withheld": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "paroled_one_year": { + "type": "object", + "additionalProperties": false, + "properties": { + "afghanistan": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "nicaragua": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "ukraine": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "venezuela": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + } + }, + "required": [ + "afghanistan", + "nicaragua", + "ukraine", + "venezuela" + ] + }, + "refugee": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "tps": { + "type": "object", + "additionalProperties": false, + "properties": { + "el_salvador": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "honduras": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "nepal": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "nicaragua": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "other_designated": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "venezuela": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + } + }, + "required": [ + "el_salvador", + "honduras", + "nepal", + "nicaragua", + "other_designated", + "venezuela" + ] + } + }, + "required": [ + "asylee", + "deportation_withheld", + "paroled_one_year", + "refugee", + "tps" + ] } }, "required": [ + "humanitarian_status_stocks", "kind", "seed_from_build_config", "time_period_from_build_config", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py index 36e6c27e8..402111495 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py @@ -741,6 +741,31 @@ def _stable_site( ), candidate_universe="all_person_rows_before_abawd_eligibility_mask", ), + *( + _stable_site( + f"immigration_humanitarian_{label.replace(':', '_')}_assignment", + salt=f"immigration:{label}", + key_grammar=("source_year:source_person_id_if_present", "else_person_id"), + candidate_universe=( + "all_person_rows_then_category_origin_window_pool_mask" + ), + ) + for label in ( + "paroled_one_year:afghanistan", + "paroled_one_year:ukraine", + "paroled_one_year:nicaragua", + "paroled_one_year:venezuela", + "refugee", + "asylee", + "deportation_withheld", + "tps:venezuela", + "tps:el_salvador", + "tps:honduras", + "tps:nicaragua", + "tps:nepal", + "tps:other_designated", + ) + ), _stable_site( "immigration_ead_workers_assignment", salt="immigration:ead_workers", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index a160533f4..ba0b87935 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097 + sha256: 619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821 stage_manifest: version: 1 country: us @@ -1896,7 +1896,9 @@ stages: format: published_estimate vintage: latest_available locator: Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration - Portal undocumented-student estimates + Portal undocumented-student estimates; OHSS refugee and asylee annual flow reports; DHS OAW/U4U + parole reports and CBP CHNV releases; CRS RS20844 TPS designations table; EOIR adjudication statistics + via CRS IN12501 operations: - kind: read_table table: person @@ -1913,21 +1915,82 @@ stages: undocumented_population_anchor: value: 11000000 source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + humanitarian_status_stocks: + paroled_one_year: + afghanistan: + target: 73566 + source: https://www.dhs.gov/sites/default/files/2023-06/PLCY%20-%20AFG%202503%20Update%202_0.pdf + ukraine: + target: 158000 + source: https://www.dhs.gov/sites/default/files/2024-12/2024_1104_dmo_plcy_uniting_for_ukraine_process_overview_and_assessment.pdf + nicaragua: + target: 93070 + source: https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update + venezuela: + target: 117330 + source: https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update + refugee: + target: 160000 + source: https://ohss.dhs.gov/sites/default/files/2024-11/2024_1108_ohss_refugee_annual_flow_report_2023.pdf + asylee: + target: 160000 + source: https://ohss.dhs.gov/sites/default/files/2024-10/2024_1002_ohss_asylees_fy2023.pdf + deportation_withheld: + target: 0 + source: https://crsreports.congress.gov/product/pdf/IN/IN12501 + tps: + venezuela: + target: 605015 + source: https://www.everycrsreport.com/reports/RS20844.html + el_salvador: + target: 170125 + source: https://www.everycrsreport.com/reports/RS20844.html + honduras: + target: 51225 + source: https://www.everycrsreport.com/reports/RS20844.html + nicaragua: + target: 2910 + source: https://www.everycrsreport.com/reports/RS20844.html + nepal: + target: 7160 + source: https://www.everycrsreport.com/reports/RS20844.html + other_designated: + target: 21215 + source: https://www.everycrsreport.com/reports/RS20844.html outputs: - ssn_card_type - immigration_status_str notes: 'Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status - indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. The CPS carries no work-authorization - variable, so among residual non-citizens, workers and students spill to NON_CITIZEN_VALID_EAD in deterministic - seeded order until the remaining undocumented worker count matches Pew''s 8.3M and the undocumented - student count matches the Higher Ed Immigration Portal''s ~408k — the only forced margins. The total - undocumented population is emergent (≈13M on the 2024 ASEC, within the range of published 2023-24 - estimates) and is gated against Pew''s 11.0M 2022 anchor with a coarse plausibility band rather than - forced; reconciling the level is the calibration lane''s job. Immigration-status tags carry only statutory - tests the data supports: DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity - plus post-1980 arrival); other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true - status — blanket REFUGEE/TPS labels are deliberately not fabricated because they would mislabel millions - and over-grant refugee-class benefit exemptions.' + indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection + statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually + exclusively in decreasing order of target precision — PAROLED_ONE_YEAR per program origin (OAW Afghans + 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and + Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS + — refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the + trailing two years), ASYLEE (FY2022-FY2024 grants 35,720+54,350+~70,000 OHSS; 88% of 2015-22 affirmative + adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS + RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El + Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin + countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere + (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility — Haiti + TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), + and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they + are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members + who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes + lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: + EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and + publishes no stock, so the accumulated population is below the CPS''s resolution for a defensible + draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE + draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive + NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), consistent with Pew''s + own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student spill then binds + the remaining residual pool to Pew''s 8.3M workers and the Higher Ed Immigration Portal''s ~408k students + — still the only forced margins. The total undocumented population is emergent and gated against Pew''s + 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are gated against their cited + targets with a coarse band (the ASEC undercovers 2022-24 arrivals — the Ukraine parole pool saturates + below its admin count); reconciling levels is the calibration lane''s job. DACA (arrival cohort among + EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; + other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status.' - stage: hours_worked survey: Census CPS ASEC source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml index e932429a6..42e100f8c 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml @@ -330,6 +330,58 @@ seed_site_bindings: owners: - kind: source_stage id: snap_abawd_discretionary_exemption +- site: immigration_humanitarian_paroled_one_year_afghanistan_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_paroled_one_year_ukraine_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_paroled_one_year_nicaragua_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_paroled_one_year_venezuela_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_refugee_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_asylee_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_deportation_withheld_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_venezuela_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_el_salvador_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_honduras_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_nicaragua_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_nepal_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_other_designated_assignment + owners: + - kind: source_stage + id: immigration_status - site: immigration_ead_workers_assignment owners: - kind: source_stage diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 465bd4a4f..9909da478 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097" + "619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/us_bundle_generation/identity_contracts.py b/tools/us_bundle_generation/identity_contracts.py index 06b2bf04d..9475b1eee 100644 --- a/tools/us_bundle_generation/identity_contracts.py +++ b/tools/us_bundle_generation/identity_contracts.py @@ -173,6 +173,26 @@ def build_seed_site_bindings( "snap_discretionary_exemption_assignment": source( "snap_abawd_discretionary_exemption" ), + **{ + f"immigration_humanitarian_{label}_assignment": source( + "immigration_status" + ) + for label in ( + "paroled_one_year_afghanistan", + "paroled_one_year_ukraine", + "paroled_one_year_nicaragua", + "paroled_one_year_venezuela", + "refugee", + "asylee", + "deportation_withheld", + "tps_venezuela", + "tps_el_salvador", + "tps_honduras", + "tps_nicaragua", + "tps_nepal", + "tps_other_designated", + ) + }, "immigration_ead_workers_assignment": source("immigration_status"), "immigration_ead_students_assignment": source("immigration_status"), "ssi_take_up_assignment": source("ssi_take_up"), @@ -203,9 +223,9 @@ def build_seed_site_bindings( } protocol_site_ids = tuple(site.id for site in LEGACY_V1_PROTOCOL.sites) - if len(protocol_site_ids) != 53 or set(owners) != set(protocol_site_ids): + if len(protocol_site_ids) != 66 or set(owners) != set(protocol_site_ids): raise RuntimeError( - "legacy-v1 seed owner ledger must cover exactly 53 protocol sites; " + "legacy-v1 seed owner ledger must cover exactly 66 protocol sites; " f"missing={sorted(set(protocol_site_ids) - owners.keys())!r}, " f"extra={sorted(owners.keys() - set(protocol_site_ids))!r}" ) From 28a733a025314581df05c4cc36dc6a204d16ec99 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 15:44:31 +0200 Subject: [PATCH 3/8] Cover the humanitarian draws, manifest validation, and gate bands with tests Refs #767 Co-Authored-By: Claude Fable 5 --- .../microcosm/build/us_runtime/immigration.py | 14 +- .../tests/test_us_immigration.py | 462 +++++++++++++++++- 2 files changed, 452 insertions(+), 24 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index 48dc63557..151ad3ebc 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -451,7 +451,9 @@ def __post_init__(self) -> None: @property def label(self) -> str: - return self.category if self.origin is None else f"{self.category}:{self.origin}" + return ( + self.category if self.origin is None else f"{self.category}:{self.origin}" + ) @property def salt(self) -> str: @@ -629,8 +631,7 @@ def _humanitarian_draws_from_parameters( missing = sorted(set(HUMANITARIAN_STATUS_CATEGORIES) - set(block)) if missing: raise SourceRuntimeError( - f"US immigration {_HUMANITARIAN_KEY} is missing category(ies): " - f"{missing}." + f"US immigration {_HUMANITARIAN_KEY} is missing category(ies): {missing}." ) draws: list[HumanitarianDraw] = [] for category in HUMANITARIAN_STATUS_CATEGORIES: @@ -879,8 +880,7 @@ def _humanitarian_draw_candidates( & (arrival_code <= max_arrival_code) ) raise SourceRuntimeError( - f"US immigration derivation has no candidate rule for draw " - f"{draw.label!r}." + f"US immigration derivation has no candidate rule for draw {draw.label!r}." ) @@ -1233,9 +1233,7 @@ def us_immigration_composition_gate( "controls": { "undocumented_workers": controls.undocumented.workers, "undocumented_students": controls.undocumented.students, - "undocumented_population_anchor": ( - controls.undocumented.population_anchor - ), + "undocumented_population_anchor": (controls.undocumented.population_anchor), "sources": dict(controls.undocumented.sources), "humanitarian_status_stocks": { draw.label: {"target": draw.target, "source": draw.source} diff --git a/packages/microcosm-build/tests/test_us_immigration.py b/packages/microcosm-build/tests/test_us_immigration.py index 23a1cb19f..3040dec19 100644 --- a/packages/microcosm-build/tests/test_us_immigration.py +++ b/packages/microcosm-build/tests/test_us_immigration.py @@ -13,6 +13,7 @@ run_source_stage, ) from microcosm.build.us_runtime import ( + HUMANITARIAN_STATUS_CATEGORIES, IMMIGRATION_STATUS_VALUES, SSN_CARD_TYPE_VALUES, US_DONORS, @@ -20,6 +21,8 @@ US_IMMIGRATION_STAGE_NAME, US_SOURCE_MANIFEST, US_STAGE_NAMES, + HumanitarianDraw, + ImmigrationControls, UndocumentedControls, derive_us_immigration_status_from_manifest, us_immigration_composition_gate, @@ -33,12 +36,56 @@ TIME_PERIOD = 2024 +_PAROLE_ORIGIN_KEYS = ("afghanistan", "ukraine", "nicaragua", "venezuela") +_TPS_ORIGIN_KEYS = ( + "venezuela", + "el_salvador", + "honduras", + "nicaragua", + "nepal", + "other_designated", +) + + +def _humanitarian_block( + *, + paroled_one_year: dict[str, float] | None = None, + refugee: float = 0.0, + asylee: float = 0.0, + deportation_withheld: float = 0.0, + tps: dict[str, float] | None = None, +) -> dict: + """Manifest-shaped humanitarian block; unnamed targets default to zero.""" + + parole_targets = dict.fromkeys(_PAROLE_ORIGIN_KEYS, 0.0) | (paroled_one_year or {}) + tps_targets = dict.fromkeys(_TPS_ORIGIN_KEYS, 0.0) | (tps or {}) + return { + "paroled_one_year": { + origin: { + "target": target, + "source": f"https://example.com/parole/{origin}", + } + for origin, target in parole_targets.items() + }, + "refugee": {"target": refugee, "source": "https://example.com/refugee"}, + "asylee": {"target": asylee, "source": "https://example.com/asylee"}, + "deportation_withheld": { + "target": deportation_withheld, + "source": "https://example.com/withheld", + }, + "tps": { + origin: {"target": target, "source": f"https://example.com/tps/{origin}"} + for origin, target in tps_targets.items() + }, + } + def _stage_spec( *, workers: float, students: float, anchor: float, + humanitarian: dict | None = None, ) -> SourceStageSpec: return SourceStageSpec.from_mapping( { @@ -64,6 +111,11 @@ def _stage_spec( "value": anchor, "source": "https://example.com/population", }, + "humanitarian_status_stocks": ( + humanitarian + if humanitarian is not None + else _humanitarian_block() + ), }, ], "outputs": list(US_IMMIGRATION_OUTPUT_COLUMNS), @@ -126,10 +178,16 @@ def _run( workers: float = 100.0, students: float = 100.0, anchor: float = 10.0, + humanitarian: dict | None = None, seed: int = 0, ) -> pd.DataFrame: return run_source_stage( - _stage_spec(workers=workers, students=students, anchor=anchor), + _stage_spec( + workers=workers, + students=students, + anchor=anchor, + humanitarian=humanitarian, + ), tables={"person": person}, operation_handlers=_HANDLERS, config=SourceRuntimeConfig(seed=seed, target_year=TIME_PERIOD), @@ -308,6 +366,269 @@ def test_emitted_values_stay_inside_engine_enum_domains(self) -> None: assert set(output["immigration_status_str"]) <= set(IMMIGRATION_STATUS_VALUES) +class TestHumanitarianDraws: + def test_parole_draw_takes_documented_origin_cohort(self) -> None: + person = _person_table( + [ + # 2022-arrived Ukrainian reporting Medicaid: the U4U signature. + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1), + # Same profile from a non-parole origin stays LPR. + _noncitizen(PENATVTY=303, PEINUSYR=28, CAID=1), + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 1.0}), + ) + assert output.loc[0, "immigration_status_str"] == "PAROLED_ONE_YEAR" + assert output.loc[0, "ssn_card_type"] == "OTHER_NON_CITIZEN" + assert output.loc[1, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_parole_origin_targets_do_not_cross(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=315, PEINUSYR=28, CAID=1), # Nicaraguan + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 5.0}), + ) + assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_parole_needs_recent_arrival(self) -> None: + # A 2014-2015-arrived Ukrainian predates every parole program. + person = _person_table([_noncitizen(PENATVTY=164, PEINUSYR=24, CAID=1)]) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 5.0}), + ) + assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_refugee_draw_excludes_residual_pool(self) -> None: + # A 2022-arrived Congolese person with no legal-status indicator is + # not refugee-consistent (refugees carry immediate benefits access). + person = _person_table( + [ + _noncitizen(PENATVTY=412, PEINUSYR=28), + _noncitizen(PENATVTY=412, PEINUSYR=28, CAID=1), + ] + ) + output = _run(person, humanitarian=_humanitarian_block(refugee=1.0)) + assert output.loc[0, "immigration_status_str"] == "UNDOCUMENTED" + assert output.loc[1, "immigration_status_str"] == "REFUGEE" + + def test_asylee_draw_uses_backlog_window(self) -> None: + person = _person_table( + [ + # 2018-2019 arrival from China with Medicaid: grant-consistent. + _noncitizen(PENATVTY=207, PEINUSYR=26, CAID=1), + # 2004-2005 arrival predates the grant backlog window. + _noncitizen(PENATVTY=207, PEINUSYR=19, CAID=1, A_AGE=50), + ] + ) + output = _run(person, humanitarian=_humanitarian_block(asylee=1.0)) + assert output.loc[0, "immigration_status_str"] == "ASYLEE" + assert output.loc[1, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_tps_residual_draw_flips_ssn_to_ead(self) -> None: + person = _person_table([_noncitizen(PENATVTY=373, PEINUSYR=28)]) + output = _run(person, humanitarian=_humanitarian_block(tps={"venezuela": 1.0})) + assert output.loc[0, "immigration_status_str"] == "TPS" + assert output.loc[0, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + none_ssn = output["ssn_card_type"] == "NONE" + undocumented = output["immigration_status_str"] == "UNDOCUMENTED" + assert (none_ssn == undocumented).all() + + def test_tps_legacy_continuous_residence_window_binds(self) -> None: + person = _person_table( + [ + # Arrived 1996-1997: inside the El Salvador 2001 cutoff. + _noncitizen(PENATVTY=312, PEINUSYR=15, CAID=1, A_AGE=55), + # Arrived 2014-2015: cannot hold El Salvador TPS. + _noncitizen(PENATVTY=312, PEINUSYR=24, CAID=1, A_AGE=40), + ] + ) + output = _run( + person, humanitarian=_humanitarian_block(tps={"el_salvador": 5.0}) + ) + assert output.loc[0, "immigration_status_str"] == "TPS" + assert output.loc[1, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_daca_cohort_is_excluded_from_humanitarian_draws(self) -> None: + # Arrived 2000-2001 aged ~7: the DACA statutory cohort. The worker + # spill gives the EAD card and the DACA tag survives TPS targeting. + person = _person_table( + [_noncitizen(PENATVTY=312, PEINUSYR=17, A_AGE=30, WSAL_VAL=20_000.0)] + ) + output = _run( + person, + workers=0.001, + humanitarian=_humanitarian_block(tps={"el_salvador": 5.0}), + ) + assert output.loc[0, "immigration_status_str"] == "DACA" + + def test_cuban_haitian_entrants_survive_humanitarian_targets(self) -> None: + person = _person_table([_noncitizen(PENATVTY=327, PEINUSYR=28, CAID=1)]) + output = _run( + person, + humanitarian=_humanitarian_block( + paroled_one_year={"venezuela": 5.0}, tps={"venezuela": 5.0} + ), + ) + assert output.loc[0, "immigration_status_str"] == "CUBAN_HAITIAN_ENTRANT" + + def test_draw_order_gives_parole_precedence_over_tps(self) -> None: + person = _person_table([_noncitizen(PENATVTY=373, PEINUSYR=28, CAID=1)]) + both = _run( + person, + humanitarian=_humanitarian_block( + paroled_one_year={"venezuela": 1.0}, tps={"venezuela": 1.0} + ), + ) + tps_only = _run( + person, humanitarian=_humanitarian_block(tps={"venezuela": 1.0}) + ) + assert both.loc[0, "immigration_status_str"] == "PAROLED_ONE_YEAR" + assert tps_only.loc[0, "immigration_status_str"] == "TPS" + + def test_draws_are_weight_targeted(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1, person_weight=3.0), + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1, person_weight=3.0), + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 3.0}), + ) + statuses = sorted(output["immigration_status_str"]) + assert statuses == ["LEGAL_PERMANENT_RESIDENT", "PAROLED_ONE_YEAR"] + + def test_zero_targets_draw_nothing(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1), + _noncitizen(PENATVTY=373, PEINUSYR=28), + _noncitizen(PENATVTY=412, PEINUSYR=28, CAID=1), + ] + ) + output = _run(person, humanitarian=_humanitarian_block()) + assert set(output["immigration_status_str"]) <= { + "LEGAL_PERMANENT_RESIDENT", + "UNDOCUMENTED", + } + + def test_humanitarian_spill_interaction_preserves_worker_control(self) -> None: + # TPS extraction removes a residual worker before the spill, so the + # remaining undocumented workers still land on the control exactly. + person = _person_table( + [_noncitizen(PENATVTY=373, PEINUSYR=28, WSAL_VAL=10_000.0)] + + [_noncitizen(WSAL_VAL=10_000.0) for _ in range(10)] + ) + output = _run( + person, + workers=4.0, + humanitarian=_humanitarian_block(tps={"venezuela": 1.0}), + ) + workers = output["WSAL_VAL"] > 0 + undocumented_workers = ((output["ssn_card_type"] == "NONE") & workers).sum() + assert output.loc[0, "immigration_status_str"] == "TPS" + assert undocumented_workers == 4 + + def test_missing_humanitarian_block_is_refused(self) -> None: + spec_mapping = { + "stage": US_IMMIGRATION_STAGE_NAME, + "survey": "test", + "source": "https://example.com", + "grain": "person", + "operations": [ + {"kind": "read_table", "table": "person"}, + { + "kind": "derive_immigration_status", + "seed_from_build_config": True, + "time_period_from_build_config": True, + "undocumented_workers": { + "target": 1, + "source": "https://example.com", + }, + "undocumented_students": { + "target": 1, + "source": "https://example.com", + }, + "undocumented_population_anchor": { + "value": 1, + "source": "https://example.com", + }, + }, + ], + "outputs": list(US_IMMIGRATION_OUTPUT_COLUMNS), + } + with pytest.raises(SourceRuntimeError, match="humanitarian_status_stocks"): + run_source_stage( + SourceStageSpec.from_mapping(spec_mapping), + tables={"person": _person_table([_noncitizen()])}, + operation_handlers=_HANDLERS, + config=SourceRuntimeConfig(seed=0, target_year=TIME_PERIOD), + ) + + @pytest.mark.parametrize( + "mutate, match", + [ + (lambda block: block.pop("refugee"), "missing category"), + ( + lambda block: block.__setitem__( + "mystery", {"target": 1, "source": "https://example.com"} + ), + "unsupported category", + ), + ( + lambda block: block["paroled_one_year"].pop("ukraine"), + "missing origin", + ), + ( + lambda block: block["tps"].__setitem__( + "atlantis", {"target": 1, "source": "https://example.com"} + ), + "outside the stage's codebook table", + ), + ( + lambda block: block["refugee"].pop("source"), + "source citation", + ), + ( + lambda block: block["refugee"].__setitem__("target", -1), + "non-negative", + ), + ], + ) + def test_malformed_humanitarian_blocks_are_refused(self, mutate, match) -> None: + block = _humanitarian_block(refugee=1.0) + mutate(block) + with pytest.raises(SourceRuntimeError, match=match): + _run(_person_table([_noncitizen()]), humanitarian=block) + + def test_humanitarian_clones_stay_consistent(self) -> None: + rows = [ + _noncitizen( + PENATVTY=164, + PEINUSYR=28, + CAID=1, + person_id=index + 1, + source_year=2024, + source_person_id=f"P{index % 5}", + ) + for index in range(10) + ] + output = _run( + _person_table(rows), + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 5.0}), + ) + by_source = output.groupby("source_person_id")["immigration_status_str"] + assert (by_source.nunique() == 1).all() + + class TestDeterminism: def _worker_pool(self) -> pd.DataFrame: return _person_table([_noncitizen(WSAL_VAL=10_000.0) for _ in range(20)]) @@ -363,6 +684,38 @@ def test_manifest_controls_carry_citations(self) -> None: assert float(block[value_key]) > 0 assert str(block["source"]).startswith("https://") + def test_manifest_humanitarian_stocks_carry_citations(self) -> None: + humanitarian = ( + us_immigration_stage_spec() + .operations[1] + .parameters["humanitarian_status_stocks"] + ) + assert set(humanitarian) == set(HUMANITARIAN_STATUS_CATEGORIES) + flat_blocks = { + "refugee": humanitarian["refugee"], + "asylee": humanitarian["asylee"], + "deportation_withheld": humanitarian["deportation_withheld"], + **{ + f"paroled_one_year:{origin}": block + for origin, block in humanitarian["paroled_one_year"].items() + }, + **{f"tps:{origin}": block for origin, block in humanitarian["tps"].items()}, + } + assert set(humanitarian["paroled_one_year"]) == set(_PAROLE_ORIGIN_KEYS) + assert set(humanitarian["tps"]) == set(_TPS_ORIGIN_KEYS) + for label, block in flat_blocks.items(): + assert float(block["target"]) >= 0, label + assert str(block["source"]).startswith("https://"), label + # The withheld stock is an explicit cited zero (no published stock); + # every other category carries a positive stock. + assert float(humanitarian["deportation_withheld"]["target"]) == 0 + positive = [ + label + for label, block in flat_blocks.items() + if label != "deportation_withheld" + ] + assert all(float(flat_blocks[label]["target"]) > 0 for label in positive) + def test_unexpected_parameter_is_refused(self) -> None: spec = SourceStageSpec.from_mapping( { @@ -521,16 +874,41 @@ def test_missing_raw_columns_raise_loudly(self) -> None: with_us_immigration_inputs(stripped, seed=0, time_period=TIME_PERIOD) -def _plausible_controls() -> UndocumentedControls: - return UndocumentedControls( - workers=20.0, - students=5.0, - population_anchor=30.0, - sources={ - "undocumented_workers": "https://example.com/workers", - "undocumented_students": "https://example.com/students", - "undocumented_population_anchor": "https://example.com/population", - }, +def _humanitarian_draws(**targets: float) -> tuple[HumanitarianDraw, ...]: + """Zero-target draw per category, overridden by keyword (national sums).""" + + status_by_category = { + "paroled_one_year": "PAROLED_ONE_YEAR", + "refugee": "REFUGEE", + "asylee": "ASYLEE", + "deportation_withheld": "DEPORTATION_WITHHELD", + "tps": "TPS", + } + return tuple( + HumanitarianDraw( + category=category, + origin=None, + status=status_by_category[category], + target=float(targets.get(category, 0.0)), + source=f"https://example.com/{category}", + ) + for category in HUMANITARIAN_STATUS_CATEGORIES + ) + + +def _plausible_controls(**humanitarian_targets: float) -> ImmigrationControls: + return ImmigrationControls( + undocumented=UndocumentedControls( + workers=20.0, + students=5.0, + population_anchor=30.0, + sources={ + "undocumented_workers": "https://example.com/workers", + "undocumented_students": "https://example.com/students", + "undocumented_population_anchor": "https://example.com/population", + }, + ), + humanitarian=_humanitarian_draws(**humanitarian_targets), ) @@ -540,6 +918,7 @@ def _composition_frame( other: int = 30, ead: int = 10, none: int = 30, + humanitarian: dict[str, int] | None = None, ) -> Frame: rows: list[dict] = [] values: list[tuple[str, str]] = ( @@ -547,6 +926,11 @@ def _composition_frame( + [("OTHER_NON_CITIZEN", "LEGAL_PERMANENT_RESIDENT")] * other + [("NON_CITIZEN_VALID_EAD", "LEGAL_PERMANENT_RESIDENT")] * ead + [("NONE", "UNDOCUMENTED")] * none + + [ + ("OTHER_NON_CITIZEN", status) + for status, count in (humanitarian or {}).items() + for _ in range(count) + ] ) for ssn, status in values: rows.append( @@ -616,11 +1000,14 @@ def test_fails_when_undocumented_far_from_anchor(self) -> None: def test_fails_when_non_citizen_share_implausible(self) -> None: gate = us_immigration_composition_gate( _composition_frame(citizens=40, other=20, ead=20, none=20), - controls=UndocumentedControls( - workers=8.0, - students=1.0, - population_anchor=20.0, - sources=_plausible_controls().sources, + controls=ImmigrationControls( + undocumented=UndocumentedControls( + workers=8.0, + students=1.0, + population_anchor=20.0, + sources=_plausible_controls().undocumented.sources, + ), + humanitarian=_humanitarian_draws(), ), ) assert not gate.passed @@ -632,6 +1019,49 @@ def test_gate_reads_packaged_controls_by_default(self) -> None: controls = gate.details["controls"] assert controls["undocumented_workers"] == 8_300_000 assert controls["undocumented_population_anchor"] == 11_000_000 + stocks = controls["humanitarian_status_stocks"] + assert stocks["paroled_one_year:afghanistan"]["target"] == 73_566 + assert stocks["tps:venezuela"]["target"] == 605_015 + assert stocks["deportation_withheld"]["target"] == 0 + + def test_gate_passes_with_in_band_humanitarian_masses(self) -> None: + gate = us_immigration_composition_gate( + _composition_frame(humanitarian={"REFUGEE": 4, "TPS": 6}), + controls=_plausible_controls(refugee=4.0, tps=6.0), + ) + assert gate.passed, gate.failures + achieved = gate.details["humanitarian_achieved"] + assert achieved["refugee"]["population"] == 4.0 + assert achieved["refugee"]["relative"] == 1.0 + assert achieved["tps"]["target"] == 6.0 + + def test_gate_fails_when_humanitarian_category_collapses(self) -> None: + gate = us_immigration_composition_gate( + _composition_frame(), + controls=_plausible_controls(refugee=4.0), + ) + assert not gate.passed + assert any( + "REFUGEE" in failure and "degenerate" in failure + for failure in gate.failures + ) + + def test_gate_fails_when_explicit_zero_category_emits(self) -> None: + gate = us_immigration_composition_gate( + _composition_frame(humanitarian={"DEPORTATION_WITHHELD": 2}), + controls=_plausible_controls(), + ) + assert not gate.passed + assert any("explicit zero target" in failure for failure in gate.failures) + + def test_gate_accepts_saturated_draws_inside_band(self) -> None: + # The ASEC undercovers 2022-24 arrivals; a draw that saturates at + # roughly three-quarters of its admin target stays inside the band. + gate = us_immigration_composition_gate( + _composition_frame(humanitarian={"PAROLED_ONE_YEAR": 3}), + controls=_plausible_controls(paroled_one_year=4.0), + ) + assert gate.passed, gate.failures def test_summary_reports_weighted_composition(self) -> None: summary = us_immigration_composition_summary( From 889b13fda622dcb60c9b8dfd8106831ffaab5b86 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 15:48:02 +0200 Subject: [PATCH 4/8] Pin four H.R.1 immigration reform-coverage probes on the release Restorations of the pre-H.R.1 Medicaid, SNAP, and ACA lawful-presence lists and the below-FPL PTC exception must each move their budget measure on the export; a ~$0 score means the humanitarian statuses regressed and blocks publication. Refs #767 Co-Authored-By: Claude Fable 5 --- ...nitarian-immigration-statuses-767.added.md | 1 + .../us/release_input_coverage_manifest.json | 97 +++++++++++++ ...uild_us_release_input_coverage_manifest.py | 131 ++++++++++++++++++ 3 files changed, 229 insertions(+) create mode 100644 changelog.d/us-humanitarian-immigration-statuses-767.added.md diff --git a/changelog.d/us-humanitarian-immigration-statuses-767.added.md b/changelog.d/us-humanitarian-immigration-statuses-767.added.md new file mode 100644 index 000000000..86176b426 --- /dev/null +++ b/changelog.d/us-humanitarian-immigration-statuses-767.added.md @@ -0,0 +1 @@ +Impute humanitarian and temporary-protection immigration statuses (REFUGEE, ASYLEE, PAROLED_ONE_YEAR per program origin, TPS per country) in the US immigration stage, drawn to cited DHS/OHSS/CBP/CRS stocks, so the enacted H.R.1 §71109/§71301/§71302 and SNAP §10108 eligibility channels stop computing $0; gate each category against its target and add four release-blocking reform-coverage probes. diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index b352ee60e..d93bdf47d 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -1414,6 +1414,103 @@ "parameter_changes": {}, "period": 2024, "reason": "PolicyEngine-US multiplies HUD HAP by the restored SPM-unit take-up leaf after eligibility. Microcosm keeps that leaf exactly equal to source-backed housing-assistance receipt, so neutralizing it must remove the assistance paid to measured/imputed recipients. A 6,000-household production-ingredient smoke scored $202.795 million baseline-minus-neutralized; the $100 million floor is below that observed subset effect but far above numerical noise. A default-only or absent carry makes the source reconciliation or this uniquely isolating probe fail." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "medicaid", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_medicaid_humanitarian_eligibility_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 50000000.0, + "name": "H.R.1 SS71109 Medicaid humanitarian-status restoration", + "parameter_changes": { + "gov.hhs.medicaid.eligibility.eligible_immigration_statuses": { + "2026-10-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR" + ] + } + }, + "period": 2027, + "reason": "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01; the probe restores the pre-narrowing list at 2027 law, which binds only through the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR values of immigration_status_str. The source stage draws roughly 160k refugees (74% reporting Medicaid on the 2024 ASEC), 160k asylees, and 405k parolees to cited DHS/OHSS stocks, so restoring their eligibility must re-enroll anchored takers and move person-level medicaid dollars by far more than the floor. A ~$0 score means the humanitarian statuses regressed to zero records (microcosm #767's silent-zero failure) or the engine channel broke." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "aca_ptc", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_aca_below_fpl_exception_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 5000000.0, + "name": "H.R.1 SS71302 ACA below-FPL immigrant exception restoration", + "parameter_changes": { + "gov.aca.below_fpl_immigration_exception_in_effect": { + "2026-01-01.2026-12-31": true + } + }, + "period": 2026, + "reason": "H.R.1 SS71302 repeals the 26 USC 36B(c)(1)(B) below-FPL lawfully-present exception from 2026. Restoring it for 2026 binds, at 2026 law, only through tax units containing a non-citizen who is ACA-lawfully-present yet Medicaid-ineligible by status - on this file exactly the TPS population the source stage draws to the CRS RS20844 per-country stocks (865k weighted persons; the other humanitarian classes remain Medicaid-qualified at 2026 annual parameters). Below-poverty TPS units with marketplace take-up must regain premium tax credits above the floor; ~$0 means the TPS records vanished or the exception channel broke." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "aca_ptc", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_aca_lawful_presence_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 10000000.0, + "name": "H.R.1 SS71301 ACA lawful-presence list restoration", + "parameter_changes": { + "gov.aca.ineligible_immigration_statuses": { + "2027-01-01.2100-12-31": [ + "DACA", + "UNDOCUMENTED" + ] + } + }, + "period": 2027, + "reason": "H.R.1 SS71301 adds TPS/REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR to the ACA ineligible-status list from 2027-01-01. The probe restores the pre-H.R.1 list (DACA and UNDOCUMENTED only) at 2027, re-qualifying every humanitarian status the source stage draws (~1.6M weighted persons across parole, refugee, asylee, and TPS). Units among them with marketplace take-up must regain premium tax credits above the floor; ~$0 means the humanitarian statuses regressed or the lawful-presence channel broke." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "snap", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_snap_humanitarian_eligibility_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 25000000.0, + "name": "H.R.1 SS10108 SNAP humanitarian-status restoration", + "parameter_changes": { + "gov.usda.snap.eligibility.eligible_immigration_statuses": { + "2025-07-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR" + ] + } + }, + "period": 2026, + "reason": "H.R.1 SS10108 narrows SNAP alien eligibility to citizens, LPRs, Cuban/Haitian entrants, and COFA citizens from 2025-07-01 (modeled 2025-07). Restoring the pre-H.R.1 list at 2026 re-includes the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR members the source stage draws to cited DHS/OHSS stocks; is_snap_excluded_member stops excluding them, so SNAP allotments for their households must rise above the floor. ~$0 means the humanitarian statuses regressed to zero records or the SNAP immigration channel broke." } ], "schema_version": 1, diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index fbb5b03c0..10043a98f 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -1406,6 +1406,137 @@ ), "issue": "PolicyEngine/microcosm#312", }, + { + "id": "hr1_medicaid_humanitarian_eligibility_restoration", + "name": "H.R.1 SS71109 Medicaid humanitarian-status restoration", + "parameter_changes": { + "gov.hhs.medicaid.eligibility.eligible_immigration_statuses": { + "2026-10-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR", + ] + } + }, + "budget_measure": "medicaid", + "period": 2027, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 50_000_000.0, + "reason": ( + "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to " + "citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01; the " + "probe restores the pre-narrowing list at 2027 law, which binds " + "only through the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" + "PAROLED_ONE_YEAR values of immigration_status_str. The source " + "stage draws roughly 160k refugees (74% reporting Medicaid on the " + "2024 ASEC), 160k asylees, and 405k parolees to cited DHS/OHSS " + "stocks, so restoring their eligibility must re-enroll anchored " + "takers and move person-level medicaid dollars by far more than " + "the floor. A ~$0 score means the humanitarian statuses regressed " + "to zero records (microcosm #767's silent-zero failure) or the " + "engine channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, + { + "id": "hr1_aca_below_fpl_exception_restoration", + "name": "H.R.1 SS71302 ACA below-FPL immigrant exception restoration", + "parameter_changes": { + "gov.aca.below_fpl_immigration_exception_in_effect": { + "2026-01-01.2026-12-31": True + } + }, + "budget_measure": "aca_ptc", + "period": 2026, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 5_000_000.0, + "reason": ( + "H.R.1 SS71302 repeals the 26 USC 36B(c)(1)(B) below-FPL " + "lawfully-present exception from 2026. Restoring it for 2026 " + "binds, at 2026 law, only through tax units containing a " + "non-citizen who is ACA-lawfully-present yet " + "Medicaid-ineligible by status - on this file exactly the TPS " + "population the source stage draws to the CRS RS20844 per-country " + "stocks (865k weighted persons; the other humanitarian classes " + "remain Medicaid-qualified at 2026 annual parameters). " + "Below-poverty TPS units with marketplace take-up must regain " + "premium tax credits above the floor; ~$0 means the TPS records " + "vanished or the exception channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, + { + "id": "hr1_aca_lawful_presence_restoration", + "name": "H.R.1 SS71301 ACA lawful-presence list restoration", + "parameter_changes": { + "gov.aca.ineligible_immigration_statuses": { + "2027-01-01.2100-12-31": ["DACA", "UNDOCUMENTED"] + } + }, + "budget_measure": "aca_ptc", + "period": 2027, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 10_000_000.0, + "reason": ( + "H.R.1 SS71301 adds TPS/REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" + "PAROLED_ONE_YEAR to the ACA ineligible-status list from " + "2027-01-01. The probe restores the pre-H.R.1 list (DACA and " + "UNDOCUMENTED only) at 2027, re-qualifying every humanitarian " + "status the source stage draws (~1.6M weighted persons across " + "parole, refugee, asylee, and TPS). Units among them with " + "marketplace take-up must regain premium tax credits above the " + "floor; ~$0 means the humanitarian statuses regressed or the " + "lawful-presence channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, + { + "id": "hr1_snap_humanitarian_eligibility_restoration", + "name": "H.R.1 SS10108 SNAP humanitarian-status restoration", + "parameter_changes": { + "gov.usda.snap.eligibility.eligible_immigration_statuses": { + "2025-07-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR", + ] + } + }, + "budget_measure": "snap", + "period": 2026, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 25_000_000.0, + "reason": ( + "H.R.1 SS10108 narrows SNAP alien eligibility to citizens, LPRs, " + "Cuban/Haitian entrants, and COFA citizens from 2025-07-01 " + "(modeled 2025-07). Restoring the pre-H.R.1 list at 2026 " + "re-includes the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" + "PAROLED_ONE_YEAR members the source stage draws to cited " + "DHS/OHSS stocks; is_snap_excluded_member stops excluding them, " + "so SNAP allotments for their households must rise above the " + "floor. ~$0 means the humanitarian statuses regressed to zero " + "records or the SNAP immigration channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, ] From d5def92a1b9aba40b1586c7036b4b028de6a9269 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 22:26:26 +0200 Subject: [PATCH 5/8] Prove the four H.R.1 probe channels on synthetic households and re-pin digests test_us_hr1_immigration_channels builds each probe's reform against the locked engine and shows nonzero, sign-correct movement per channel (medicaid, aca_ptc twice, snap) so the release smoke gate's probes have a PR-CI early warning. The thirteen humanitarian draw sites join EXPECTED_LEGACY_V1_SITES and the audited-source manifest; BE/UK envelope digests re-pin because the shared seed-protocol source is attested. Refs #767 Co-Authored-By: Claude Fable 5 --- .../tests/test_spec_engine_country_bundles.py | 4 +- .../tests/test_spec_engine_seeds.py | 26 +++ .../tests/test_us_hr1_immigration_channels.py | 172 ++++++++++++++++++ 3 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 packages/microcosm-build/tests/test_us_hr1_immigration_channels.py diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 5ede29d29..2e045a409 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -32,7 +32,7 @@ [ ( "be", - "86143c1c3f98980e34490c75706fe7dbf72e96e8accb8d7d4bbd2c1ae1a29b65", + "29e0820ad7779164794f81671a2e51c799ffad601410e7261a59e5d2e0fdcf25", { "household.household_id", "person.person_id", @@ -42,7 +42,7 @@ ), ( "uk", - "e3b171bdaae1bdfe842e61c129fd279a6b29001cdef4c18de51823768911ec8c", + "8a4da147cecd416f9a475b929b80a7010778df9902cd64dd149a5995bc59650d", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_seeds.py b/packages/microcosm-build/tests/test_spec_engine_seeds.py index ec1a50ab1..fb0e9a3cb 100644 --- a/packages/microcosm-build/tests/test_spec_engine_seeds.py +++ b/packages/microcosm-build/tests/test_spec_engine_seeds.py @@ -43,6 +43,19 @@ "housing_inputs_training_cap", "immigration_ead_students_assignment", "immigration_ead_workers_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_other_designated_assignment", + "immigration_humanitarian_tps_venezuela_assignment", "legacy_congressional_district_assignment", "legacy_geography_ladder", "legacy_puma_ladder", @@ -140,6 +153,19 @@ ( "immigration_ead_workers_assignment", "immigration_ead_students_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", ), "packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py", ), diff --git a/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py b/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py new file mode 100644 index 000000000..a62550aa9 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py @@ -0,0 +1,172 @@ +"""The four H.R.1 immigration probes bind through the engine (microcosm #767). + +The release-blocking reform-coverage smoke gate scores the pinned probe +reforms on the full export; these tests prove the same channels move, +nonzero and sign-correct, on synthetic households whose +``immigration_status_str`` carries the humanitarian values the source stage +now imputes. A probe whose channel dies here would score $0 on every +release, so this is the PR-CI early warning for the engine side of the +silent-zero failure — without needing restricted microdata. + +Each situation pins the take-up flag its program reads (the same +data-seeded surface the release carries) and, for the ACA channels, an +``slcsp`` override so no rating-area geography is required. +""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("policyengine_us") + +from policyengine_us import Simulation # noqa: E402 + +from microcosm.build.us_runtime.release_input_coverage import ( # noqa: E402 + us_release_reform_coverage_probes, +) +from microcosm.build.us_runtime.reform_coverage_smoke import ( # noqa: E402 + _build_reform, +) + +_HR1_PROBE_IDS = ( + "hr1_medicaid_humanitarian_eligibility_restoration", + "hr1_aca_below_fpl_exception_restoration", + "hr1_aca_lawful_presence_restoration", + "hr1_snap_humanitarian_eligibility_restoration", +) + + +@pytest.fixture(scope="module") +def hr1_probes(): + probes = { + probe.id: probe + for probe in us_release_reform_coverage_probes() + if probe.id.startswith("hr1_") + } + assert set(probes) == set(_HR1_PROBE_IDS) + return probes + + +def _household( + status: str, + income: float, + year: int, + *, + person: dict | None = None, + tax_unit: dict | None = None, + spm_unit: dict | None = None, +) -> dict: + people = { + "parent": { + "age": {year: 35}, + "employment_income": {year: income}, + "immigration_status_str": {year: status}, + **(person or {}), + }, + "child": { + "age": {year: 8}, + "immigration_status_str": {year: status}, + **(person or {}), + }, + } + return { + "people": people, + "families": {"f": {"members": ["parent", "child"]}}, + "marital_units": {"m": {"members": ["parent"]}}, + "tax_units": {"t": {"members": ["parent", "child"], **(tax_unit or {})}}, + "spm_units": {"s": {"members": ["parent", "child"], **(spm_unit or {})}}, + "households": { + "h": {"members": ["parent", "child"], "state_name": {year: "TX"}} + }, + } + + +def _effect(probe, situation: dict, year: int) -> tuple[float, float]: + baseline = Simulation(situation=situation) + reformed = Simulation(situation=situation, reform=_build_reform(probe)) + base = float(baseline.calculate(probe.budget_measure, year).sum()) + reform = float(reformed.calculate(probe.budget_measure, year).sum()) + return base, reform + + +def test_medicaid_restoration_reenrolls_refugee_family(hr1_probes) -> None: + # At 2027 law refugees are outside the §71109-narrowed qualified list; + # restoring the pre-H.R.1 list must re-enroll the anchored taker. + probe = hr1_probes["hr1_medicaid_humanitarian_eligibility_restoration"] + situation = _household( + "REFUGEE", + 25_000.0, + 2027, + person={"takes_up_medicaid_if_eligible": {2027: True}}, + ) + base, reform = _effect(probe, situation, 2027) + assert base == 0.0 + assert reform > 0.0 + + +def test_below_fpl_exception_restoration_pays_ptc_to_tps_unit(hr1_probes) -> None: + # A below-poverty TPS unit is lawfully present for the ACA at 2026 law + # yet Medicaid-ineligible by status — exactly the §71302 population. + probe = hr1_probes["hr1_aca_below_fpl_exception_restoration"] + situation = _household( + "TPS", + 12_000.0, + 2026, + tax_unit={ + "takes_up_aca_if_eligible": {2026: True}, + "slcsp": {2026: 8_000.0}, + }, + ) + base, reform = _effect(probe, situation, 2026) + assert base == 0.0 + assert reform > 0.0 + + +def test_lawful_presence_restoration_requalifies_asylee_unit(hr1_probes) -> None: + # At 2027 law §71301 adds asylees to the ACA ineligible list; restoring + # the pre-H.R.1 list (DACA/UNDOCUMENTED only) must re-qualify the unit. + probe = hr1_probes["hr1_aca_lawful_presence_restoration"] + situation = _household( + "ASYLEE", + 35_000.0, + 2027, + tax_unit={ + "takes_up_aca_if_eligible": {2027: True}, + "slcsp": {2027: 8_000.0}, + }, + ) + base, reform = _effect(probe, situation, 2027) + assert base == 0.0 + assert reform > 0.0 + + +def test_snap_restoration_reincludes_refugee_members(hr1_probes) -> None: + # At 2026 law §10108 makes refugees excluded SNAP members; restoring the + # pre-H.R.1 list must raise the household allotment. + probe = hr1_probes["hr1_snap_humanitarian_eligibility_restoration"] + situation = _household( + "REFUGEE", + 20_000.0, + 2026, + spm_unit={"takes_up_snap_if_eligible": {2026: True}}, + ) + base, reform = _effect(probe, situation, 2026) + assert reform > base + + +def test_probe_magnitude_floors_sit_far_below_plausible_scale(hr1_probes) -> None: + # The floors are release-scale guards: far above numerical noise, far + # below the plausible aggregate effect of the imputed stocks (about + # 1.6M weighted persons across the humanitarian categories). + floors = { + "hr1_medicaid_humanitarian_eligibility_restoration": 50_000_000.0, + "hr1_aca_below_fpl_exception_restoration": 5_000_000.0, + "hr1_aca_lawful_presence_restoration": 10_000_000.0, + "hr1_snap_humanitarian_eligibility_restoration": 25_000_000.0, + } + for probe_id, floor in floors.items(): + probe = hr1_probes[probe_id] + assert probe.min_abs_effect == floor + assert probe.expected_sign == "positive" + assert probe.binding_inputs == ("immigration_status_str",) + assert probe.issue == "PolicyEngine/microcosm#767" From 44d6ae486be60c2a967a7979b46f731ffdbd0de0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 22:30:48 +0200 Subject: [PATCH 6/8] Format the touched tooling and fix the stale site-count docstring Refs #767 Co-Authored-By: Claude Fable 5 --- .../tests/test_us_hr1_immigration_channels.py | 6 +++--- tools/us_bundle_generation/identity_contracts.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py b/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py index a62550aa9..ef3c2dfab 100644 --- a/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py +++ b/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py @@ -21,12 +21,12 @@ from policyengine_us import Simulation # noqa: E402 -from microcosm.build.us_runtime.release_input_coverage import ( # noqa: E402 - us_release_reform_coverage_probes, -) from microcosm.build.us_runtime.reform_coverage_smoke import ( # noqa: E402 _build_reform, ) +from microcosm.build.us_runtime.release_input_coverage import ( # noqa: E402 + us_release_reform_coverage_probes, +) _HR1_PROBE_IDS = ( "hr1_medicaid_humanitarian_eligibility_restoration", diff --git a/tools/us_bundle_generation/identity_contracts.py b/tools/us_bundle_generation/identity_contracts.py index 9475b1eee..8173dabf3 100644 --- a/tools/us_bundle_generation/identity_contracts.py +++ b/tools/us_bundle_generation/identity_contracts.py @@ -102,7 +102,7 @@ def _stages_with_operation( def build_seed_site_bindings( source_document: Mapping[str, Any], ) -> list[dict[str, object]]: - """Bind all 53 legacy-v1 sites to their concrete execution owners.""" + """Bind all 66 legacy-v1 sites to their concrete execution owners.""" source_ids = _source_stage_ids(source_document) transfer_nodes = tuple(group.name for group in CANONICAL_US_LATE_TRANSFER_GROUPS) @@ -174,9 +174,7 @@ def build_seed_site_bindings( "snap_abawd_discretionary_exemption" ), **{ - f"immigration_humanitarian_{label}_assignment": source( - "immigration_status" - ) + f"immigration_humanitarian_{label}_assignment": source("immigration_status") for label in ( "paroled_one_year_afghanistan", "paroled_one_year_ukraine", From 4c6eb44da8a0359740b5d57c7d7732a0c79b7389 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 22:33:55 +0200 Subject: [PATCH 7/8] Re-pin the country envelope digests on the merged tree Main's #743 moved the UK bundle while this branch moves the shared seed protocol; the merged UK digest combines both, BE carries only the protocol move. Refs #767 Co-Authored-By: Claude Fable 5 --- .../microcosm-build/tests/test_spec_engine_country_bundles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index cc672a159..de42902d2 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -32,7 +32,7 @@ [ ( "be", - "86143c1c3f98980e34490c75706fe7dbf72e96e8accb8d7d4bbd2c1ae1a29b65", + "29e0820ad7779164794f81671a2e51c799ffad601410e7261a59e5d2e0fdcf25", { "household.household_id", "person.person_id", @@ -42,7 +42,7 @@ ), ( "uk", - "0c85845b4d463638ae3e5c5a25e17de8b720794e3653c5991dc4f069d95762d3", + "9b00257e0b36c3f34d81581b707c6dd042b786f29851a1bc2399dfc6fbb3b97f", { "benunit.benunit_id", "household.household_id", From cf066ab63a827022b199a38ce24b9d641158c848 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 26 Aug 2026 07:48:00 +0200 Subject: [PATCH 8/8] Re-pin the seed-site, field-count, and claim attestations for the new draws The thirteen humanitarian sites move every closed-world census the fail-closed machinery keeps: protocol site counts (53 -> 66), the owner-link sum (112 -> 125, one owner each, distinct owners unchanged), the configuration field universe (42,120 -> 42,419: +13 legacy-behavior target fields, +286 attestation-only resolved seed fields), four pointer claims (source_stages, spine/resolved seed-site bindings, resolved seed protocol), and the source_stages byte attestation in the bundle tests. Refs #767 Co-Authored-By: Claude Fable 5 --- .../build/spec_engine/field_usage.py | 22 +++++++++---------- .../tests/test_spec_engine_compiler_ir.py | 6 ++--- .../tests/test_spec_engine_field_usage.py | 16 +++++++------- .../test_spec_engine_identity_contracts.py | 2 +- .../tests/test_spec_engine_plan_lock.py | 2 +- .../tests/test_us_bundle_core_contracts.py | 2 +- .../tests/test_us_spec_bundle.py | 4 ++-- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index a5dd282ab..89f230faf 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_351 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_769 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_120 +EXPECTED_AUTHORED_FIELD_COUNT = 32_403 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_016 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_419 class FieldUsageError(AssertionError): @@ -457,12 +457,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "6a781915fd491d2c4b16d2b7d482f69cf362c904130093c59f9629f7a319269b", ), "resolved_seed_protocol": ( - 823, - "c05ebb88538195798e9365f325381c14d92b74007f846f1fef50a08268e6d27d", + 1_031, + "1a8f3e74018287f184cf171cc4e6571490ba1de419cf0121a611fa93a9b7fa17", ), "resolved_seed_site_bindings": ( - 277, - "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + 316, + "199ec98045871070860164b522892327ab840b48c0c6a53972ebc0ff0d7d091f", ), "resolved_vintage_authorities": ( 63, @@ -489,8 +489,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_702, - "be3e24b53955b9dabccfc7e779212b1e97daf0c0992be3c183e391bf695ed4ee", + 1_715, + "78227f34c3f4ef946c06693dec9177dd906069f98016854bfd4d39c4cd4c51fa", ), "spine_assembly_household_mass_shares": ( 2, @@ -517,8 +517,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "29c6c1b3243e178783e7ab139993ba3a9b42d62edd4e9e4e1f3b28688daf2c6d", ), "spine_seed_site_bindings": ( - 277, - "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + 316, + "bb848f38e068858ac23e964a4ffbbf5542253da50515c72fe7753d8c56912486", ), "spine_support_roles": ( 29, diff --git a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py index 393d98dd1..c669fa342 100644 --- a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py @@ -109,9 +109,9 @@ def test_us_seed_stream_map_is_complete_and_owner_typed( ) -> None: seed_map = compiled_us.seed_stream_map assert seed_map.protocol_id == "legacy-v1" - assert len(seed_map.sites) == 53 + assert len(seed_map.sites) == 66 assert len(seed_map.owners) == 54 - assert sum(len(site.owners) for site in seed_map.sites) == 112 + assert sum(len(site.owners) for site in seed_map.sites) == 125 assert {site.id for site in seed_map.sites} == { site.id for site in resolved_us.seed_protocol.sites } @@ -250,5 +250,5 @@ def test_bundle_without_imputation_compiles_empty_graph( assert compiled.producer_graph.nodes == () assert compiled.stage_dag.nodes == () assert compiled.nodes == () - assert len(compiled.seed_stream_map.sites) == 53 + assert len(compiled.seed_stream_map.sites) == 66 assert compiled.seed_stream_map.owners == () diff --git a/packages/microcosm-build/tests/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/test_spec_engine_field_usage.py index 77fad4942..04e118087 100644 --- a/packages/microcosm-build/tests/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/test_spec_engine_field_usage.py @@ -92,22 +92,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_120 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_419 assert field_ledger.source_counts == { - "authored": 32_351, - "resolved_bindings": 9_769, + "authored": 32_403, + "resolved_bindings": 10_016, } assert field_ledger.mode_counts == { - "legacy_behavior": 13_970, - "compiler_semantic": 27_699, + "legacy_behavior": 13_983, + "compiler_semantic": 27_985, "front_end_validation": 348, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_443, - "no_generation0_effect": 3_677, + "legacy_behavior": 38_456, + "no_generation0_effect": 3_963, } - assert len({field.pointer for field in field_ledger.fields}) == 42_120 + assert len({field.pointer for field in field_ledger.fields}) == 42_419 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py b/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py index a2a682f68..4849a2bf4 100644 --- a/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py +++ b/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py @@ -124,7 +124,7 @@ def test_all_53_seed_sites_resolve_to_typed_real_owners( spine, source_stages = identity_documents bindings = _resolve(spine, source_stages) - assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 53 + assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 66 assert [binding.site for binding in bindings] == [ site.id for site in LEGACY_V1_PROTOCOL.sites ] diff --git a/packages/microcosm-build/tests/test_spec_engine_plan_lock.py b/packages/microcosm-build/tests/test_spec_engine_plan_lock.py index a0cd91c4d..6764e2437 100644 --- a/packages/microcosm-build/tests/test_spec_engine_plan_lock.py +++ b/packages/microcosm-build/tests/test_spec_engine_plan_lock.py @@ -42,7 +42,7 @@ def test_plan_lock_is_complete_and_closed_world(compiled_us: CompiledSpecIR) -> assert payload["spec_binding"]["attestation"] == "mirror-attested" assert len(payload["producer_graph"]["nodes"]) == 38 assert len(payload["producer_graph"]["authored"]["ownership_matrix"]) == 18 - assert len(payload["seed_stream_map"]["sites"]) == 53 + assert len(payload["seed_stream_map"]["sites"]) == 66 assert len(payload["nodes"]) == 38 mutated = copy.deepcopy(payload) diff --git a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py index 1ab095883..74ad1b285 100644 --- a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py @@ -129,7 +129,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097", + "sha256": "619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/test_us_spec_bundle.py b/packages/microcosm-build/tests/test_us_spec_bundle.py index 586907804..7db19defb 100644 --- a/packages/microcosm-build/tests/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/test_us_spec_bundle.py @@ -190,7 +190,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097" + "619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" @@ -917,7 +917,7 @@ def test_legacy_seed_vintage_and_publication_grammars_are_pinned( "identity_generation": 1, "seed_protocol": LEGACY_V1_PROTOCOL.id, } - assert len(LEGACY_V1_PROTOCOL.sites) == 53 + assert len(LEGACY_V1_PROTOCOL.sites) == 66 assert len(LEGACY_V1_PROTOCOL.streams) == 14 assert LEGACY_V1_PROTOCOL.site("survey_sample_asec").default == 578 assert LEGACY_V1_PROTOCOL.site("puf_live_aggregate_disaggregation").default == 0