From 4f769f5fc23184f3a71eeb1d2c1958378b27588e Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 13 Jul 2026 19:24:06 -0400 Subject: [PATCH] test(lwdid): independent methodology-validation suite (skip-gated until PR #588 lands) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FZK3FD9jxWGxBrPDw5APSg --- TODO.md | 3 +- .../data/lwdid_walmart_eventstudy_golden.json | 637 ++++++++++++ benchmarks/data/real/castle_lw_subset.csv | 551 +++++++++++ docs/methodology/REGISTRY.md | 1 + tests/test_methodology_lwdid.py | 912 ++++++++++++++++++ 5 files changed, 2103 insertions(+), 1 deletion(-) create mode 100644 benchmarks/data/lwdid_walmart_eventstudy_golden.json create mode 100644 benchmarks/data/real/castle_lw_subset.csv create mode 100644 tests/test_methodology_lwdid.py diff --git a/TODO.md b/TODO.md index f914a0c97..f757211d3 100644 --- a/TODO.md +++ b/TODO.md @@ -50,7 +50,8 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m | ChangesInChanges/QDiD tutorial notebook (2x2 distributional walkthrough: QTE grid, interior range, uniform bands, CiC-vs-QDiD comparison) - deferred from the implementation PR as a documented decision. | `docs/tutorials/` | #682 | Mid | Low | | Tighten the mypy suppressions that back the enforced-zero posture: burn down `prep_dgp`'s per-module `[index]` override (needs a None-vs-array restructure that preserves the seeded RNG stream), and evaluate re-enabling the globally disabled codes (`arg-type`, `return-value`, `var-annotated`, `assignment`) one at a time — `assignment` alone hid several real annotation drifts found during the 2026-07 triage. | `pyproject.toml` `[tool.mypy]`, `diff_diff/prep_dgp.py` | lint-CI | Mid | Low | | `practitioner_next_steps()` dedicated handler for `ChangesInChangesResults` (currently falls back to `_handle_generic`, which is safe; a dedicated handler is the established full-integration step, cf. HAD Phase 5). | `diff_diff/practitioner.py` | #682 | Quick | Low | -| Align the four legacy dataset loaders (`load_card_krueger`, `load_castle_doctrine`, `load_divorce_laws`, `load_mpdta`) with the loud-fallback pattern of `load_prop99`/`load_walmart`: `UserWarning` + `df.attrs["source"]` marker on synthetic fallback (currently silent), plus optional checksum pinning for the CSV downloads. | `diff_diff/datasets.py` | LWDiD precursor | Quick | Low | +| Align the four legacy dataset loaders (`load_card_krueger`, `load_castle_doctrine`, `load_divorce_laws`, `load_mpdta`) with the loud-fallback pattern of `load_prop99`/`load_walmart`: `UserWarning` + `df.attrs["source"]` marker on synthetic fallback (currently silent), plus optional checksum pinning for the CSV downloads. **Upgraded to a live defect 2026-07-13: the `causaldata/causal_datasets` GitHub repo backing castle/card_krueger/divorce is dead (404), so those loaders silently serve synthetic data everywhere - needs loud fallback + replacement sources.** | `diff_diff/datasets.py` | LWDiD precursor | Quick | Medium | +| Real-data CI canary for dataset-backed replication tests: `test_methodology_lwdid.py`'s Prop 99 / Walmart goldens skip (visibly) when loaders fall back to synthetic; add a lane or canary asserting `df.attrs["source"] == "lwdid_ssc_ancillary"` in CI so network regressions cannot silently de-gate the replication tests. Pairs with the loader-fallback repair row above. | `tests/test_methodology_lwdid.py`, `.github/workflows/` | LWDiD validation suite | Quick | Low | --- diff --git a/benchmarks/data/lwdid_walmart_eventstudy_golden.json b/benchmarks/data/lwdid_walmart_eventstudy_golden.json new file mode 100644 index 000000000..ebbc76694 --- /dev/null +++ b/benchmarks/data/lwdid_walmart_eventstudy_golden.json @@ -0,0 +1,637 @@ +{ + "_provenance": { + "paper": "Lee, S.J. & Wooldridge, J.M. (2025). A Simple Transformation Approach to Difference-in-Differences Estimation for Panel Data. SSRN Working Paper No. 4516518, revision dated June 8, 2026 (61 pages).", + "pdf_sha256": "78460841def3f15fdac6a2c6b04bc0c80ecc192493b9aa441e465a81f6846ea0", + "source_tables": "Appendix F, Tables A4 (log retail employment) and A5 (log wholesale employment), pp. 58-61; Walmart entry county panel (1,277 counties, 1977-1999).", + "se_method": "Influence-function multiplier bootstrap (Algorithm 1, Appendix E), B = 999, unit-level Rademacher multipliers.", + "value_format": "Each cell is [WATT(r) point estimate, bootstrap SE] at event time r = 0..13.", + "column_mapping": { + "etwfe": "ETWFE (Wooldridge 2025b) - biased under heterogeneous trends", + "cs2021": "Callaway & Sant'Anna (2021) - biased under heterogeneous trends", + "rolling_ipwra_demean": "Rolling IPWRA, unit-specific demeaning (no detrending)", + "rolling_ra_detrend": "Rolling RA, unit-specific detrending (heterogeneous-trend robust)", + "rolling_ipwra_detrend": "Rolling IPWRA, unit-specific detrending (heterogeneous-trend robust)" + }, + "extraction": "pdftotext -layout, cross-checked against plain-mode extraction and the WATT(1) = 0.032 (0.005) anchor recorded in docs/methodology/papers/lee-wooldridge-2025-review.md; extracted 2026-07-13." + }, + "table_a4_log_retail": { + "0": { + "etwfe": [ + 0.041, + 0.006 + ], + "cs2021": [ + 0.022, + 0.003 + ], + "rolling_ipwra_demean": [ + 0.04, + 0.006 + ], + "rolling_ra_detrend": [ + 0.007, + 0.004 + ], + "rolling_ipwra_detrend": [ + 0.007, + 0.004 + ] + }, + "1": { + "etwfe": [ + 0.072, + 0.007 + ], + "cs2021": [ + 0.052, + 0.004 + ], + "rolling_ipwra_demean": [ + 0.072, + 0.007 + ], + "rolling_ra_detrend": [ + 0.032, + 0.005 + ], + "rolling_ipwra_detrend": [ + 0.032, + 0.005 + ] + }, + "2": { + "etwfe": [ + 0.072, + 0.008 + ], + "cs2021": [ + 0.051, + 0.004 + ], + "rolling_ipwra_demean": [ + 0.072, + 0.008 + ], + "rolling_ra_detrend": [ + 0.024, + 0.006 + ], + "rolling_ipwra_detrend": [ + 0.024, + 0.006 + ] + }, + "3": { + "etwfe": [ + 0.073, + 0.009 + ], + "cs2021": [ + 0.051, + 0.005 + ], + "rolling_ipwra_demean": [ + 0.074, + 0.009 + ], + "rolling_ra_detrend": [ + 0.018, + 0.008 + ], + "rolling_ipwra_detrend": [ + 0.019, + 0.008 + ] + }, + "4": { + "etwfe": [ + 0.078, + 0.01 + ], + "cs2021": [ + 0.055, + 0.006 + ], + "rolling_ipwra_demean": [ + 0.079, + 0.01 + ], + "rolling_ra_detrend": [ + 0.015, + 0.009 + ], + "rolling_ipwra_detrend": [ + 0.016, + 0.009 + ] + }, + "5": { + "etwfe": [ + 0.088, + 0.011 + ], + "cs2021": [ + 0.061, + 0.007 + ], + "rolling_ipwra_demean": [ + 0.091, + 0.011 + ], + "rolling_ra_detrend": [ + 0.016, + 0.011 + ], + "rolling_ipwra_detrend": [ + 0.016, + 0.011 + ] + }, + "6": { + "etwfe": [ + 0.098, + 0.012 + ], + "cs2021": [ + 0.071, + 0.009 + ], + "rolling_ipwra_demean": [ + 0.101, + 0.012 + ], + "rolling_ra_detrend": [ + 0.02, + 0.013 + ], + "rolling_ipwra_detrend": [ + 0.017, + 0.013 + ] + }, + "7": { + "etwfe": [ + 0.116, + 0.013 + ], + "cs2021": [ + 0.089, + 0.01 + ], + "rolling_ipwra_demean": [ + 0.122, + 0.013 + ], + "rolling_ra_detrend": [ + 0.036, + 0.015 + ], + "rolling_ipwra_detrend": [ + 0.033, + 0.015 + ] + }, + "8": { + "etwfe": [ + 0.129, + 0.014 + ], + "cs2021": [ + 0.104, + 0.011 + ], + "rolling_ipwra_demean": [ + 0.137, + 0.015 + ], + "rolling_ra_detrend": [ + 0.04, + 0.017 + ], + "rolling_ipwra_detrend": [ + 0.039, + 0.016 + ] + }, + "9": { + "etwfe": [ + 0.134, + 0.016 + ], + "cs2021": [ + 0.114, + 0.013 + ], + "rolling_ipwra_demean": [ + 0.144, + 0.017 + ], + "rolling_ra_detrend": [ + 0.038, + 0.021 + ], + "rolling_ipwra_detrend": [ + 0.039, + 0.019 + ] + }, + "10": { + "etwfe": [ + 0.155, + 0.019 + ], + "cs2021": [ + 0.131, + 0.015 + ], + "rolling_ipwra_demean": [ + 0.168, + 0.019 + ], + "rolling_ra_detrend": [ + 0.024, + 0.025 + ], + "rolling_ipwra_detrend": [ + 0.035, + 0.022 + ] + }, + "11": { + "etwfe": [ + 0.163, + 0.023 + ], + "cs2021": [ + 0.141, + 0.019 + ], + "rolling_ipwra_demean": [ + 0.178, + 0.023 + ], + "rolling_ra_detrend": [ + 0.0, + 0.031 + ], + "rolling_ipwra_detrend": [ + 0.016, + 0.027 + ] + }, + "12": { + "etwfe": [ + 0.164, + 0.03 + ], + "cs2021": [ + 0.15, + 0.024 + ], + "rolling_ipwra_demean": [ + 0.182, + 0.029 + ], + "rolling_ra_detrend": [ + -0.002, + 0.039 + ], + "rolling_ipwra_detrend": [ + 0.015, + 0.034 + ] + }, + "13": { + "etwfe": [ + 0.204, + 0.043 + ], + "cs2021": [ + 0.188, + 0.035 + ], + "rolling_ipwra_demean": [ + 0.231, + 0.042 + ], + "rolling_ra_detrend": [ + 0.028, + 0.055 + ], + "rolling_ipwra_detrend": [ + 0.046, + 0.043 + ] + } + }, + "table_a5_log_wholesale": { + "0": { + "etwfe": [ + 0.043, + 0.011 + ], + "cs2021": [ + 0.004, + 0.007 + ], + "rolling_ipwra_demean": [ + 0.041, + 0.01 + ], + "rolling_ra_detrend": [ + -0.001, + 0.008 + ], + "rolling_ipwra_detrend": [ + -0.002, + 0.008 + ] + }, + "1": { + "etwfe": [ + 0.034, + 0.012 + ], + "cs2021": [ + -0.006, + 0.009 + ], + "rolling_ipwra_demean": [ + 0.032, + 0.012 + ], + "rolling_ra_detrend": [ + -0.017, + 0.01 + ], + "rolling_ipwra_detrend": [ + -0.02, + 0.011 + ] + }, + "2": { + "etwfe": [ + 0.031, + 0.014 + ], + "cs2021": [ + -0.011, + 0.01 + ], + "rolling_ipwra_demean": [ + 0.03, + 0.014 + ], + "rolling_ra_detrend": [ + -0.028, + 0.013 + ], + "rolling_ipwra_detrend": [ + -0.032, + 0.013 + ] + }, + "3": { + "etwfe": [ + 0.028, + 0.016 + ], + "cs2021": [ + -0.015, + 0.012 + ], + "rolling_ipwra_demean": [ + 0.028, + 0.017 + ], + "rolling_ra_detrend": [ + -0.039, + 0.016 + ], + "rolling_ipwra_detrend": [ + -0.044, + 0.016 + ] + }, + "4": { + "etwfe": [ + 0.032, + 0.018 + ], + "cs2021": [ + -0.012, + 0.014 + ], + "rolling_ipwra_demean": [ + 0.033, + 0.019 + ], + "rolling_ra_detrend": [ + -0.046, + 0.019 + ], + "rolling_ipwra_detrend": [ + -0.052, + 0.02 + ] + }, + "5": { + "etwfe": [ + 0.041, + 0.02 + ], + "cs2021": [ + -0.003, + 0.017 + ], + "rolling_ipwra_demean": [ + 0.042, + 0.021 + ], + "rolling_ra_detrend": [ + -0.045, + 0.023 + ], + "rolling_ipwra_detrend": [ + -0.052, + 0.023 + ] + }, + "6": { + "etwfe": [ + 0.048, + 0.022 + ], + "cs2021": [ + 0.005, + 0.018 + ], + "rolling_ipwra_demean": [ + 0.051, + 0.023 + ], + "rolling_ra_detrend": [ + -0.043, + 0.026 + ], + "rolling_ipwra_detrend": [ + -0.057, + 0.027 + ] + }, + "7": { + "etwfe": [ + 0.049, + 0.024 + ], + "cs2021": [ + 0.008, + 0.023 + ], + "rolling_ipwra_demean": [ + 0.055, + 0.025 + ], + "rolling_ra_detrend": [ + -0.04, + 0.031 + ], + "rolling_ipwra_detrend": [ + -0.052, + 0.032 + ] + }, + "8": { + "etwfe": [ + 0.054, + 0.026 + ], + "cs2021": [ + 0.012, + 0.023 + ], + "rolling_ipwra_demean": [ + 0.064, + 0.028 + ], + "rolling_ra_detrend": [ + -0.055, + 0.035 + ], + "rolling_ipwra_detrend": [ + -0.067, + 0.036 + ] + }, + "9": { + "etwfe": [ + 0.061, + 0.029 + ], + "cs2021": [ + 0.021, + 0.025 + ], + "rolling_ipwra_demean": [ + 0.075, + 0.031 + ], + "rolling_ra_detrend": [ + -0.058, + 0.04 + ], + "rolling_ipwra_detrend": [ + -0.062, + 0.041 + ] + }, + "10": { + "etwfe": [ + 0.111, + 0.034 + ], + "cs2021": [ + 0.064, + 0.031 + ], + "rolling_ipwra_demean": [ + 0.129, + 0.035 + ], + "rolling_ra_detrend": [ + -0.054, + 0.047 + ], + "rolling_ipwra_detrend": [ + -0.053, + 0.049 + ] + }, + "11": { + "etwfe": [ + 0.117, + 0.04 + ], + "cs2021": [ + 0.059, + 0.039 + ], + "rolling_ipwra_demean": [ + 0.138, + 0.04 + ], + "rolling_ra_detrend": [ + -0.101, + 0.057 + ], + "rolling_ipwra_detrend": [ + -0.089, + 0.061 + ] + }, + "12": { + "etwfe": [ + 0.1, + 0.049 + ], + "cs2021": [ + 0.055, + 0.045 + ], + "rolling_ipwra_demean": [ + 0.127, + 0.048 + ], + "rolling_ra_detrend": [ + -0.135, + 0.067 + ], + "rolling_ipwra_detrend": [ + -0.114, + 0.068 + ] + }, + "13": { + "etwfe": [ + 0.134, + 0.062 + ], + "cs2021": [ + 0.116, + 0.05 + ], + "rolling_ipwra_demean": [ + 0.165, + 0.06 + ], + "rolling_ra_detrend": [ + -0.088, + 0.083 + ], + "rolling_ipwra_detrend": [ + -0.062, + 0.083 + ] + } + } +} diff --git a/benchmarks/data/real/castle_lw_subset.csv b/benchmarks/data/real/castle_lw_subset.csv new file mode 100644 index 000000000..8ea84cfb9 --- /dev/null +++ b/benchmarks/data/real/castle_lw_subset.csv @@ -0,0 +1,551 @@ +state,year,effyear,lhomicide,homicide,population +Arkansas,2000,,1.8660628,6.462801,2599492 +Florida,2000,2005,1.7562876,5.7908993,15593433 +Arizona,2000,2006,1.9671516,7.1502805,5020782 +Colorado,2000,,1.1605736,3.1917636,4198306 +Indiana,2000,2006,1.7856988,5.9637456,5902331 +Connecticut,2000,,1.0891795,2.9718349,3297626 +Vermont,2000,,.42555386,1.5304378,588067 +Delaware,2000,,1.1920218,3.2937338,759017 +California,2000,,1.8389783,6.2901087,33051894 +Virginia,2000,,1.7675487,5.8564796,6847117 +Kentucky,2000,2006,1.5922382,4.9147367,3926965 +New Jersey,2000,,1.2573286,3.5160165,8219529 +Mississippi,2000,2006,2.2273524,9.2752771,2749244 +Wyoming,2000,,.91691804,2.5015688,479699 +Louisiana,2000,2006,2.5590889,12.924038,4333011 +Maryland,2000,,2.1197927,8.3294106,5162430 +North Carolina,2000,,1.9718138,7.1836944,7795432 +Illinois,2000,,2.004585,7.4230137,12097512 +Missouri,2000,2007,1.85422,6.3867149,5433153 +Maine,2000,,.19034488,1.2096667,1240011 +Alabama,2000,2006,2.0273557,7.5939784,4332380 +Ohio,2000,2008,1.3301022,3.78143,11054019 +West Virginia,2000,2008,.95779407,2.6059415,1765197 +New York,2000,,1.643847,5.1750393,18395996 +Alaska,2000,2006,1.4915183,4.4438372,607583 +North Dakota,2000,2007,-.43594417,.64665383,618569 +Georgia,2000,2006,2.1024218,8.1859703,7952631 +Hawaii,2000,,1.0908525,2.9768107,1175755 +New Mexico,2000,,2.0245388,7.5726171,1782739 +Nevada,2000,,1.8819478,6.5662823,1964582 +Rhode Island,2000,,1.4946193,4.4576392,1009503 +Minnesota,2000,,1.1495022,3.156621,4783596 +Massachusetts,2000,,.71287966,2.0398569,6127881 +Montana,2000,2009,.82390183,2.2793763,877433 +Pennsylvania,2000,,1.6255342,5.0811324,11847753 +Oklahoma,2000,2006,1.6959662,5.451911,3338279 +New Hampshire,2000,,.60593003,1.8329561,1200247 +Idaho,2000,,.23694378,1.2673699,1262457 +South Carolina,2000,2006,2.0156829,7.5058517,3876975 +Texas,2000,2007,1.8085042,6.1013141,20290711 +Michigan,2000,2006,1.9322537,6.9050546,9688555 +Kansas,2000,2006,1.8693175,6.48387,2606468 +Wisconsin,2000,,1.1771721,3.2451839,5207717 +Washington,2000,,1.2249846,3.404114,5757739 +Oregon,2000,,.73876995,2.093359,3343908 +Utah,2000,,.67348641,1.9610624,2192689 +Iowa,2000,,.48855552,1.6299601,2822155 +Nebraska,2000,,1.333464,3.7941637,1660445 +Tennessee,2000,2007,2.0013363,7.3989363,5541337 +South Dakota,2000,2006,-.037056267,.96362191,726426 +Tennessee,2001,2007,2.0234468,7.564352,5541337 +North Carolina,2001,,1.851024,6.3663354,7795432 +Arkansas,2001,,1.7321641,5.6528745,2599492 +Colorado,2001,,1.2979765,3.6618795,4198306 +Mississippi,2001,2006,2.3231585,10.207864,2749244 +Montana,2001,2009,1.3520191,3.865222,877433 +Nebraska,2001,,.95036596,2.5866561,1660445 +Delaware,2001,,1.092224,2.9808962,759017 +Nevada,2001,,2.1616688,8.6856194,1964582 +Oregon,2001,,.90582275,2.4739666,3343908 +Vermont,2001,,.16703165,1.1817917,588067 +Hawaii,2001,,.99036205,2.692209,1175755 +Louisiana,2001,2006,2.4485838,11.571946,4333011 +Georgia,2001,2006,1.9929868,7.3374162,7952631 +Indiana,2001,2006,1.9397343,6.9569025,5902331 +Utah,2001,,1.1004221,3.0054345,2192689 +Michigan,2001,2006,1.931419,6.8992939,9688555 +Iowa,2001,,.573071,1.7737057,2822155 +Washington,2001,,1.1184769,3.0601895,5757739 +California,2001,,1.8794022,6.5495877,33051894 +Kentucky,2001,2006,1.5220182,4.5814619,3926965 +Wyoming,2001,,.62791729,1.8737041,479699 +Massachusetts,2001,,.84229529,2.3216898,6127881 +Missouri,2001,2007,1.9174697,6.8037214,5433153 +Rhode Island,2001,,1.3410739,3.8231471,1009503 +Ohio,2001,2008,1.4064639,4.0814972,11054019 +West Virginia,2001,2008,.82169271,2.2743464,1765197 +Connecticut,2001,,1.152275,3.165386,3297626 +Minnesota,2001,,.90037459,2.4605246,4783596 +North Dakota,2001,2007,.13632226,1.1460512,618569 +New York,2001,,1.6501856,5.2079463,18395996 +Pennsylvania,2001,,1.7032735,5.4918962,11847753 +New Jersey,2001,,1.3995246,4.0532722,8219529 +South Carolina,2001,2006,2.1283817,8.4012604,3876975 +Virginia,2001,,1.6549245,5.2326846,6847117 +Maine,2001,,.4173153,1.517881,1240011 +Alabama,2001,2006,2.1648669,8.7134428,4332380 +Florida,2001,2005,1.697463,5.4600778,15593433 +Illinois,2001,,2.0888155,8.0753431,12097512 +Alaska,2001,2006,1.8462465,6.3359928,607583 +Oklahoma,2001,2006,1.7095091,5.526248,3338279 +Kansas,2001,2006,1.2588438,3.5213478,2606468 +Idaho,2001,,.84436393,2.3264976,1262457 +South Dakota,2001,2006,-.039176732,.96158075,726426 +Maryland,2001,,2.1412051,8.5096855,5162430 +Arizona,2001,2006,2.0407066,7.6960464,5020782 +New Hampshire,2001,,.32880148,1.389302,1200247 +New Mexico,2001,,1.7087433,5.5220175,1782739 +Texas,2001,2007,1.8586098,6.4148126,20290711 +Wisconsin,2001,,1.2974609,3.659992,5207717 +Idaho,2002,,1.01219,2.7516203,1262457 +Florida,2002,2005,1.7196488,5.5825677,15593433 +Mississippi,2002,2006,2.2526314,9.5127354,2749244 +Maine,2002,,.10572013,1.1115108,1240011 +Arizona,2002,2006,1.9793757,7.2382231,5020782 +Iowa,2002,,.44048294,1.5534573,2822155 +Georgia,2002,2006,1.9848815,7.2781854,7952631 +Louisiana,2002,2006,2.6129756,13.639578,4333011 +Alaska,2002,2006,1.6651237,5.2863269,607583 +Maryland,2002,,2.2658429,9.639246,5162430 +Hawaii,2002,,.68605816,1.9858721,1175755 +Pennsylvania,2002,,1.6572351,5.2447896,11847753 +California,2002,,1.9436159,6.9839587,33051894 +Arkansas,2002,,1.6844164,5.3893051,2599492 +Kentucky,2002,2006,1.5688833,4.8012834,3926965 +Nevada,2002,,2.1352024,8.4587584,1964582 +Rhode Island,2002,,1.3806887,3.9776399,1009503 +North Carolina,2002,,1.9162961,6.7957411,7795432 +West Virginia,2002,2008,1.1762348,3.2421439,1765197 +Wisconsin,2002,,1.0693805,2.9135742,5207717 +Colorado,2002,,1.402365,4.0648017,4198306 +Connecticut,2002,,.91916841,2.5072045,3297626 +Minnesota,2002,,.83029675,2.2939994,4783596 +Massachusetts,2002,,1.0244526,2.7855701,6127881 +Kansas,2002,2006,1.0855738,2.9611385,2606468 +New Jersey,2002,,1.3957362,4.0379462,8219529 +South Carolina,2002,2006,2.015264,7.5027084,3876975 +North Dakota,2002,2007,-.19925244,.81934303,618569 +Oregon,2002,,.73740298,2.0904994,3343908 +New Hampshire,2002,,-.031916067,.96858788,1200247 +Illinois,2002,,2.0574965,7.8263531,12097512 +Ohio,2002,2008,1.5537809,4.7293177,11054019 +Washington,2002,,1.1322747,3.1027062,5757739 +Delaware,2002,,1.2011294,3.3238688,759017 +Tennessee,2002,2007,2.0069497,7.440587,5541337 +Michigan,2002,2006,1.9344658,6.9203458,9688555 +South Dakota,2002,2006,.40733549,1.5028082,726426 +Virginia,2002,,1.7035154,5.4932241,6847117 +Indiana,2002,2006,1.8004912,6.0526199,5902331 +Montana,2002,2009,.59263808,1.8087537,877433 +Vermont,2002,,.7801708,2.1818449,588067 +New Mexico,2002,,2.1231635,8.3575354,1782739 +Oklahoma,2002,2006,1.5734329,4.8231773,3338279 +Alabama,2002,2006,1.9363341,6.9332876,4332380 +Wyoming,2002,,1.1294159,3.0938487,479699 +Missouri,2002,2007,1.7937015,6.0116639,5433153 +Utah,2002,,.72520405,2.0651524,2192689 +Nebraska,2002,,1.0510263,2.8605857,1660445 +New York,2002,,1.5881294,4.8945847,18395996 +Texas,2002,2007,1.81434,6.1370244,20290711 +Indiana,2003,2006,1.7257981,5.617002,5902331 +South Dakota,2003,2006,.30741003,1.3598984,726426 +Arizona,2003,2006,2.0870271,8.060915,5020782 +Colorado,2003,,1.4253404,4.1592736,4198306 +Maine,2003,,.23051284,1.2592456,1240011 +Alabama,2003,2006,1.9195671,6.8180065,4332380 +Ohio,2003,2008,1.5526478,4.7239618,11054019 +Utah,2003,,.95470458,2.597903,2192689 +Iowa,2003,,.56568038,1.7606453,2822155 +Massachusetts,2003,,.81148469,2.2512479,6127881 +Hawaii,2003,,.58806264,1.8004968,1175755 +New Hampshire,2003,,.30622792,1.3582919,1200247 +Connecticut,2003,,1.2006328,3.3222187,3297626 +Pennsylvania,2003,,1.6913738,5.4269314,11847753 +New Mexico,2003,,1.8421764,6.310257,1782739 +Tennessee,2003,2007,1.9402487,6.9604821,5541337 +Kansas,2003,2006,1.5542881,4.7317171,2606468 +Florida,2003,2005,1.7156318,5.5601873,15593433 +Texas,2003,2007,1.8869598,6.5992751,20290711 +Maryland,2003,,2.279525,9.7720375,5162430 +Wyoming,2003,,1.0552635,2.8727319,479699 +Washington,2003,,1.1113269,3.0383873,5757739 +Oklahoma,2003,2006,1.802459,6.0645418,3338279 +California,2003,,1.9382337,6.9464707,33051894 +Mississippi,2003,2006,2.2602386,9.5853767,2749244 +Michigan,2003,2006,1.8291315,6.2284751,9688555 +Oregon,2003,,.66921896,1.9527116,3343908 +Missouri,2003,2007,1.6527946,5.2215519,5433153 +Montana,2003,2009,1.2123444,3.3613558,877433 +Kentucky,2003,2006,1.5088589,4.5215683,3926965 +Vermont,2003,,.91904116,2.5068855,588067 +North Carolina,2003,,1.8263623,6.2112503,7795432 +Alaska,2003,2006,1.8232787,6.1921272,607583 +Arkansas,2003,,1.9157888,6.792294,2599492 +Rhode Island,2003,,.8797698,2.4103448,1009503 +Illinois,2003,,1.9823071,7.2594714,12097512 +North Dakota,2003,2007,.39018902,1.47726,618569 +Georgia,2003,2006,2.0523295,7.7860179,7952631 +South Carolina,2003,2006,2.0227175,7.5588374,3876975 +West Virginia,2003,2008,1.4190463,4.1331768,1765197 +Delaware,2003,,.97450769,2.6498623,759017 +Nevada,2003,,2.1887245,8.9238234,1964582 +Nebraska,2003,,1.1994231,3.318202,1660445 +Virginia,2003,,1.7607137,5.8165874,6847117 +Wisconsin,2003,,1.2361394,3.4422987,5207717 +Idaho,2003,,.66795564,1.9502462,1262457 +Louisiana,2003,2006,2.5945523,13.390592,4333011 +New York,2003,,1.6137015,5.0213633,18395996 +Minnesota,2003,,.94837803,2.5815191,4783596 +New Jersey,2003,,1.570303,4.8081045,8219529 +Idaho,2004,,.82380563,2.2791569,1262457 +Louisiana,2004,2006,2.5727932,13.102372,4333011 +Vermont,2004,,.9797765,2.6638608,588067 +Alabama,2004,2006,1.7498412,5.7536893,4332380 +California,2004,,1.9203752,6.8235188,33051894 +Michigan,2004,2006,1.8751842,6.5220203,9688555 +Florida,2004,2005,1.7170218,5.5679212,15593433 +South Carolina,2004,2006,1.9522637,7.0446162,3876975 +Delaware,2004,,1.2460096,3.4764428,759017 +Kansas,2004,2006,1.5255737,4.5977807,2606468 +Washington,2004,,1.1422263,3.1337373,5757739 +Oregon,2004,,.94053811,2.5613594,3343908 +Connecticut,2004,,1.0819076,2.9503024,3297626 +Wyoming,2004,,.80350196,2.2333484,479699 +New Mexico,2004,,2.2052131,9.0721836,1782739 +Indiana,2004,2006,1.6516238,5.2154422,5902331 +Mississippi,2004,2006,2.0909514,8.0926113,2749244 +Colorado,2004,,1.4969511,4.4680457,4198306 +Ohio,2004,2008,1.5121714,4.5365705,11054019 +Massachusetts,2004,,1.0143008,2.7574348,6127881 +Iowa,2004,,.43387687,1.5432289,2822155 +Illinois,2004,,1.8397771,6.295135,12097512 +North Dakota,2004,2007,.27165946,1.3121401,618569 +Maine,2004,,.34192318,1.4076521,1240011 +Rhode Island,2004,,.91917765,2.5072277,1009503 +Minnesota,2004,,.82364941,2.278801,4783596 +Arkansas,2004,,1.8836236,6.5772953,2599492 +New Hampshire,2004,,.29795423,1.3471001,1200247 +Nevada,2004,,2.0114799,7.474371,1964582 +North Carolina,2004,,1.8614205,6.432868,7795432 +New Jersey,2004,,1.5282232,4.6099782,8219529 +Montana,2004,2009,1.2018628,3.3263075,877433 +New York,2004,,1.5625062,4.7707629,18395996 +Missouri,2004,2007,1.8464279,6.3371425,5433153 +Kentucky,2004,2006,1.767199,5.8544321,3926965 +Nebraska,2004,,.85771936,2.3577774,1660445 +Tennessee,2004,2007,1.8262328,6.2104464,5541337 +Arizona,2004,2006,1.9944618,7.3482466,5020782 +Maryland,2004,,2.2627244,9.6092329,5162430 +Alaska,2004,2006,1.7609477,5.8179483,607583 +Texas,2004,2007,1.8285499,6.224853,20290711 +Wisconsin,2004,,1.0571437,2.8781383,5207717 +Virginia,2004,,1.6862195,5.3990307,6847117 +Pennsylvania,2004,,1.6929965,5.4357448,11847753 +Georgia,2004,2006,1.9661723,7.1432824,7952631 +Oklahoma,2004,2006,1.6958038,5.4510255,3338279 +Utah,2004,,.6718657,1.9578867,2192689 +Hawaii,2004,,.98934382,2.6894691,1175755 +South Dakota,2004,2006,.8297354,2.292712,726426 +West Virginia,2004,2008,1.3457154,3.8409333,1765197 +Oklahoma,2005,2006,1.6949445,5.4463439,3338279 +Kansas,2005,2006,1.3332263,3.793262,2606468 +California,2005,,1.9593788,7.0949187,33051894 +Idaho,2005,,.91941416,2.5078208,1262457 +Georgia,2005,2006,1.8553178,6.3937302,7952631 +Texas,2005,2007,1.8433821,6.3178697,20290711 +Michigan,2005,2006,1.852494,6.3757005,9688555 +Illinois,2005,,1.8228601,6.1895361,12097512 +Alaska,2005,2006,1.6067477,4.9865675,607583 +Virginia,2005,,1.8319528,6.2460723,6847117 +Wisconsin,2005,,1.3433927,3.8320227,5207717 +New Hampshire,2005,,.40088144,1.4931402,1200247 +Vermont,2005,,.28387263,1.3282638,588067 +Minnesota,2005,,.83494157,2.3046794,4783596 +Kentucky,2005,2006,1.5435928,4.6813793,3926965 +Indiana,2005,2006,1.7651441,5.8424139,5902331 +Ohio,2005,2008,1.6655953,5.2888207,11054019 +Wyoming,2005,,1.0392133,2.826992,479699 +Louisiana,2005,2006,2.3273909,10.251161,4333011 +New Mexico,2005,,2.0321341,7.630352,1782739 +Arizona,2005,2006,2.0324998,7.6331439,5020782 +Maine,2005,,.39212835,1.4801277,1240011 +Pennsylvania,2005,,1.8422889,6.310967,11847753 +New Jersey,2005,,1.5879173,4.8935466,8219529 +North Dakota,2005,2007,.67720002,1.9683586,618569 +Montana,2005,2009,.68138218,1.9766079,877433 +Missouri,2005,2007,1.9654183,7.137898,5433153 +South Dakota,2005,2006,.88077211,2.4127619,726426 +Florida,2005,2005,1.6252755,5.0798182,15593433 +Massachusetts,2005,,1.0574175,2.8789265,6127881 +Arkansas,2005,,1.9453803,6.9962921,2599492 +Colorado,2005,,1.3328919,3.7919936,4198306 +Utah,2005,,.83596647,2.3070426,2192689 +Washington,2005,,1.2045684,3.3353193,5757739 +Oregon,2005,,.80965042,2.2471223,3343908 +Delaware,2005,,1.5085084,4.5199838,759017 +North Carolina,2005,,1.9394815,6.9551439,7795432 +South Carolina,2005,2006,2.0324216,7.6325469,3876975 +Rhode Island,2005,,1.1916355,3.2924616,1009503 +Connecticut,2005,,1.1291449,3.0930104,3297626 +Mississippi,2005,2006,2.0251813,7.5774851,2749244 +Iowa,2005,,.33458462,1.3973598,2822155 +New York,2005,,1.5443664,4.6850019,18395996 +Alabama,2005,2006,2.1304405,8.4185734,4332380 +West Virginia,2005,2008,1.5321664,4.6281924,1765197 +Nebraska,2005,,.94688112,2.5776577,1660445 +Hawaii,2005,,.66184396,1.9383633,1175755 +Tennessee,2005,2007,2.0038409,7.417491,5541337 +Maryland,2005,,2.3132727,10.10745,5162430 +Nevada,2005,,2.1576526,8.6508064,1964582 +North Dakota,2006,2007,.22962229,1.2581247,618569 +Washington,2006,,1.1249774,3.0801473,5757739 +California,2006,,1.9213043,6.8298612,33051894 +Indiana,2006,2006,1.7463658,5.733727,5902331 +Texas,2006,2007,1.7742605,5.8959198,20290711 +Rhode Island,2006,,.92782927,2.5290134,1009503 +Maine,2006,,.55408567,1.7403491,1240011 +West Virginia,2006,2008,1.5061387,4.5092854,1765197 +South Dakota,2006,2006,1.3107148,3.7088242,726426 +Kentucky,2006,2006,1.437035,4.2082,3926965 +Montana,2006,2009,1.2807353,3.5992851,877433 +North Carolina,2006,,1.8078319,6.0972133,7795432 +Kansas,2006,2006,1.5248959,4.5946655,2606468 +Virginia,2006,,1.6625764,5.2728786,6847117 +Mississippi,2006,2006,2.0758135,7.9710293,2749244 +Michigan,2006,2006,1.9687202,7.1615052,9688555 +Connecticut,2006,,1.3632598,3.9089148,3297626 +Illinois,2006,,1.8047692,6.078568,12097512 +Idaho,2006,,.78029603,2.1821182,1262457 +Florida,2006,2005,1.8311493,6.2410555,15593433 +Ohio,2006,2008,1.5466979,4.6959376,11054019 +Nevada,2006,,2.2034492,9.0561962,1964582 +Minnesota,2006,,.8992902,2.4578578,4783596 +Louisiana,2006,2006,2.5731504,13.107053,4333011 +Vermont,2006,,.80822456,2.2439206,588067 +Colorado,2006,,1.2802231,3.5974424,4198306 +Alaska,2006,2006,1.6813323,5.3727093,607583 +New Jersey,2006,,1.5880568,4.8942294,8219529 +Maryland,2006,,2.2744627,9.7226944,5162430 +Arizona,2006,2006,2.1436143,8.5302124,5020782 +Georgia,2006,2006,1.8805436,6.5570683,7952631 +Alabama,2006,2006,2.0796523,8.001687,4332380 +Massachusetts,2006,,1.0717643,2.9205277,6127881 +South Carolina,2006,2006,2.1282732,8.4003487,3876975 +Iowa,2006,,.64784348,1.9114144,2822155 +Missouri,2006,2007,1.8457227,6.3326745,5433153 +Arkansas,2006,,2.0157783,7.5065675,2599492 +New Hampshire,2006,,-.011392572,.98867208,1200247 +New Mexico,2006,,1.9544835,7.0602717,1782739 +Wisconsin,2006,,1.0883908,2.969492,5207717 +Hawaii,2006,,.49079114,1.6336081,1175755 +Wyoming,2006,,.92594486,2.5242522,479699 +Oklahoma,2006,2006,1.7598103,5.8113351,3338279 +Delaware,2006,,1.5935224,4.9210525,759017 +Utah,2006,,.71254051,2.0391653,2192689 +Nebraska,2006,,1.0976707,2.9971764,1660445 +Tennessee,2006,2007,1.9394639,6.9550209,5541337 +New York,2006,,1.5657016,4.7860317,18395996 +Pennsylvania,2006,,1.7951869,6.0205998,11847753 +Oregon,2006,,.8662141,2.3778913,3343908 +Alaska,2007,2006,1.8720165,6.5013933,607583 +Maine,2007,,.41947782,1.521167,1240011 +Pennsylvania,2007,,1.7795116,5.9269609,11847753 +West Virginia,2007,2008,1.2944132,3.6488543,1765197 +Arizona,2007,2006,2.1887076,8.9236736,5020782 +Montana,2007,2009,.88756067,2.4291968,877433 +Illinois,2007,,1.7720312,5.8827906,12097512 +Texas,2007,2007,1.8037177,6.0721803,20290711 +Wyoming,2007,,1.4073837,4.0852532,479699 +New Hampshire,2007,,-.087899446,.91585296,1200247 +Arkansas,2007,,1.9691905,7.1648736,2599492 +Indiana,2007,2006,1.7258945,5.6175432,5902331 +Minnesota,2007,,.8109712,2.2500923,4783596 +Maryland,2007,,2.2921982,9.8966694,5162430 +Washington,2007,,1.0046525,2.7309582,5757739 +South Dakota,2007,2006,1.3693535,3.9328074,726426 +North Dakota,2007,2007,.78634548,2.1953588,618569 +Louisiana,2007,2006,2.6759129,14.525605,4333011 +California,2007,,1.8296967,6.2319956,33051894 +Oregon,2007,,.69598961,2.005693,3343908 +Iowa,2007,,.29703704,1.3458651,2822155 +New Jersey,2007,,1.4803668,4.3945575,8219529 +Kentucky,2007,2006,1.6454816,5.1835055,3926965 +Georgia,2007,2006,2.0584309,7.8336692,7952631 +Tennessee,2007,2007,1.8973429,6.6681528,5541337 +Vermont,2007,,.73945057,2.0947843,588067 +North Carolina,2007,,1.8880756,6.6066422,7795432 +Mississippi,2007,2006,1.9487168,7.0196738,2749244 +Massachusetts,2007,,1.0716764,2.9202709,6127881 +South Carolina,2007,2006,2.1287858,8.4046555,3876975 +Alabama,2007,2006,2.1456904,8.5479403,4332380 +Wisconsin,2007,,1.1946696,3.3024666,5207717 +Kansas,2007,2006,1.3743197,3.9523871,2606468 +Nebraska,2007,,1.3782519,3.9679594,1660445 +Colorado,2007,,1.1790891,3.251411,4198306 +New York,2007,,1.4279068,4.1699615,18395996 +Utah,2007,,.82847154,2.2898161,2192689 +Connecticut,2007,,1.173512,3.2333281,3297626 +Florida,2007,2005,1.8979564,6.6722445,15593433 +Rhode Island,2007,,.58163875,1.7889677,1009503 +Nevada,2007,,2.0482574,7.7543769,1964582 +Ohio,2007,2008,1.5292437,4.6146855,11054019 +New Mexico,2007,,2.2590072,9.5735798,1782739 +Missouri,2007,2007,1.8389635,6.2900152,5433153 +Hawaii,2007,,.63132018,1.880091,1175755 +Michigan,2007,2006,1.882246,6.5682406,9688555 +Idaho,2007,,1.2087058,3.3491473,1262457 +Oklahoma,2007,2006,1.8300821,6.2343984,3338279 +Delaware,2007,,1.520337,4.5737662,759017 +Virginia,2007,,1.6927352,5.4343243,6847117 +Vermont,2008,,1.0074744,2.7386756,588067 +Maryland,2008,,2.1719038,8.7749748,5162430 +Alaska,2008,2006,1.3770998,3.9633901,607583 +Massachusetts,2008,,.94826812,2.5812354,6127881 +Colorado,2008,,1.1694129,3.2201014,4198306 +New Jersey,2008,,1.4684432,4.3424692,8219529 +Arkansas,2008,,1.756988,5.7949567,2599492 +South Dakota,2008,2006,1.5367942,4.6496606,726426 +South Carolina,2008,2006,1.9419367,6.9722414,3876975 +Connecticut,2008,,1.3294542,3.77898,3297626 +Wyoming,2008,,.83081472,2.295188,479699 +Iowa,2008,,.9478246,2.5800908,2822155 +Missouri,2008,2007,2.0493162,7.7625914,5433153 +New Mexico,2008,,2.0339706,7.6443796,1782739 +Illinois,2008,,1.8177385,6.1579165,12097512 +Nevada,2008,,1.8687446,6.4801559,1964582 +Hawaii,2008,,.7084381,2.0308168,1175755 +Delaware,2008,,1.8891935,6.6140327,759017 +Wisconsin,2008,,.9585312,2.6078632,5207717 +Mississippi,2008,2006,2.0815668,8.0170212,2749244 +Tennessee,2008,2007,1.9029453,6.7056155,5541337 +Michigan,2008,2006,1.7074353,5.5147991,9688555 +North Carolina,2008,,1.9013771,6.6951079,7795432 +Oregon,2008,,.82217807,2.2754505,3343908 +Georgia,2008,2006,1.9221219,6.8354468,7952631 +Nebraska,2008,,1.3600358,3.8963327,1660445 +Rhode Island,2008,,1.0785193,2.9403229,1009503 +Indiana,2008,2006,1.6257827,5.0823956,5902331 +Kentucky,2008,2006,1.5622995,4.7697768,3926965 +West Virginia,2008,2008,1.3085829,3.7009254,1765197 +Washington,2008,,1.0955471,2.9908185,5757739 +Louisiana,2008,2006,2.5223651,12.458026,4333011 +Florida,2008,2005,1.8608692,6.4293222,15593433 +Maine,2008,,.85751295,2.3572907,1240011 +Pennsylvania,2008,,1.7392377,5.6930017,11847753 +Idaho,2008,,.43164414,1.5397871,1262457 +New York,2008,,1.4592829,4.3028727,18395996 +Arizona,2008,2006,1.9680102,7.1564221,5020782 +Ohio,2008,2008,1.5690676,4.8021684,11054019 +New Hampshire,2008,,.064692453,1.0668309,1200247 +Alabama,2008,2006,2.0435977,7.718328,4332380 +Minnesota,2008,,.74357897,2.1034503,4783596 +Virginia,2008,,1.5698615,4.8059826,6847117 +North Dakota,2008,2007,-.24469057,.78294677,618569 +Montana,2008,2009,1.2384012,3.4500928,877433 +Oklahoma,2008,2006,1.7713466,5.8787642,3338279 +Texas,2008,2007,1.7483649,5.7452011,20290711 +Kansas,2008,2006,1.4028559,4.0667977,2606468 +Utah,2008,,.40665331,1.5017834,2192689 +California,2008,,1.7718333,5.8816261,33051894 +Hawaii,2009,,.58586079,1.7965368,1175755 +Texas,2009,2007,1.7198638,5.5837679,20290711 +Arkansas,2009,,1.8416507,6.306941,2599492 +Mississippi,2009,2006,1.8929226,6.6387429,2749244 +Minnesota,2009,,.35501978,1.4262089,4783596 +Idaho,2009,,.47496724,1.6079615,1262457 +Vermont,2009,,.25422469,1.2894615,588067 +Alaska,2009,2006,1.1695099,3.2204139,607583 +Nevada,2009,,1.8128406,6.1278291,1964582 +Montana,2009,2009,1.2078794,3.3463807,877433 +Florida,2009,2005,1.7193747,5.581037,15593433 +North Carolina,2009,,1.6874874,5.4058805,7795432 +Utah,2009,,.38573185,1.4706903,2192689 +Wisconsin,2009,,.95835847,2.6074128,5207717 +New York,2009,,1.391486,4.0208206,18395996 +Iowa,2009,,.24345373,1.2756473,2822155 +Wyoming,2009,,.74167114,2.0994411,479699 +Connecticut,2009,,1.1096674,3.0333493,3297626 +New Jersey,2009,,1.3049833,3.6876276,8219529 +Nebraska,2009,,.83880818,2.3136079,1660445 +Kentucky,2009,2006,1.4649612,4.3273754,3926965 +New Hampshire,2009,,-.17884505,.83623546,1200247 +North Dakota,2009,2007,.62903839,1.875806,618569 +Louisiana,2009,2006,2.4841781,11.991261,4333011 +Oklahoma,2009,2006,1.8690081,6.4818635,3338279 +Ohio,2009,2008,1.5212379,4.5778885,11054019 +California,2009,,1.6921659,5.431231,33051894 +Kansas,2009,2006,1.5040568,4.4999075,2606468 +Missouri,2009,2007,1.8801448,6.5544538,5433153 +New Mexico,2009,,2.310261,10.077054,1782739 +Maryland,2009,,2.0547636,7.8049917,5162430 +Michigan,2009,2006,1.8254632,6.2056689,9688555 +Tennessee,2009,2007,2.0279765,7.5986943,5541337 +Illinois,2009,,1.799418,6.0461273,12097512 +Alabama,2009,2006,1.9386841,6.9496002,4332380 +Oregon,2009,,.85903895,2.3608906,3343908 +Arizona,2009,2006,1.7930975,6.0080333,5020782 +Indiana,2009,2006,1.59315,4.91922,5902331 +South Carolina,2009,2006,1.932296,6.9053473,3876975 +South Dakota,2009,2006,1.3261241,3.7664168,726426 +Pennsylvania,2009,,1.6686417,5.3049569,11847753 +Massachusetts,2009,,.97718638,2.65697,6127881 +Maine,2009,,.68062586,1.9751135,1240011 +Rhode Island,2009,,1.1073557,3.0263453,1009503 +Colorado,2009,,1.1887412,3.2829459,4198306 +West Virginia,2009,2008,1.5341301,4.6372895,1765197 +Washington,2009,,1.0779191,2.9385586,5757739 +Georgia,2009,2006,1.7849628,5.9593582,7952631 +Virginia,2009,,1.5668795,4.7916722,6847117 +Delaware,2009,,1.5573639,4.7462931,759017 +Montana,2010,2009,.97188169,2.6429129,877433 +Florida,2010,2005,1.66493,5.2853031,15593433 +West Virginia,2010,2008,1.210796,3.3561552,1765197 +Kansas,2010,2006,1.2611367,3.5294309,2606468 +Idaho,2010,,.3014887,1.3518698,1262457 +Maryland,2010,,2.0148253,7.4994168,5162430 +Arizona,2010,2006,1.8633447,6.4452577,5020782 +Georgia,2010,2006,1.7586826,5.8047853,7952631 +Iowa,2010,,.25137061,1.2857865,2822155 +Colorado,2010,,.88141984,2.4143252,4198306 +Delaware,2010,,1.6831394,5.3824272,759017 +North Carolina,2010,,1.6178561,5.0422688,7795432 +North Dakota,2010,2007,.40694466,1.502221,618569 +Kentucky,2010,2006,1.4657922,4.3309727,3926965 +Nebraska,2010,,1.0913255,2.9782193,1660445 +Louisiana,2010,2006,2.4298644,11.357343,4333011 +Virginia,2010,,1.5378079,4.6543765,6847117 +New Hampshire,2010,,-.012426315,.98765057,1200247 +Oregon,2010,,.87184489,2.3913186,3343908 +Alaska,2010,2006,1.4879156,4.4278564,607583 +Ohio,2010,2008,1.4181559,4.1294985,11054019 +Nevada,2010,,1.7738272,5.8933654,1964582 +New York,2010,,1.5009911,4.4861331,18395996 +Washington,2010,,.82546842,2.2829499,5757739 +Wyoming,2010,,.36242896,1.4368151,479699 +Michigan,2010,2006,1.7443621,5.72225,9688555 +Massachusetts,2010,,1.1704749,3.2235231,6127881 +South Carolina,2010,2006,1.809402,6.1067944,3876975 +Illinois,2010,,1.7079242,5.5174966,12097512 +Minnesota,2010,,.59791255,1.8183192,4783596 +Arkansas,2010,,1.5607613,4.7624454,2599492 +Connecticut,2010,,1.2947714,3.6501617,3297626 +Oklahoma,2010,2006,1.6577432,5.2474551,3338279 +Indiana,2010,2006,1.5088098,4.5213466,5902331 +Pennsylvania,2010,,1.6464205,5.1883745,11847753 +Hawaii,2010,,.57720435,1.7810522,1175755 +New Mexico,2010,,1.9413717,6.9683027,1782739 +Mississippi,2010,2006,1.9501445,7.0297036,2749244 +Tennessee,2010,2007,1.7340556,5.6635771,5541337 +New Jersey,2010,,1.4438659,4.2370443,8219529 +California,2010,,1.5877932,4.8929396,33051894 +South Dakota,2010,2006,1.0466433,2.8480749,726426 +Wisconsin,2010,,1.0061625,2.735085,5207717 +Utah,2010,,.66671687,1.9478319,2192689 +Alabama,2010,2006,1.7479398,5.7427597,4332380 +Texas,2010,2007,1.6171008,5.0384617,20290711 +Vermont,2010,,.11336711,1.120043,588067 +Maine,2010,,.59087497,1.8055675,1240011 +Rhode Island,2010,,1.0122645,2.7518253,1009503 +Missouri,2010,2007,1.9525299,7.0464916,5433153 diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index fda492c1e..23d261ae7 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -2144,6 +2144,7 @@ Exact reviewed artifacts (PDF SHA-256) and live-verified SSRN metadata are pinne Full maintainer paper reviews (equation-level detail, replication targets): `docs/methodology/papers/lee-wooldridge-2025-review.md`, `docs/methodology/papers/lee-wooldridge-2026-review.md`. - **Note:** Registry entry authored with the paper reviews ahead of the implementation (PR #588, third-party contribution under maintainer revision). Checklist boxes are unchecked until the implementation lands; implementation-specific edge-case notes will be finalized in that PR. +- **Note:** Maintainer validation suite: `tests/test_methodology_lwdid.py` (import-skip-gated until `diff_diff.lwdid` exists; xfail markers encode PR #588's outstanding acceptance criteria; replication goldens in `benchmarks/data/lwdid_walmart_eventstudy_golden.json` and `benchmarks/data/real/castle_lw_subset.csv`). **Key implementation requirements:** diff --git a/tests/test_methodology_lwdid.py b/tests/test_methodology_lwdid.py new file mode 100644 index 000000000..1a2fe9684 --- /dev/null +++ b/tests/test_methodology_lwdid.py @@ -0,0 +1,912 @@ +"""Independent methodology-validation suite for the LWDiD estimator (PR #588). + +This module is the maintainer-side acceptance suite for the third-party LWDiD +(Lee & Wooldridge rolling-transformation DiD) contribution. It is merged to +main BEFORE the estimator exists: the module-level ``pytest.importorskip`` +makes it skip cleanly until ``diff_diff.lwdid`` lands, at which point every +test activates automatically on the estimator branch. + +This module verifies that the LWDiD implementation matches: + +1. The published replication targets of Lee & Wooldridge (2026), Tables 3, 4 + and A1 (California Prop 99, three donor pools) including exact-inference + and randomization-inference p-values. +2. The castle-doctrine staggered application of LW (2026) Section 7.2 + (tau_omega via the composite-outcome regression (7.18)/(7.19)). +3. The event-study replication targets of Lee & Wooldridge (2025), Appendix F + Tables A4/A5 (Walmart entry), via the normative event-study API specified + in ``TestEventStudySpec`` (xfail until PR #588's Appendix D work lands). +4. Estimator-independent properties: translation invariance of SEs, + cross-estimator equivalences (plain DiD; per-period panel-DiD identity of + LW 2026 eq. (2.20); the T=3 detrending closed form of LW 2025 eq. (5.7)), + from-scratch reference implementations of Procedures 2.1/3.1 and the + staggered per-cohort demeaning, exact small-sample t inference + (T_{N-2} / T_{N-K-2}), and the Monte Carlo bias ordering of LW 2026 + Section 5 under heterogeneous trends. +5. REGISTRY.md edge cases: minimum pre-treatment periods (>= 1 demeaning, + >= 2 detrending). + +xfail semantics (first use of xfail in this codebase): an ``xfail`` marker +encodes an agreed, outstanding work item of PR #588 - the marker's ``reason`` +names the item. ``strict=True`` markers MUST be removed by the commit that +fixes the item (the test then passing turns XPASS into a hard failure, +forcing explicit acceptance). ``strict=False`` is used only where numerical +fragility across environments is plausible. + +Data sources: + +- Prop 99 / Walmart: ``load_prop99()`` / ``load_walmart()`` (authors' SSC + ancillary data; checksum-pinned). Tests skip unless + ``df.attrs["source"] == "lwdid_ssc_ancillary"`` (i.e. real data, not the + synthetic offline fallback). +- Castle doctrine: ``benchmarks/data/real/castle_lw_subset.csv``, the real + Cheng & Hoekstra (2013) panel as packaged by Cunningham (2021) and + distributed with PR #588 (extracted from commit 8c5cccea; columns state, + year, effyear, lhomicide, homicide, population). Committed here because + the ``load_castle_doctrine`` upstream source is currently unavailable. +- Walmart event-study goldens: + ``benchmarks/data/lwdid_walmart_eventstudy_golden.json`` (Tables A4/A5, + provenance embedded in the file). + +References: + +- Lee, S.J., & Wooldridge, J.M. (2025). A Simple Transformation Approach to + Difference-in-Differences Estimation for Panel Data. SSRN No. 4516518 + (revision of June 8, 2026). https://ssrn.com/abstract=4516518 +- Lee, S.J., & Wooldridge, J.M. (2026). Simple Approaches to Inference with + Difference-in-Differences Estimators with Small Cross-Sectional Sample + Sizes. SSRN No. 5325686 (cover date February 3, 2026). + https://ssrn.com/abstract=5325686 +- docs/methodology/REGISTRY.md, section "LWDiD". +- docs/methodology/papers/lee-wooldridge-{2025,2026}-review.md. +""" + +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from scipy import stats + +pytest.importorskip( + "diff_diff.lwdid", + reason="LWDiD estimator not yet on main (arrives via PR #588)", +) + +from diff_diff.lwdid import LWDiD # noqa: E402 + +from diff_diff import ( # noqa: E402 + DifferenceInDifferences, # noqa: E402 + load_prop99, + load_walmart, +) + +# --------------------------------------------------------------------------- +# Published replication targets (LW 2026; see module docstring for provenance) +# --------------------------------------------------------------------------- + +# Table 3 (38-state donor pool): {procedure: (ATT, SE)} +TABLE3_AVERAGE = {"demean": (-0.422, 0.121), "detrend": (-0.227, 0.094)} +TABLE3_PER_PERIOD = { + "demean": {1989: (-0.168, 0.096), 1995: (-0.484, 0.137), 2000: (-0.667, 0.164)}, + "detrend": {1989: (-0.043, 0.059), 1995: (-0.282, 0.112), 2000: (-0.403, 0.152)}, +} +TABLE3_DETREND_EXACT_P = 0.021 +TABLE3_DETREND_RI_P = 0.020 + +# Table 4 (Southern pool) and Table A1 (Midwestern pool): Average rows +SOUTHERN_POOL = ["Alabama", "Arkansas", "Louisiana", "Mississippi"] +TABLE4_AVERAGE = {"demean": (-0.556, 0.080), "detrend": (-0.215, 0.039)} +MIDWEST_POOL = ["Illinois", "Iowa", "Minnesota", "Ohio"] +TABLEA1_AVERAGE = {"demean": (-0.413, 0.118), "detrend": (-0.198, 0.079)} + +# Castle-laws application (Section 7.2): tau_omega targets +CASTLE_COHORTS = {2005: 1, 2006: 13, 2007: 4, 2008: 2, 2009: 1} +CASTLE_TAU_DEMEAN = (0.092, 0.057) # (tau_omega, usual OLS SE) +CASTLE_TAU_DETREND = 0.067 + +# Printed-precision tolerance for 3-decimal published tables +PRINTED_ATOL = 1e-3 + +_CASTLE_CSV = ( + Path(__file__).resolve().parent.parent / "benchmarks" / "data" / "real" / "castle_lw_subset.csv" +) +_WALMART_ES_GOLDEN = ( + Path(__file__).resolve().parent.parent + / "benchmarks" + / "data" + / "lwdid_walmart_eventstudy_golden.json" +) + +XFAIL_IPW_CENTERING = pytest.mark.xfail( + strict=True, + reason="PR #588 step-2 item 1: IPW influence function is un-centered, " + "making the IPW SE translation-variant. Remove this marker in the " + "commit that centers the IPW IF.", +) +XFAIL_EVENT_STUDY = pytest.mark.xfail( + strict=True, + reason="PR #588 Option A: Appendix D event study + Algorithm 1 " + "multiplier bootstrap not yet implemented. Remove this marker in the " + "commit that implements the event study (deterministic spec tests).", +) +XFAIL_EVENT_STUDY_GOLDENS = pytest.mark.xfail( + strict=False, + reason="PR #588 Option A: Appendix D event study + Algorithm 1 not yet " + "implemented. Non-strict (numerical fragility): the golden SEs are the " + "paper's printed B=999 multiplier-bootstrap draws; a re-seeded bootstrap " + "can sit near the printed-precision tolerance boundary across " + "platforms. Re-calibrate the SE tolerance when the event study lands, " + "then remove the marker.", +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _real_prop99(): + """Load the real Prop 99 panel or skip (offline / synthetic fallback). + + Skips are VISIBLE (not silent): CI runners have network and the loaders + are SHA-256-pinned, so these tests run in practice; a dedicated + real-data canary lane is tracked in TODO.md alongside the legacy-loader + fallback repair. + """ + df = load_prop99() + if df.attrs.get("source") != "lwdid_ssc_ancillary": + pytest.skip("real Prop 99 data unavailable (synthetic fallback in use)") + return df + + +def _real_walmart(): + """Load the real Walmart panel or skip (offline / synthetic fallback).""" + df = load_walmart() + if df.attrs.get("source") != "lwdid_ssc_ancillary": + pytest.skip("real Walmart data unavailable (synthetic fallback in use)") + return df + + +def _fit_prop99(df, rolling, **kwargs): + est = LWDiD(rolling=rolling, estimator="ra", vce="classical", **kwargs) + return est.fit(df, outcome="lcigsale", unit="state", time="year", treatment="treated") + + +def _demean_reference(df, unit, time, outcome, pre_end): + """From-scratch Procedure 2.1: per-unit pre-mean, post-average residual.""" + out = {} + for u, g in df.groupby(unit): + g = g.sort_values(time) + pre = g.loc[g[time] <= pre_end, outcome].to_numpy() + post = g.loc[g[time] > pre_end, outcome].to_numpy() + out[u] = post.mean() - pre.mean() + return pd.Series(out) + + +def _detrend_reference(df, unit, time, outcome, pre_end): + """From-scratch Procedure 3.1: per-unit pre-period OLS on (1, t), + out-of-sample residuals for post periods, averaged.""" + out = {} + for u, g in df.groupby(unit): + g = g.sort_values(time) + pre = g[g[time] <= pre_end] + post = g[g[time] > pre_end] + X = np.column_stack([np.ones(len(pre)), pre[time].to_numpy(dtype=float)]) + beta, *_ = np.linalg.lstsq(X, pre[outcome].to_numpy(dtype=float), rcond=None) + yhat = beta[0] + beta[1] * post[time].to_numpy(dtype=float) + out[u] = float((post[outcome].to_numpy(dtype=float) - yhat).mean()) + return pd.Series(out) + + +def _cross_section_did(ybar, treated): + """OLS of the collapsed outcome on (1, D): returns (tau, classical SE, df).""" + y = ybar.to_numpy(dtype=float) + d = treated.to_numpy(dtype=float) + X = np.column_stack([np.ones_like(d), d]) + beta, *_ = np.linalg.lstsq(X, y, rcond=None) + resid = y - X @ beta + n, k = X.shape + dof = n - k + sigma2 = resid @ resid / dof + cov = sigma2 * np.linalg.inv(X.T @ X) + return float(beta[1]), float(np.sqrt(cov[1, 1])), dof + + +def _synthetic_common_timing(n_treat=60, n_control=140, t_max=8, s=5, effect=1.5, seed=11): + """Seeded synthetic common-timing panel with one covariate.""" + rng = np.random.default_rng(seed) + rows = [] + for i in range(n_treat + n_control): + is_treated = i < n_treat + a = rng.normal(0, 1) + x = rng.normal(0, 1) + for t in range(1, t_max + 1): + post = t >= s + y = ( + a + + 0.4 * x + + 0.2 * t + + rng.normal(0, 0.6) + + (effect if is_treated and post else 0.0) + ) + rows.append( + { + "unit": i, + "time": t, + "y": y, + "x": x, + "treat": int(is_treated and post), + "treated_group": int(is_treated), + "post": int(post), + } + ) + return pd.DataFrame(rows) + + +def _synthetic_staggered(n_units=120, t_max=10, cohorts=(5, 7), nt_share=0.4, seed=23): + """Seeded synthetic staggered panel: cohorts + never-treated (first_year=0).""" + rng = np.random.default_rng(seed) + rows = [] + for i in range(n_units): + u = rng.uniform() + if u < nt_share: + g = 0 + else: + g = cohorts[int(rng.integers(0, len(cohorts)))] + a = rng.normal(0, 1) + for t in range(1, t_max + 1): + te = 1.0 + 0.1 * (t - g) if (g > 0 and t >= g) else 0.0 + y = a + 0.15 * t + rng.normal(0, 0.5) + te + rows.append( + { + "unit": i, + "time": t, + "y": y, + "first_year": g, + "treat": int(g > 0 and t >= g), + } + ) + return pd.DataFrame(rows) + + +# --------------------------------------------------------------------------- +# 1. Prop 99, Table 3 (38-state donor pool) +# --------------------------------------------------------------------------- + + +@pytest.mark.realdata +class TestProp99Table3Goldens: + """LW (2026) Table 3: the authors' Stata `lwdid` output, frozen in print.""" + + @pytest.fixture(scope="class") + def prop99(self): + return _real_prop99() + + @pytest.mark.parametrize("rolling", ["demean", "detrend"]) + def test_average_att_and_se(self, prop99, rolling): + res = _fit_prop99(prop99, rolling) + att, se = TABLE3_AVERAGE[rolling] + np.testing.assert_allclose(res.att, att, atol=PRINTED_ATOL) + np.testing.assert_allclose(res.se, se, atol=PRINTED_ATOL) + + @pytest.mark.parametrize("rolling", ["demean", "detrend"]) + def test_per_period_atts(self, prop99, rolling): + res = _fit_prop99(prop99, rolling, period_specific=True) + assert res.period_effects, "period_specific=True should populate period_effects" + for year, (att, se) in TABLE3_PER_PERIOD[rolling].items(): + eff = res.period_effects[year] + np.testing.assert_allclose(eff["att"], att, atol=PRINTED_ATOL) + np.testing.assert_allclose(eff["se"], se, atol=PRINTED_ATOL) + + def test_detrend_exact_inference_p_value(self, prop99): + res = _fit_prop99(prop99, "detrend") + np.testing.assert_allclose(res.p_value, TABLE3_DETREND_EXACT_P, atol=PRINTED_ATOL) + + @pytest.mark.xfail( + strict=False, + reason="PR #588 step-2 discussion: RI p-value convention diverges " + "from LW 2026 Table 3 Note 2 (implementation gives the seed-stable " + "~2/39 two-sided exact permutation atom for N1=1 among 39 states - " + "arguably the standard exact answer - vs the paper's 0.020, whose " + "permutation scheme is under-documented; see the maintainer review " + "doc Gaps section). Reconcile against the authors' Stata " + "`lwdid, ri` behavior.", + ) + def test_detrend_randomization_inference_p_value(self, prop99): + from diff_diff.lwdid_randomization import randomization_inference + + ybar = _detrend_reference(prop99, "state", "year", "lcigsale", pre_end=1988) + treated = prop99.groupby("state")["first_year"].first().loc[ybar.index] > 0 + ri = randomization_inference( + ybar.to_numpy(), treated.to_numpy(dtype=float), n_reps=1000, seed=2026 + ) + # Paper: RI p = 0.020 from 1,000 permutations; tolerance covers ~3 + # binomial standard errors at p ~= 0.02 with 1,000 replications. + np.testing.assert_allclose(ri.pvalue, TABLE3_DETREND_RI_P, atol=0.015) + + +# --------------------------------------------------------------------------- +# 2. Prop 99, alternative donor pools (Tables 4 and A1) +# --------------------------------------------------------------------------- + + +@pytest.mark.realdata +class TestDonorPoolVariants: + """LW (2026) Tables 4 / A1: Southern and Midwestern donor pools.""" + + @pytest.fixture(scope="class") + def prop99(self): + return _real_prop99() + + @pytest.mark.parametrize( + "pool,targets", + [(SOUTHERN_POOL, TABLE4_AVERAGE), (MIDWEST_POOL, TABLEA1_AVERAGE)], + ids=["table4_southern", "tableA1_midwest"], + ) + @pytest.mark.parametrize("rolling", ["demean", "detrend"]) + def test_average_att_and_se(self, prop99, pool, targets, rolling): + sub = prop99[prop99["state"].isin(pool + ["California"])] + assert sub["state"].nunique() == 5 + res = _fit_prop99(sub, rolling) + att, se = targets[rolling] + np.testing.assert_allclose(res.att, att, atol=PRINTED_ATOL) + np.testing.assert_allclose(res.se, se, atol=PRINTED_ATOL) + + +# --------------------------------------------------------------------------- +# 3. Castle doctrine (Section 7.2): the aggregation adjudicator +# --------------------------------------------------------------------------- + + +@pytest.mark.realdata +class TestCastleTauOmegaAdjudicator: + """LW (2026) Section 7.2: staggered tau_omega via composite regression. + + This class adjudicates PR #588's staggered aggregation: the paper's + tau_omega comes from the single composite-outcome regression + (7.18)/(7.19); a numerically different aggregation cannot reproduce + the paper's point estimate and OLS SE simultaneously. + """ + + @pytest.fixture(scope="class") + def castle(self): + if not _CASTLE_CSV.exists(): + pytest.skip(f"{_CASTLE_CSV.name} not committed (partial checkout)") + df = pd.read_csv(_CASTLE_CSV) + # Data-integrity assertion: this must be the paper's exact sample. + fy = df.groupby("state")["effyear"].first() + counts = fy.dropna().astype(int).value_counts().sort_index().to_dict() + assert counts == CASTLE_COHORTS, f"castle cohorts {counts} != paper {CASTLE_COHORTS}" + assert df["state"].nunique() == 50 + assert int(fy.isna().sum()) == 29 + df = df.copy() + df["first_year"] = df["effyear"].fillna(0).astype(int) + df["treat"] = ((df["first_year"] > 0) & (df["year"] >= df["first_year"])).astype(int) + return df + + @staticmethod + def _fit(castle, rolling): + # The paper's per-cohort effects (7.10) use never-treated controls; + # PR #588's default control pool is not-yet-treated, which moves the + # castle point estimate from 0.092 to 0.074 (calibrated 2026-07-13). + est = LWDiD(rolling=rolling, estimator="ra", vce="classical", control_group="never_treated") + return est.fit( + castle, + outcome="lhomicide", + unit="state", + time="year", + treatment="treat", + cohort="first_year", + ) + + @staticmethod + def _composite_reference(castle, rolling): + """From-scratch LW 2026 (7.18)/(7.19): composite outcome + single + cross-sectional regression. Reproduces the paper's printed targets + (verified: demean 0.0917/SE 0.0571; detrend 0.0666).""" + fy = castle.groupby("state")["first_year"].first() + cohorts = sorted(set(fy[fy > 0])) + n_treat = int((fy > 0).sum()) + transform = _demean_reference if rolling == "demean" else _detrend_reference + ydot = {g: transform(castle, "state", "year", "lhomicide", pre_end=g - 1) for g in cohorts} + y, d = [], [] + for s in fy.index: + g = fy[s] + if g > 0: + y.append(ydot[g][s]) + d.append(1.0) + else: + y.append(sum((fy[fy == gg].size / n_treat) * ydot[gg][s] for gg in cohorts)) + d.append(0.0) + tau, se, dof = _cross_section_did(pd.Series(y), pd.Series(d)) + return tau, se + + def test_composite_regression_reference_reproduces_paper(self, castle): + """The (7.18)/(7.19) reference implementation hits the printed targets - + proving the targets are reproducible and the sample construction is + correct, independent of the PR #588 estimator.""" + tau_dm, se_dm = self._composite_reference(castle, "demean") + np.testing.assert_allclose(tau_dm, CASTLE_TAU_DEMEAN[0], atol=PRINTED_ATOL) + np.testing.assert_allclose(se_dm, CASTLE_TAU_DEMEAN[1], atol=PRINTED_ATOL) + tau_dt, _ = self._composite_reference(castle, "detrend") + np.testing.assert_allclose(tau_dt, CASTLE_TAU_DETREND, atol=PRINTED_ATOL) + + def test_demean_tau_omega_point(self, castle): + res = self._fit(castle, "demean") + np.testing.assert_allclose(res.att, CASTLE_TAU_DEMEAN[0], atol=PRINTED_ATOL) + + @pytest.mark.xfail( + strict=True, + reason="PR #588 step-2 aggregation: the overall SE must come from the " + "composite-outcome regression (7.18)/(7.19) (paper OLS SE 0.057; " + "implementation's independence-across-cohorts SE gives 0.051). " + "Remove this marker in the commit that adopts the composite " + "regression.", + ) + def test_demean_tau_omega_ols_se(self, castle): + res = self._fit(castle, "demean") + np.testing.assert_allclose(res.se, CASTLE_TAU_DEMEAN[1], atol=PRINTED_ATOL) + + def test_detrend_tau_omega_point(self, castle): + res = self._fit(castle, "detrend") + np.testing.assert_allclose(res.att, CASTLE_TAU_DETREND, atol=PRINTED_ATOL) + + +# --------------------------------------------------------------------------- +# 4. Translation invariance of SEs (property test) +# --------------------------------------------------------------------------- + + +class TestTranslationInvariance: + """A constant added to all post-period outcomes shifts every transformed + outcome equally: the ATT is invariant, and any correct SE is invariant.""" + + SHIFT = 100.0 + + def _fit_pair(self, estimator): + df = _synthetic_common_timing() + df2 = df.copy() + df2["y"] = df2["y"] + self.SHIFT * df2["post"] + kw = dict(outcome="y", unit="unit", time="time", treatment="treat", controls=["x"]) + r1 = LWDiD(rolling="demean", estimator=estimator, vce="hc1").fit(df, **kw) + r2 = LWDiD(rolling="demean", estimator=estimator, vce="hc1").fit(df2, **kw) + return r1, r2 + + @pytest.mark.parametrize("estimator", ["ra", "ipwra", "ipw"]) + def test_att_translation_invariant(self, estimator): + r1, r2 = self._fit_pair(estimator) + np.testing.assert_allclose(r1.att, r2.att, rtol=0, atol=1e-10) + + @pytest.mark.parametrize("estimator", ["ra", "ipwra"]) + def test_se_translation_invariant(self, estimator): + r1, r2 = self._fit_pair(estimator) + np.testing.assert_allclose(r1.se, r2.se, rtol=0, atol=1e-10) + + @XFAIL_IPW_CENTERING + def test_ipw_se_translation_invariant(self): + r1, r2 = self._fit_pair("ipw") + np.testing.assert_allclose(r1.se, r2.se, rtol=0, atol=1e-10) + + +# --------------------------------------------------------------------------- +# 5. Cross-estimator equivalences +# --------------------------------------------------------------------------- + + +class TestCrossEstimatorEquivalence: + """Theory-mandated numerical identities against already-validated code.""" + + def test_demean_ra_equals_plain_did(self): + """Common timing, no covariates: rolling demeaning + RA reproduces the + standard DiD estimator exactly (LW 2026, eq. (2.5) / Section 9).""" + df = _synthetic_common_timing() + lw = LWDiD(rolling="demean", estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + dd = DifferenceInDifferences().fit(df, outcome="y", treatment="treated_group", time="post") + np.testing.assert_allclose(lw.att, dd.att, rtol=0, atol=1e-10) + + @pytest.mark.parametrize("r", [5, 7]) + def test_per_period_equals_subset_panel_did(self, r): + """LW 2026 eq. (2.20): the per-period effect tau_hat_{t,DD} is + numerically identical to a standard panel DiD run on periods + {1, ..., S-1, t}.""" + df = _synthetic_common_timing(t_max=8, s=5) + res = LWDiD(rolling="demean", estimator="ra", vce="classical", period_specific=True).fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + sub = df[(df["time"] < 5) | (df["time"] == r)] + dd = DifferenceInDifferences().fit(sub, outcome="y", treatment="treated_group", time="post") + np.testing.assert_allclose(res.period_effects[r]["att"], dd.att, rtol=0, atol=1e-10) + + def test_detrend_t3_closed_form(self): + """LW 2025 eq. (5.7): with T=3, S=3, no covariates, the detrending + estimator equals the difference-in-difference-in-differences of + period means.""" + rng = np.random.default_rng(7) + rows = [] + for i in range(80): + treated = i < 30 + a, b = rng.normal(0, 1), rng.normal(0.1, 0.05) + for t in (1, 2, 3): + y = a + b * t + rng.normal(0, 0.3) + (0.8 if treated and t == 3 else 0.0) + rows.append({"unit": i, "time": t, "y": y, "treat": int(treated and t == 3)}) + df = pd.DataFrame(rows) + res = LWDiD(rolling="detrend", estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + m = df.assign(g=(df["unit"] < 30).astype(int)).groupby(["g", "time"])["y"].mean() + closed = ((m[1, 3] - m[1, 2]) - (m[0, 3] - m[0, 2])) - ( + (m[1, 2] - m[1, 1]) - (m[0, 2] - m[0, 1]) + ) + np.testing.assert_allclose(res.att, closed, rtol=0, atol=1e-10) + + +# --------------------------------------------------------------------------- +# 6. From-scratch reference implementations +# --------------------------------------------------------------------------- + + +class TestFromScratchReference: + """Pure-numpy reimplementation of the procedures, blind to the estimator.""" + + def test_procedure_2_1_demeaning(self): + df = _synthetic_common_timing(t_max=8, s=5) + ybar = _demean_reference(df, "unit", "time", "y", pre_end=4) + treated = df.groupby("unit")["treated_group"].first().loc[ybar.index] + tau, se, dof = _cross_section_did(ybar, treated) + res = LWDiD(rolling="demean", estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + np.testing.assert_allclose(res.att, tau, rtol=0, atol=1e-10) + np.testing.assert_allclose(res.se, se, rtol=1e-8) + + def test_procedure_3_1_detrending(self): + df = _synthetic_common_timing(t_max=8, s=5) + ybar = _detrend_reference(df, "unit", "time", "y", pre_end=4) + treated = df.groupby("unit")["treated_group"].first().loc[ybar.index] + tau, se, dof = _cross_section_did(ybar, treated) + res = LWDiD(rolling="detrend", estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + np.testing.assert_allclose(res.att, tau, rtol=0, atol=1e-10) + np.testing.assert_allclose(res.se, se, rtol=1e-8) + + def test_staggered_per_cohort_demeaning(self): + """LW 2026 (7.4)/(7.9)/(7.10): per-cohort post-average demeaned + outcome, regression on the cohort + never-treated subsample + (control_group='never_treated' matches the (7.10) sample).""" + df = _synthetic_staggered() + res = LWDiD( + rolling="demean", estimator="ra", vce="classical", control_group="never_treated" + ).fit(df, outcome="y", unit="unit", time="time", treatment="treat", cohort="first_year") + assert res.cohort_effects, "staggered fit should populate cohort_effects" + fy = df.groupby("unit")["first_year"].first() + for g in sorted(set(fy[fy > 0])): + members = fy.index[(fy == g) | (fy == 0)] + sub = df[df["unit"].isin(members)] + ybar = _demean_reference(sub, "unit", "time", "y", pre_end=g - 1) + treated = (fy.loc[ybar.index] == g).astype(int) + tau, se, dof = _cross_section_did(ybar, treated) + eff = res.cohort_effects[g] + np.testing.assert_allclose(eff["att"], tau, rtol=0, atol=1e-10) + + +# --------------------------------------------------------------------------- +# 7. Exact small-sample inference (LW 2026, Section 2) +# --------------------------------------------------------------------------- + + +class TestExactSmallSampleInference: + """The collapsed cross-sectional regression carries exact t inference.""" + + def test_classical_p_value_uses_t_n_minus_2(self): + df = _synthetic_common_timing(n_treat=6, n_control=10, t_max=6, s=4) + res = LWDiD(rolling="demean", estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + n = df["unit"].nunique() + p_expected = 2 * stats.t.sf(abs(res.t_stat), n - 2) + np.testing.assert_allclose(res.p_value, p_expected, rtol=1e-10) + + def test_single_treated_unit_inference_is_finite(self): + """N1 = 1: exact inference remains valid (studentized-residual t).""" + df = _synthetic_common_timing(n_treat=1, n_control=12, t_max=6, s=4) + res = LWDiD(rolling="demean", estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + assert np.isfinite(res.att) + assert np.isfinite(res.se) and res.se > 0 + assert np.isfinite(res.p_value) + n = df["unit"].nunique() + p_expected = 2 * stats.t.sf(abs(res.t_stat), n - 2) + np.testing.assert_allclose(res.p_value, p_expected, rtol=1e-10) + + @pytest.mark.xfail( + strict=True, + reason="PR #588 step-2: no N_infinity >= 2 guard exists for the " + "never-treated-only staggered control strategy (LW 2026, p26).", + ) + def test_never_treated_pool_of_one_is_rejected(self): + df = _synthetic_staggered(n_units=30, nt_share=0.0, seed=5) + # Force exactly one never-treated unit + first_unit = df["unit"] == 0 + df.loc[first_unit, "first_year"] = 0 + df.loc[first_unit, "treat"] = 0 + with pytest.raises(ValueError, match="[Nn]ever[- ]treated"): + # The N_infinity >= 2 requirement applies to the NT-only control + # strategy (LW 2026, p26); NYT controls are exempt. + LWDiD( + rolling="demean", + estimator="ra", + vce="classical", + control_group="never_treated", + ).fit(df, outcome="y", unit="unit", time="time", treatment="treat", cohort="first_year") + + +# --------------------------------------------------------------------------- +# 8. Event-study specification (normative API; Option A work) +# --------------------------------------------------------------------------- + + +@pytest.mark.realdata +class TestEventStudySpec: + """Normative event-study API for LWDiD (maintainer-specified). + + Invocation: ``fit(..., aggregate="event_study")`` (CallawaySantAnna + precedent). Results must expose ``event_study_effects: Dict[int, Dict]`` + keyed by relative period r, each with keys ``effect, se, t_stat, + p_value, conf_int`` and (when simultaneous bands are computed via + Algorithm 1) ``cband_conf_int``; result-level metadata ``cband_method, + cband_crit_value, cband_n_bootstrap``. Anchor periods are excluded: + r = -1 (demeaning); r = -2, -1 (detrending). + + All tests xfail until PR #588's Appendix D + Algorithm 1 work lands. + """ + + @pytest.fixture(scope="class") + def walmart(self): + return _real_walmart() + + @pytest.fixture(scope="class") + def golden(self): + if not _WALMART_ES_GOLDEN.exists(): + pytest.skip(f"{_WALMART_ES_GOLDEN.name} not committed (partial checkout)") + return json.loads(_WALMART_ES_GOLDEN.read_text()) + + def _fit_es(self, walmart, rolling, estimator, outcome="log_retail_emp"): + # The golden SEs are Algorithm 1 multiplier-bootstrap SEs (B = 999): + # the spec requires the bootstrap path, not analytical vce. + est = LWDiD(rolling=rolling, estimator=estimator, n_bootstrap=999, bootstrap_seed=42) + return est.fit( + walmart, + outcome=outcome, + unit="cid", + time="year", + treatment="treated", + cohort="first_year", + aggregate="event_study", + ) + + @XFAIL_EVENT_STUDY + @pytest.mark.parametrize( + "outcome,table_key", + [ + ("log_retail_emp", "table_a4_log_retail"), + ("log_wholesale_emp", "table_a5_log_wholesale"), + ], + ids=["a4_retail", "a5_wholesale"], + ) + @pytest.mark.parametrize( + "rolling,estimator,column", + [ + ("detrend", "ra", "rolling_ra_detrend"), + ("detrend", "ipwra", "rolling_ipwra_detrend"), + ("demean", "ipwra", "rolling_ipwra_demean"), + ], + ) + def test_walmart_eventstudy_point_goldens( + self, walmart, golden, rolling, estimator, column, outcome, table_key + ): + """Deterministic WATT(r) point estimates vs Tables A4/A5 (strict).""" + res = self._fit_es(walmart, rolling, estimator, outcome=outcome) + table = golden[table_key] + for r_str, cols in table.items(): + r = int(r_str) + att, _se = cols[column] + eff = res.event_study_effects[r] + np.testing.assert_allclose(eff["effect"], att, atol=PRINTED_ATOL) + + @XFAIL_EVENT_STUDY_GOLDENS + @pytest.mark.parametrize( + "outcome,table_key", + [ + ("log_retail_emp", "table_a4_log_retail"), + ("log_wholesale_emp", "table_a5_log_wholesale"), + ], + ids=["a4_retail", "a5_wholesale"], + ) + @pytest.mark.parametrize( + "rolling,estimator,column", + [ + ("detrend", "ra", "rolling_ra_detrend"), + ("detrend", "ipwra", "rolling_ipwra_detrend"), + ("demean", "ipwra", "rolling_ipwra_demean"), + ], + ) + def test_walmart_eventstudy_se_goldens( + self, walmart, golden, rolling, estimator, column, outcome, table_key + ): + """Bootstrap SEs vs the paper's printed B=999 draws (non-strict: + re-seeded bootstrap noise can sit near printed precision).""" + res = self._fit_es(walmart, rolling, estimator, outcome=outcome) + table = golden[table_key] + for r_str, cols in table.items(): + r = int(r_str) + _att, se = cols[column] + eff = res.event_study_effects[r] + np.testing.assert_allclose(eff["se"], se, atol=PRINTED_ATOL) + + @XFAIL_EVENT_STUDY + def test_anchor_periods_excluded(self, walmart): + res_dm = self._fit_es(walmart, "demean", "ra") + assert -1 not in res_dm.event_study_effects + res_dt = self._fit_es(walmart, "detrend", "ra") + assert -1 not in res_dt.event_study_effects + assert -2 not in res_dt.event_study_effects + + def test_detrend_insample_residuals_sum_to_zero(self): + """LW 2026 (pp20-21): per-unit detrended residuals sum to zero over + the fitted pre-window - a property of OLS-with-intercept residuals + IN SAMPLE. NOTE: this identity does NOT transfer to the reported + staggered placebo coefficients (anchor-excluded, D.3-pooled, + cohort-weighted contrasts) - do not assert a sum-to-zero on + ``event_study_effects``; a correct implementation need not satisfy + it there. Verified here on the reference transformation.""" + df = _synthetic_common_timing(t_max=8, s=5) + for _, g in df.groupby("unit"): + g = g.sort_values("time") + pre = g[g["time"] <= 4] + X = np.column_stack([np.ones(len(pre)), pre["time"].to_numpy(dtype=float)]) + beta, *_ = np.linalg.lstsq(X, pre["y"].to_numpy(dtype=float), rcond=None) + resid = pre["y"].to_numpy(dtype=float) - X @ beta + np.testing.assert_allclose(resid.sum(), 0.0, atol=1e-9) + + @XFAIL_EVENT_STUDY + def test_simultaneous_band_metadata(self, walmart): + res = self._fit_es(walmart, "detrend", "ra") + assert res.cband_method is not None + assert res.cband_n_bootstrap >= 999 + any_r = next(iter(res.event_study_effects)) + assert "cband_conf_int" in res.event_study_effects[any_r] + + +# --------------------------------------------------------------------------- +# 9. Monte Carlo bias ordering (LW 2026, Section 5) +# --------------------------------------------------------------------------- + + +@pytest.mark.slow +class TestMonteCarloBiasOrdering: + """Under the paper's heterogeneous-trend DGP (Table 1, Scenario 1), + demeaning is badly biased while detrending is nearly unbiased + (Table 2: bias 1.914 vs 0.009).""" + + N, T, S = 20, 20, 11 + LAMBDAS = np.array( + [0, 0, 0, 0, 0.2, 0.6, 0.7, 0.8, 0.6, 0.9, 0.9, 1, 1.1, 1.3, 1.2, 1.5, 0.6, 1.4, 1.8, 1.9] + ) + DELTAS = np.array([1, 2, 3, 3, 3, 2, 2, 2, 1, 1], dtype=float) + + def _one_rep(self, rng): + n, t_max, s = self.N, self.T, self.S + c = rng.normal(0, 2, n) + g = rng.normal(1, 1, n) + a0, a1, a2 = -1.0, -1.0 / 3.0, 0.25 + d = (a0 - a1 * c + a2 * g + rng.logistic(0, 1, n) > 0).astype(int) + if d.sum() in (0, n): # degenerate assignment; caller redraws + return None + u = np.zeros((n, t_max)) + u[:, 0] = rng.normal(0, np.sqrt(2 / (1 - 0.75**2)), n) + for t in range(1, t_max): + u[:, t] = 0.75 * u[:, t - 1] + rng.normal(0, np.sqrt(2), n) + ts = np.arange(1, t_max + 1) + y0 = self.LAMBDAS[None, :] - c[:, None] + g[:, None] * ts[None, :] + u + y = y0.copy() + post = ts >= s + delta_full = np.zeros(t_max) + delta_full[s - 1 :] = self.DELTAS + y[d == 1] = y0[d == 1] + delta_full[None, :] + rng.normal(0, np.sqrt(2), (d.sum(), t_max)) + sample_att = delta_full[post].mean() + df = pd.DataFrame( + { + "unit": np.repeat(np.arange(n), t_max), + "time": np.tile(ts, n), + "y": y.ravel(), + "treat": (np.repeat(d, t_max) * np.tile(post.astype(int), n)), + } + ) + out = {} + for rolling in ("demean", "detrend"): + res = LWDiD(rolling=rolling, estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + out[rolling] = res.att - sample_att + return out + + def test_bias_ordering_under_heterogeneous_trends(self): + rng = np.random.default_rng(20260713) + errs = {"demean": [], "detrend": []} + reps = 0 + while reps < 200: + rep = self._one_rep(rng) + if rep is None: + continue + errs["demean"].append(rep["demean"]) + errs["detrend"].append(rep["detrend"]) + reps += 1 + bias_dm = abs(float(np.mean(errs["demean"]))) + bias_dt = abs(float(np.mean(errs["detrend"]))) + assert bias_dt < 0.5, f"detrending bias {bias_dt:.3f} unexpectedly large" + assert bias_dm > 3 * max( + bias_dt, 0.15 + ), f"bias ordering violated: demean {bias_dm:.3f} vs detrend {bias_dt:.3f}" + + +# --------------------------------------------------------------------------- +# 10. Minimum pre-treatment periods (REGISTRY edge case) +# --------------------------------------------------------------------------- + + +class TestMinimumPrePeriods: + """Demeaning requires >= 1 pre-period; detrending >= 2 (rank condition, + LW 2025 Appendix B).""" + + @staticmethod + def _panel(first_period_treated): + rng = np.random.default_rng(3) + rows = [] + for i in range(30): + treated = i < 10 + for t in range(1, 6): + post = t >= first_period_treated + rows.append( + { + "unit": i, + "time": t, + "y": rng.normal(0, 1) + (0.5 if treated and post else 0), + "treat": int(treated and post), + } + ) + return pd.DataFrame(rows) + + def test_demeaning_with_single_pre_period_works(self): + df = self._panel(first_period_treated=2) # exactly one pre-period + res = LWDiD(rolling="demean", estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + assert np.isfinite(res.att) + + def test_detrending_with_single_pre_period_warns_and_nans(self): + """One pre-period is rank-deficient for detrending. Calibrated + behavior (2026-07-13): the implementation warns per unit and returns + NaN inference - loud, house-compatible (warn + NaN, never silent).""" + df = self._panel(first_period_treated=2) # one pre-period: rank-deficient + with pytest.warns(UserWarning, match="at least 2 pre-treatment periods"): + res = LWDiD(rolling="detrend", estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + assert np.isnan(res.att) + assert np.isnan(res.se) + # The FULL inference tuple must be NaN together (house NaN contract) + assert np.isnan(res.t_stat) + assert np.isnan(res.p_value) + assert np.isnan(res.conf_int[0]) and np.isnan(res.conf_int[1]) + + def test_detrending_with_two_pre_periods_works(self): + df = self._panel(first_period_treated=3) # two pre-periods: minimum + res = LWDiD(rolling="detrend", estimator="ra", vce="classical").fit( + df, outcome="y", unit="unit", time="time", treatment="treat" + ) + assert np.isfinite(res.att)