From e36ba50d3853688fda1157491edf6b2c0f5935cd Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sat, 22 Aug 2026 11:53:00 -0500 Subject: [PATCH 1/8] perf(shaping): vectorize point-geometry and flat-properties frame builds Two fast paths in the OGC feature-frame builders, both falling back to the previous implementation whenever their precondition fails: - _properties_frame: flat properties (every Water Data / NGWMN collection) build with pd.DataFrame instead of pd.json_normalize (~2x). One nested value anywhere routes the page through json_normalize as before. - _spatial_feature_frame: all-2D-point pages build geometry with one vectorized shapely.points call instead of the per-feature Python walk in GeoDataFrame.from_features (~3.4x). Any non-point or malformed geometry falls back. This CPU runs on the fan-out's event loop, so it is on the critical path of every chunked call. Measured on a 93k-row, 12-chunk get_daily: CPU path -45%, cache-hot wall 1.33s -> 0.95s median (7 trials/arm); output verified identical against recorded API responses, including missing-geometry, polygon, nested-properties, and properties-id edge cases. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UKweWmHg8cu1WuJ17kiHdh --- dataretrieval/ogc/shaping.py | 77 +++++++++++++++++++++++++++- tests/waterdata_utils_test.py | 95 +++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index eb8a692b7..fd6eda1de 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -22,6 +22,7 @@ try: import geopandas as gpd + import shapely GEOPANDAS = True except ImportError: @@ -99,20 +100,94 @@ def _geo_feature_frame(features: list[dict[str, Any]]) -> pd.DataFrame: ) +def _properties_frame(properties: list[dict[str, Any]]) -> pd.DataFrame: + """Build the properties frame, normalizing only when values nest. + + ``json_normalize`` pays a per-cell Python walk to flatten nested objects, + but every Water Data / NGWMN collection publishes flat properties, where a + plain ``DataFrame`` build is ~2x faster. The guard must scan every row: + a nested value in *any* feature means a plain build would leave raw dicts + in rows that ``json_normalize`` would have flattened into columns. + """ + if any(isinstance(value, dict) for row in properties for value in row.values()): + return pd.json_normalize(properties, sep="_") + return pd.DataFrame(properties) + + def _plain_feature_frame( features: list[dict[str, Any]], *, include_geometry: bool ) -> pd.DataFrame: """Build a plain DataFrame from GeoJSON features.""" properties = [feature.get("properties") or {} for feature in features] - df = pd.json_normalize(properties, sep="_") + df = _properties_frame(properties) df["id"] = [feature.get("id") for feature in features] if include_geometry: _attach_coordinates(df, features) return df +#: Sentinel for a geometry the vectorized point build can't represent. +_NON_POINT = object() + + +def _point_xy(feature: dict[str, Any]) -> Any: + """One feature's 2-D point coordinates: ``None`` when it has no + geometry, :data:`_NON_POINT` for anything else the fast path can't build. + """ + geometry = feature.get("geometry") + if geometry is None: + return None + xy = geometry.get("coordinates") + if ( + geometry.get("type") == "Point" + and isinstance(xy, (list, tuple)) + and len(xy) == 2 + ): + return xy + return _NON_POINT + + +def _point_geometries(features: list[dict[str, Any]]) -> Any: + """Build a vectorized shapely geometry array for all-2D-point features. + + Returns ``None`` when any feature carries a non-point (or malformed) + geometry, so the caller falls back to ``GeoDataFrame.from_features``. + ``shapely.points`` over one coordinate array skips the per-feature + Python object walk ``from_features`` does — ~4x faster at page scale. + A feature with no geometry stays ``None`` in the result, matching + ``from_features``. + """ + coords = [_point_xy(feature) for feature in features] + if any(xy is _NON_POINT for xy in coords): + return None + nan = float("nan") + try: + points = shapely.points([xy if xy is not None else (nan, nan) for xy in coords]) + except (TypeError, ValueError): + return None + missing = [index for index, xy in enumerate(coords) if xy is None] + if missing: + points[missing] = None + return points + + def _spatial_feature_frame(features: list[dict[str, Any]]) -> pd.DataFrame: """Build a GeoDataFrame from GeoJSON features with ``id`` first.""" + points = _point_geometries(features) + if points is not None: + frame = _properties_frame( + [feature.get("properties") or {} for feature in features] + ) + # A ``geometry`` *property* would collide with the geometry column + # this constructor names; no known collection has one, so take the + # slow path rather than guess a resolution here. + if "geometry" not in frame.columns: + df = gpd.GeoDataFrame(frame, geometry=points, crs=_CRS) + # Assignment, not ``insert``: a properties ``id`` column must be + # overwritten by the feature-level id, as the fallback path does. + df["id"] = [f.get("id") for f in features] + ordered = ["id", "geometry"] + return df[ordered + [col for col in df.columns if col not in ordered]] df = _geo_feature_frame(features) df["id"] = [f.get("id") for f in features] return df[["id"] + [col for col in df.columns if col != "id"]] diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 27e4b9b79..e80c23c51 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1345,3 +1345,98 @@ def test_credential_shaped_queryables_are_rejected(name): ) def test_real_queryables_still_pass_through(name): assert _flatten_queryables({"queryables": {name: "v"}}) == {name: "v"} + + +# --------------------------------------------------------------------------- +# Feature-frame fast paths (vectorized points, flat properties) +# --------------------------------------------------------------------------- + +_POINT_FEATURES = [ + { + "id": "f-1", + "properties": {"id": "wire-1", "value": "1", "site": "USGS-A"}, + "geometry": {"type": "Point", "coordinates": [-77.1, 38.9]}, + }, + { + "id": "f-2", + "properties": {"id": "wire-2", "value": "2", "site": "USGS-B"}, + "geometry": {"type": "Point", "coordinates": [-80.0, 40.0]}, + }, + { + # No geometry at all (NGWMN observation shape). + "id": "f-3", + "properties": {"id": "wire-3", "value": "3", "site": "USGS-C"}, + }, +] + + +@pytest.mark.skipif(not _shaping_module.GEOPANDAS, reason="requires geopandas") +def test_spatial_fast_path_matches_from_features(): + """The vectorized point build is a pure speedup: identical frame, + column order, CRS, and missing-geometry handling as the + ``from_features`` fallback — including the feature-level ``id`` + overwriting a properties ``id`` column.""" + fast = _shaping_module._spatial_feature_frame(_POINT_FEATURES) + with mock.patch.object(_shaping_module, "_point_geometries", return_value=None): + fallback = _shaping_module._spatial_feature_frame(_POINT_FEATURES) + + pd.testing.assert_frame_equal(fast, fallback) + assert fast.crs == "EPSG:4326" + assert list(fast["id"]) == ["f-1", "f-2", "f-3"] + assert fast.geometry.isna().tolist() == [False, False, True] + + +@pytest.mark.skipif(not _shaping_module.GEOPANDAS, reason="requires geopandas") +def test_spatial_non_point_geometry_uses_from_features(): + """A non-point geometry anywhere disables the vectorized build; the + result still carries the real geometry via ``from_features``.""" + features = _POINT_FEATURES[:1] + [ + { + "id": "f-poly", + "properties": {"value": "4"}, + "geometry": { + "type": "Polygon", + "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 0]]], + }, + } + ] + assert _shaping_module._point_geometries(features) is None + df = _shaping_module._spatial_feature_frame(features) + assert df.geometry.iloc[1].geom_type == "Polygon" + + +def test_point_geometries_rejects_malformed_coordinates(): + """3-D or non-pair coordinates disable the fast path rather than + building a wrong geometry.""" + pytest.importorskip("geopandas") + threed = [ + { + "id": "f", + "properties": {}, + "geometry": {"type": "Point", "coordinates": [1.0, 2.0, 3.0]}, + } + ] + assert _shaping_module._point_geometries(threed) is None + + +def test_properties_frame_flat_matches_normalize(): + """Flat properties take the plain-DataFrame path and match + ``json_normalize`` exactly.""" + properties = [ + {"a": "1", "b": None}, + {"a": "2", "b": "x"}, + ] + fast = _shaping_module._properties_frame(properties) + pd.testing.assert_frame_equal(fast, pd.json_normalize(properties, sep="_")) + + +def test_properties_frame_nested_still_normalizes(): + """One nested value anywhere routes the whole page through + ``json_normalize`` so no row keeps a raw dict.""" + properties = [ + {"a": "1", "nested": None}, + {"a": "2", "nested": {"x": "y"}}, + ] + df = _shaping_module._properties_frame(properties) + assert "nested_x" in df.columns + assert not any(isinstance(v, dict) for v in df.to_numpy().ravel()) From beac75d6da102237ab05b211c2b8a2fd2a1f75fe Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sat, 22 Aug 2026 11:53:09 -0500 Subject: [PATCH 2/8] perf(transport): free response bodies once their pages are parsed An aggregated response's content was one arbitrary page's bytes (its base's), and keeping it had a real cost: every per-chunk aggregate in FanOut._chunks shared its first page's body, so a ~1-page-per-chunk fan-out held the entire decompressed download until the call finished (32 full pages ~ 1.3 GB). paginate likewise held the first page's body for the whole walk. Clear the body on the merged copy and on the initial response once parsed; status, headers, URL, and elapsed are unchanged, and live per-page responses are untouched. Measured: -19% peak Python-heap on a 12-chunk, 93k-row replay (164 MB -> 133 MB), scaling with chunk count x page size. Behavior change (documented in NEWS): the response behind a completed call's metadata and FanOutInterrupted.partial_response now has an empty body. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UKweWmHg8cu1WuJ17kiHdh --- NEWS.md | 2 ++ dataretrieval/combining.py | 5 +++++ dataretrieval/transport/pagination.py | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/NEWS.md b/NEWS.md index 137ba5bf2..1500a70d2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**08/22/2026:** The OGC feature-frame builders take vectorized fast paths, halving the client-side CPU of a chunked call. Flat feature properties — every Water Data / NGWMN collection — build through a plain `DataFrame` constructor instead of `pd.json_normalize` (~2x), and all-point geometry pages build their `GeoDataFrame` through one vectorized `shapely.points` call instead of the per-feature Python walk in `GeoDataFrame.from_features` (~3–4x). Results are identical — a page with nested properties, or any non-point or malformed geometry, takes the previous path — and this CPU runs inside the fan-out's event loop, so trimming it also tightens chunk overlap (measured: a 93k-row, 12-chunk `get_daily` re-pull fell from 1.33 s to 0.95 s median wall; frame construction alone is 3.1–3.5x faster at 10k–500k rows). Aggregated responses also no longer retain a page body: the response behind a getter's metadata and each per-chunk aggregate held for resume each carried one full JSON page's bytes, which on a many-chunk fan-out kept roughly the whole download resident until the call finished (~19% lower peak Python-heap in a 12-chunk replay, growing with chunk count and page size). **Behavior change:** the aggregated `httpx.Response` a completed call or `FanOutInterrupted.partial_response` carries now has an empty body — it was previously one arbitrary page's bytes, not the query's data; status, headers, URL, and elapsed are unchanged, and per-page responses are untouched. + **08/13/2026:** Warning categories now say what they mean. Two advisories about *upstream data* were emitted as `DeprecationWarning` and as an uncategorized `warnings.warn` respectively; both are now `DataCurrencyWarning` (a `UserWarning` subclass, exported as `dataretrieval.DataCurrencyWarning`). **Behavior change:** WQP's legacy-WQX notice moves from `DeprecationWarning` to `DataCurrencyWarning`. Because `legacy=True` is the default on every WQP getter and the notice is unconditional, a downstream project running `-W error::DeprecationWarning` previously could not call any WQP getter with default arguments; it now can. The flip side is visibility — `DeprecationWarning` is silent by default outside `__main__`, so this notice will now print to stderr for library and notebook callers who never saw it. Silence it with `warnings.filterwarnings("ignore", category=dataretrieval.DataCurrencyWarning)`, or set `legacy=False` where a WQX3.0 profile exists. The NWIS qw-endpoint retirement notice gains the same category (it previously had none, arriving as a bare `UserWarning` that deprecation filters ignored). `DeprecationWarning` now means only that a name in this package is going away, always with a replacement and, where published, a removal date; those horizons are declared once in `dataretrieval._deprecation.REMOVALS` rather than spelled at each call site. **08/11/2026:** Settings resolve through a layered chain instead of the environment alone, and a *configuration profile* is a named set of settings for **one adapter**. The new `dataretrieval.configuration` module resolves every setting in one order, highest first: a configuration passed to an active `dataretrieval.configure(...)` block, a profile that block selected, the setting's `API_USGS_*` environment variable, the adapter's `[]` table in `~/.dataretrieval/config.toml` (or `DATARETRIEVAL_CONFIG`), the file's top-level keys, the adapter's own built-in preference, then the package default. Precedence applies **per setting**, so a file that sets only `concurrency` leaves an environment `API_USGS_PAT` in effect, and a `[ngwmn]` table still inherits every top-level key it does not name. `configure()` takes configuration objects positionally, at most one per adapter and nothing else: `configure(Configuration(api_key=vault.read("usgs/pat")), WaterdataConfiguration.load("bulk"), NgwmnConfiguration(concurrency=4))`. The adapter a configuration targets is a property of its class, so a caller never restates it, and each adapter owns its class in the module that *reads* those settings (`waterdata.WaterdataConfiguration`, `ngwmn.NgwmnConfiguration`, `nwdc.NwdcConfiguration`, `wqp.WqpConfiguration`, `nldi.NldiConfiguration`, `streamstats.StreamstatsConfiguration`) — an adapter accepts only the settings it reads, so `[streamstats] parallel_chunks = 8` is an error rather than a line that quietly does nothing. The block is delivered through a `ContextVar`, so a credential set inside it cannot leak across threads or asyncio tasks, which is what makes it safe for a server or notebook handling several users' keys and is the thing assigning to `os.environ` could never do (issue #352). The file gains named profiles beside each adapter's default profile: `[waterdata]` is always in effect, `[waterdata.bulk]` only when a caller selects it with `WaterdataConfiguration.load("bulk")`, and a selected profile still inherits the default profile and the package-wide keys per setting. A profile named in code outranks the setting's environment variable — the one place the ladder inverts the environment-above-file rule, because losing a deliberate selection to a stale shell export is what a caller would file a bug about. An adapter's configuration may also carry a `base_url`, which redirects that adapter's requests for the duration of the block — a staging instance, a mirror, a recording proxy — and no other adapter's; for Water Data one value moves the OGC collections, the Samples database, the statistics service and the STAC catalog together. It is settable in a `configure()` block only: a `base_url` key in the file and an exported `API_USGS_BASE_URL` each raise rather than being read, since a redirect a config file or a shell profile can set is one no reader of the script can see. The API key does not follow a redirect — it is scoped to the single host that honors it — and is deliberately not per-adapter: it authenticates to the gateway fronting a host, and Water Data and NGWMN share that host, one key, and one hourly quota. `dataretrieval.show_configuration()` reports each setting's effective value and where it came from, naming the profile behind each value (`configure() block [waterdata.bulk]` rather than a bare block), listing the profiles a file defines whether or not this run selected any, and naming any adapter this process has not imported rather than omitting it — without ever printing the key. One parser per setting owns its grammar, so a value means the same thing whichever source wrote it. **Breaking change:** `RetryPolicy.from_env()` is now `RetryPolicy.from_configuration()` and resolves through the whole chain rather than the environment alone. **Behavior change:** a credential-shaped keyword passed to a getter's `**kwargs` query passthrough — Water Data's `**queryables` and every WQP getter's search filters — now raises `TypeError` naming `configure(Configuration(api_key=...))` instead of putting a secret in a URL that clients, proxies and logs retain. The names refused are `api_key=`, `token=`, `x_api_key=`, `password=`, `auth=`, `pat=` and similar spellings; a filter the server actually defines is unaffected. **Bug fix:** `API_USGS_STALL_TIMEOUT` was read straight from `os.environ`, so it could not be set by a `configure()` block or the config file and never appeared in `show_configuration()`; it now resolves through the chain like every other setting. Rationale in ADRs 0009, 0010 and 0011; terms in `CONTEXT.md`. diff --git a/dataretrieval/combining.py b/dataretrieval/combining.py index d052ea712..6ec89a74d 100644 --- a/dataretrieval/combining.py +++ b/dataretrieval/combining.py @@ -111,6 +111,11 @@ def _merge_response( (:func:`~dataretrieval.transport.pagination.paginate`) and the chunked / fan-out aggregation (:func:`_combine_chunk_responses`).""" merged = copy.copy(base) + # Drop the body: an aggregate's content would be one arbitrary page's + # bytes (the base's), and holding it keeps every chunk's first page + # resident for the whole call — the frames are the product, not the raw + # JSON. Cleared on the copy only; ``base`` keeps its body. + merged._content = b"" merged.headers = httpx.Headers(headers_from.headers) merged.elapsed = elapsed if url is not None: diff --git a/dataretrieval/transport/pagination.py b/dataretrieval/transport/pagination.py index 1cc2e27cb..04a7b2d2c 100644 --- a/dataretrieval/transport/pagination.py +++ b/dataretrieval/transport/pagination.py @@ -117,6 +117,11 @@ def report_page(page: httpx.Response, frame: pd.DataFrame) -> None: nrows = len(frame) seen: set[Any] = set() report_page(response, frame) + # The first page's body is parsed and never read again, but the + # response object must survive to the final merge (status, request, + # URL). Free the body now rather than holding one page of JSON for + # the whole walk. + initial_response._content = b"" while ( cursor is not None From 87b70b61c6fbc905601a5bd96343f244e6038b49 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sat, 22 Aug 2026 13:21:02 -0500 Subject: [PATCH 3/8] refactor: converge fast-path tails and name the body-release idiom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-review cleanup of the two perf commits, output-identical (verified against recorded API responses and the full suite): - _spatial_feature_frame: extract the fast build as _point_feature_frame and share one id-overwrite + reorder tail between fast path and from_features fallback (CC 10 -> 5). - _point_geometries: one pass instead of four, bailing on the first non-point feature instead of scanning the whole page first. - _properties_frame: take features (both callers spelled the same extraction) and detect nested values by scanning only object-dtype columns after the cheap build — the full pre-scan cost a third of the win it guarded. MRE: plain 1.8x -> 2.5x, spatial 3.4x -> 4.6x. - combining: name the httpx body-release idiom once (_drop_body, beside _set_response_url) and state the emptied-body contract in _merge_response's docstring; pagination calls the helper. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UKweWmHg8cu1WuJ17kiHdh --- dataretrieval/combining.py | 19 +++++-- dataretrieval/ogc/shaping.py | 80 ++++++++++++++++----------- dataretrieval/transport/pagination.py | 9 ++- tests/waterdata_utils_test.py | 6 +- 4 files changed, 68 insertions(+), 46 deletions(-) diff --git a/dataretrieval/combining.py b/dataretrieval/combining.py index 6ec89a74d..07c9b2b86 100644 --- a/dataretrieval/combining.py +++ b/dataretrieval/combining.py @@ -72,6 +72,15 @@ def _set_response_url(response: httpx.Response, url: str | httpx.URL) -> None: response.request = httpx.Request(method=old.method, url=target, headers=old.headers) +def _drop_body(response: httpx.Response) -> None: + """Free a response's fetched body, keeping status/headers/URL readable. + + ``_content`` is the slot httpx caches a read body in; ``b""`` releases + the bytes while leaving the ``.content`` accessors valid. + """ + response._content = b"" + + def _lowest_remaining(responses: list[httpx.Response]) -> httpx.Response: """The response reporting the lowest ``x-ratelimit-remaining``. @@ -104,18 +113,16 @@ def _merge_response( The copy's ``.headers`` are rebuilt as a fresh ``httpx.Headers`` from ``headers_from``, ``.elapsed`` is set to ``elapsed``, and ``.url`` is - overridden when ``url`` is given. ``base`` and ``headers_from`` are never + overridden when ``url`` is given. The copy's body is emptied — an + aggregate's content would be one arbitrary page's bytes, not the + combined query's data. ``base`` and ``headers_from`` are never mutated, and the fresh ``httpx.Headers`` means downstream mutations don't back-propagate into any underlying response — so callers may re-fold idempotently. This is the one low-level merge behind both pagination (:func:`~dataretrieval.transport.pagination.paginate`) and the chunked / fan-out aggregation (:func:`_combine_chunk_responses`).""" merged = copy.copy(base) - # Drop the body: an aggregate's content would be one arbitrary page's - # bytes (the base's), and holding it keeps every chunk's first page - # resident for the whole call — the frames are the product, not the raw - # JSON. Cleared on the copy only; ``base`` keeps its body. - merged._content = b"" + _drop_body(merged) merged.headers = httpx.Headers(headers_from.headers) merged.elapsed = elapsed if url is not None: diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index fd6eda1de..86e58a87a 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -100,26 +100,29 @@ def _geo_feature_frame(features: list[dict[str, Any]]) -> pd.DataFrame: ) -def _properties_frame(properties: list[dict[str, Any]]) -> pd.DataFrame: - """Build the properties frame, normalizing only when values nest. +def _properties_frame(features: list[dict[str, Any]]) -> pd.DataFrame: + """Build the frame of feature properties, normalizing only when values nest. ``json_normalize`` pays a per-cell Python walk to flatten nested objects, but every Water Data / NGWMN collection publishes flat properties, where a - plain ``DataFrame`` build is ~2x faster. The guard must scan every row: - a nested value in *any* feature means a plain build would leave raw dicts - in rows that ``json_normalize`` would have flattened into columns. + plain ``DataFrame`` build is ~2x faster. A nested value in *any* feature + must route the whole page through ``json_normalize`` so no row keeps a raw + dict — and dicts can only land in object-dtype columns, so scanning those + columns after the cheap build catches every one without a full pre-scan. """ - if any(isinstance(value, dict) for row in properties for value in row.values()): - return pd.json_normalize(properties, sep="_") - return pd.DataFrame(properties) + properties = [feature.get("properties") or {} for feature in features] + frame = pd.DataFrame(properties) + for _, column in frame.items(): + if column.dtype == object and any(isinstance(v, dict) for v in column): + return pd.json_normalize(properties, sep="_") + return frame def _plain_feature_frame( features: list[dict[str, Any]], *, include_geometry: bool ) -> pd.DataFrame: """Build a plain DataFrame from GeoJSON features.""" - properties = [feature.get("properties") or {} for feature in features] - df = _properties_frame(properties) + df = _properties_frame(features) df["id"] = [feature.get("id") for feature in features] if include_geometry: _attach_coordinates(df, features) @@ -157,40 +160,53 @@ def _point_geometries(features: list[dict[str, Any]]) -> Any: A feature with no geometry stays ``None`` in the result, matching ``from_features``. """ - coords = [_point_xy(feature) for feature in features] - if any(xy is _NON_POINT for xy in coords): - return None nan = float("nan") + coords: list[Any] = [] + missing: list[int] = [] + for index, feature in enumerate(features): + xy = _point_xy(feature) + if xy is _NON_POINT: + return None + if xy is None: + missing.append(index) + xy = (nan, nan) + coords.append(xy) try: - points = shapely.points([xy if xy is not None else (nan, nan) for xy in coords]) + points = shapely.points(coords) except (TypeError, ValueError): return None - missing = [index for index, xy in enumerate(coords) if xy is None] if missing: points[missing] = None return points +def _point_feature_frame(features: list[dict[str, Any]]) -> Any: + """Fast-path GeoDataFrame for an all-2D-point page, or ``None`` to fall + back to :func:`_geo_feature_frame`. + + A ``geometry`` *property* would collide with the geometry column this + constructor names; no known collection has one, so that page takes the + slow path rather than guessing a resolution here. + """ + points = _point_geometries(features) + if points is None: + return None + frame = _properties_frame(features) + if "geometry" in frame.columns: + return None + return gpd.GeoDataFrame(frame, geometry=points, crs=_CRS) + + def _spatial_feature_frame(features: list[dict[str, Any]]) -> pd.DataFrame: """Build a GeoDataFrame from GeoJSON features with ``id`` first.""" - points = _point_geometries(features) - if points is not None: - frame = _properties_frame( - [feature.get("properties") or {} for feature in features] - ) - # A ``geometry`` *property* would collide with the geometry column - # this constructor names; no known collection has one, so take the - # slow path rather than guess a resolution here. - if "geometry" not in frame.columns: - df = gpd.GeoDataFrame(frame, geometry=points, crs=_CRS) - # Assignment, not ``insert``: a properties ``id`` column must be - # overwritten by the feature-level id, as the fallback path does. - df["id"] = [f.get("id") for f in features] - ordered = ["id", "geometry"] - return df[ordered + [col for col in df.columns if col not in ordered]] - df = _geo_feature_frame(features) + df = _point_feature_frame(features) + if df is None: + df = _geo_feature_frame(features) + # Assignment, not ``insert``: a properties ``id`` column must be + # overwritten by the feature-level id. df["id"] = [f.get("id") for f in features] - return df[["id"] + [col for col in df.columns if col != "id"]] + ordered = ["id", "geometry"] + return df[ordered + [col for col in df.columns if col not in ordered]] def _get_resp_data( diff --git a/dataretrieval/transport/pagination.py b/dataretrieval/transport/pagination.py index 04a7b2d2c..ea4f0aaf7 100644 --- a/dataretrieval/transport/pagination.py +++ b/dataretrieval/transport/pagination.py @@ -18,6 +18,7 @@ from dataretrieval import progress as _progress from dataretrieval.combining import ( _QUOTA_HEADER, + _drop_body, _merge_response, _safe_elapsed, ) @@ -117,11 +118,9 @@ def report_page(page: httpx.Response, frame: pd.DataFrame) -> None: nrows = len(frame) seen: set[Any] = set() report_page(response, frame) - # The first page's body is parsed and never read again, but the - # response object must survive to the final merge (status, request, - # URL). Free the body now rather than holding one page of JSON for - # the whole walk. - initial_response._content = b"" + # Parsed and never read again; free the body now rather than pinning + # one page of JSON for the whole walk. + _drop_body(initial_response) while ( cursor is not None diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index e80c23c51..20bf69817 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1405,10 +1405,10 @@ def test_spatial_non_point_geometry_uses_from_features(): assert df.geometry.iloc[1].geom_type == "Polygon" +@pytest.mark.skipif(not _shaping_module.GEOPANDAS, reason="requires geopandas") def test_point_geometries_rejects_malformed_coordinates(): """3-D or non-pair coordinates disable the fast path rather than building a wrong geometry.""" - pytest.importorskip("geopandas") threed = [ { "id": "f", @@ -1426,7 +1426,7 @@ def test_properties_frame_flat_matches_normalize(): {"a": "1", "b": None}, {"a": "2", "b": "x"}, ] - fast = _shaping_module._properties_frame(properties) + fast = _shaping_module._properties_frame([{"properties": p} for p in properties]) pd.testing.assert_frame_equal(fast, pd.json_normalize(properties, sep="_")) @@ -1437,6 +1437,6 @@ def test_properties_frame_nested_still_normalizes(): {"a": "1", "nested": None}, {"a": "2", "nested": {"x": "y"}}, ] - df = _shaping_module._properties_frame(properties) + df = _shaping_module._properties_frame([{"properties": p} for p in properties]) assert "nested_x" in df.columns assert not any(isinstance(v, dict) for v in df.to_numpy().ravel()) From 4e494f9e6a569116665e840a91e8459d4e0a82c8 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sat, 22 Aug 2026 13:22:24 -0500 Subject: [PATCH 4/8] docs(news): refresh fast-path speedups after the cleanup pass Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UKweWmHg8cu1WuJ17kiHdh --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 1500a70d2..9d04a0e93 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -**08/22/2026:** The OGC feature-frame builders take vectorized fast paths, halving the client-side CPU of a chunked call. Flat feature properties — every Water Data / NGWMN collection — build through a plain `DataFrame` constructor instead of `pd.json_normalize` (~2x), and all-point geometry pages build their `GeoDataFrame` through one vectorized `shapely.points` call instead of the per-feature Python walk in `GeoDataFrame.from_features` (~3–4x). Results are identical — a page with nested properties, or any non-point or malformed geometry, takes the previous path — and this CPU runs inside the fan-out's event loop, so trimming it also tightens chunk overlap (measured: a 93k-row, 12-chunk `get_daily` re-pull fell from 1.33 s to 0.95 s median wall; frame construction alone is 3.1–3.5x faster at 10k–500k rows). Aggregated responses also no longer retain a page body: the response behind a getter's metadata and each per-chunk aggregate held for resume each carried one full JSON page's bytes, which on a many-chunk fan-out kept roughly the whole download resident until the call finished (~19% lower peak Python-heap in a 12-chunk replay, growing with chunk count and page size). **Behavior change:** the aggregated `httpx.Response` a completed call or `FanOutInterrupted.partial_response` carries now has an empty body — it was previously one arbitrary page's bytes, not the query's data; status, headers, URL, and elapsed are unchanged, and per-page responses are untouched. +**08/22/2026:** The OGC feature-frame builders take vectorized fast paths, halving the client-side CPU of a chunked call. Flat feature properties — every Water Data / NGWMN collection — build through a plain `DataFrame` constructor instead of `pd.json_normalize` (~2.5x), and all-point geometry pages build their `GeoDataFrame` through one vectorized `shapely.points` call instead of the per-feature Python walk in `GeoDataFrame.from_features` (~4.5x). Results are identical — a page with nested properties, or any non-point or malformed geometry, takes the previous path — and this CPU runs inside the fan-out's event loop, so trimming it also tightens chunk overlap (measured: a 93k-row, 12-chunk `get_daily` re-pull fell from 1.33 s to 0.95 s median wall; spatial frame construction alone is 4.2–4.6x faster at 10k–500k rows). Aggregated responses also no longer retain a page body: the response behind a getter's metadata and each per-chunk aggregate held for resume each carried one full JSON page's bytes, which on a many-chunk fan-out kept roughly the whole download resident until the call finished (~19% lower peak Python-heap in a 12-chunk replay, growing with chunk count and page size). **Behavior change:** the aggregated `httpx.Response` a completed call or `FanOutInterrupted.partial_response` carries now has an empty body — it was previously one arbitrary page's bytes, not the query's data; status, headers, URL, and elapsed are unchanged, and per-page responses are untouched. **08/13/2026:** Warning categories now say what they mean. Two advisories about *upstream data* were emitted as `DeprecationWarning` and as an uncategorized `warnings.warn` respectively; both are now `DataCurrencyWarning` (a `UserWarning` subclass, exported as `dataretrieval.DataCurrencyWarning`). **Behavior change:** WQP's legacy-WQX notice moves from `DeprecationWarning` to `DataCurrencyWarning`. Because `legacy=True` is the default on every WQP getter and the notice is unconditional, a downstream project running `-W error::DeprecationWarning` previously could not call any WQP getter with default arguments; it now can. The flip side is visibility — `DeprecationWarning` is silent by default outside `__main__`, so this notice will now print to stderr for library and notebook callers who never saw it. Silence it with `warnings.filterwarnings("ignore", category=dataretrieval.DataCurrencyWarning)`, or set `legacy=False` where a WQX3.0 profile exists. The NWIS qw-endpoint retirement notice gains the same category (it previously had none, arriving as a bare `UserWarning` that deprecation filters ignored). `DeprecationWarning` now means only that a name in this package is going away, always with a replacement and, where published, a removal date; those horizons are declared once in `dataretrieval._deprecation.REMOVALS` rather than spelled at each call site. From da6eca99d41230865934638124442bb80168afc6 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sat, 22 Aug 2026 13:37:24 -0500 Subject: [PATCH 5/8] docs(shaping): trim comments to current-code constraints Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UKweWmHg8cu1WuJ17kiHdh --- dataretrieval/ogc/shaping.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index 86e58a87a..77882e6f3 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -103,12 +103,11 @@ def _geo_feature_frame(features: list[dict[str, Any]]) -> pd.DataFrame: def _properties_frame(features: list[dict[str, Any]]) -> pd.DataFrame: """Build the frame of feature properties, normalizing only when values nest. - ``json_normalize`` pays a per-cell Python walk to flatten nested objects, - but every Water Data / NGWMN collection publishes flat properties, where a - plain ``DataFrame`` build is ~2x faster. A nested value in *any* feature - must route the whole page through ``json_normalize`` so no row keeps a raw - dict — and dicts can only land in object-dtype columns, so scanning those - columns after the cheap build catches every one without a full pre-scan. + Flat properties — every Water Data / NGWMN collection — build with the + plain ``DataFrame`` constructor; a nested value in *any* feature routes + the whole page through ``json_normalize`` so no row keeps a raw dict. + Dicts can only land in object-dtype columns, so scanning those after the + cheap build catches every one. """ properties = [feature.get("properties") or {} for feature in features] frame = pd.DataFrame(properties) @@ -151,14 +150,13 @@ def _point_xy(feature: dict[str, Any]) -> Any: def _point_geometries(features: list[dict[str, Any]]) -> Any: - """Build a vectorized shapely geometry array for all-2D-point features. + """Build the geometry array for all-2D-point features in one vectorized + ``shapely.points`` call. Returns ``None`` when any feature carries a non-point (or malformed) geometry, so the caller falls back to ``GeoDataFrame.from_features``. - ``shapely.points`` over one coordinate array skips the per-feature - Python object walk ``from_features`` does — ~4x faster at page scale. - A feature with no geometry stays ``None`` in the result, matching - ``from_features``. + A feature with no geometry stays ``None`` in the result, matching the + fallback. """ nan = float("nan") coords: list[Any] = [] @@ -202,8 +200,7 @@ def _spatial_feature_frame(features: list[dict[str, Any]]) -> pd.DataFrame: df = _point_feature_frame(features) if df is None: df = _geo_feature_frame(features) - # Assignment, not ``insert``: a properties ``id`` column must be - # overwritten by the feature-level id. + # A properties ``id`` column is overwritten by the feature-level id. df["id"] = [f.get("id") for f in features] ordered = ["id", "geometry"] return df[ordered + [col for col in df.columns if col not in ordered]] From 1257a65196be37746f59fd26a43ac0bf4c504e1e Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sat, 22 Aug 2026 13:37:31 -0500 Subject: [PATCH 6/8] chore(utils): deprecate the orphaned format_datetime A dead-function sweep found exactly one: format_datetime shaped qw service responses and lost its last caller when qwdata usage was removed (491eb5c3); it appears in no docs or demos. Public name, so it warns through the shared mechanism with a 2027-08-22 horizon rather than vanishing. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UKweWmHg8cu1WuJ17kiHdh --- NEWS.md | 2 +- dataretrieval/_deprecation.py | 1 + dataretrieval/utils.py | 11 ++++++++++- tests/deprecation_test.py | 11 +++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9d04a0e93..4a0678a5c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -**08/22/2026:** The OGC feature-frame builders take vectorized fast paths, halving the client-side CPU of a chunked call. Flat feature properties — every Water Data / NGWMN collection — build through a plain `DataFrame` constructor instead of `pd.json_normalize` (~2.5x), and all-point geometry pages build their `GeoDataFrame` through one vectorized `shapely.points` call instead of the per-feature Python walk in `GeoDataFrame.from_features` (~4.5x). Results are identical — a page with nested properties, or any non-point or malformed geometry, takes the previous path — and this CPU runs inside the fan-out's event loop, so trimming it also tightens chunk overlap (measured: a 93k-row, 12-chunk `get_daily` re-pull fell from 1.33 s to 0.95 s median wall; spatial frame construction alone is 4.2–4.6x faster at 10k–500k rows). Aggregated responses also no longer retain a page body: the response behind a getter's metadata and each per-chunk aggregate held for resume each carried one full JSON page's bytes, which on a many-chunk fan-out kept roughly the whole download resident until the call finished (~19% lower peak Python-heap in a 12-chunk replay, growing with chunk count and page size). **Behavior change:** the aggregated `httpx.Response` a completed call or `FanOutInterrupted.partial_response` carries now has an empty body — it was previously one arbitrary page's bytes, not the query's data; status, headers, URL, and elapsed are unchanged, and per-page responses are untouched. +**08/22/2026:** The OGC feature-frame builders take vectorized fast paths, halving the client-side CPU of a chunked call. Flat feature properties — every Water Data / NGWMN collection — build through a plain `DataFrame` constructor instead of `pd.json_normalize` (~2.5x), and all-point geometry pages build their `GeoDataFrame` through one vectorized `shapely.points` call instead of the per-feature Python walk in `GeoDataFrame.from_features` (~4.5x). Results are identical — a page with nested properties, or any non-point or malformed geometry, takes the previous path — and this CPU runs inside the fan-out's event loop, so trimming it also tightens chunk overlap (measured: a 93k-row, 12-chunk `get_daily` re-pull fell from 1.33 s to 0.95 s median wall; spatial frame construction alone is 4.2–4.6x faster at 10k–500k rows). Aggregated responses also no longer retain a page body: the response behind a getter's metadata and each per-chunk aggregate held for resume each carried one full JSON page's bytes, which on a many-chunk fan-out kept roughly the whole download resident until the call finished (~19% lower peak Python-heap in a 12-chunk replay, growing with chunk count and page size). **Behavior change:** the aggregated `httpx.Response` a completed call or `FanOutInterrupted.partial_response` carries now has an empty body — it was previously one arbitrary page's bytes, not the query's data; status, headers, URL, and elapsed are unchanged, and per-page responses are untouched. **Deprecation:** `utils.format_datetime` — orphaned since the qw services it shaped responses for were retired — now emits a `DeprecationWarning`; it will be removed on or after 2027-08-22. Combine the columns with `pandas.to_datetime` directly. **08/13/2026:** Warning categories now say what they mean. Two advisories about *upstream data* were emitted as `DeprecationWarning` and as an uncategorized `warnings.warn` respectively; both are now `DataCurrencyWarning` (a `UserWarning` subclass, exported as `dataretrieval.DataCurrencyWarning`). **Behavior change:** WQP's legacy-WQX notice moves from `DeprecationWarning` to `DataCurrencyWarning`. Because `legacy=True` is the default on every WQP getter and the notice is unconditional, a downstream project running `-W error::DeprecationWarning` previously could not call any WQP getter with default arguments; it now can. The flip side is visibility — `DeprecationWarning` is silent by default outside `__main__`, so this notice will now print to stderr for library and notebook callers who never saw it. Silence it with `warnings.filterwarnings("ignore", category=dataretrieval.DataCurrencyWarning)`, or set `legacy=False` where a WQX3.0 profile exists. The NWIS qw-endpoint retirement notice gains the same category (it previously had none, arriving as a bare `UserWarning` that deprecation filters ignored). `DeprecationWarning` now means only that a name in this package is going away, always with a replacement and, where published, a removal date; those horizons are declared once in `dataretrieval._deprecation.REMOVALS` rather than spelled at each call site. diff --git a/dataretrieval/_deprecation.py b/dataretrieval/_deprecation.py index d8a2917cc..2e63fb784 100644 --- a/dataretrieval/_deprecation.py +++ b/dataretrieval/_deprecation.py @@ -27,6 +27,7 @@ "nwis": "2027-05-06", "waterdata.get_cql(service=)": "2027-08-09", "wateruse": "2027-08-11", + "utils.format_datetime": "2027-08-22", } diff --git a/dataretrieval/utils.py b/dataretrieval/utils.py index 924f88816..c2ea393e4 100644 --- a/dataretrieval/utils.py +++ b/dataretrieval/utils.py @@ -17,6 +17,7 @@ import pandas as pd +import dataretrieval._deprecation as _deprecation import dataretrieval._querying as _querying import dataretrieval.transport.http as _transport_http from dataretrieval._ambient import Ambient # noqa: F401 - compatibility re-export @@ -59,8 +60,16 @@ def format_datetime( df: ``pandas.DataFrame`` The data frame with a formatted 'datetime' column. + .. deprecated:: + The qw services this shaped responses for are retired; nothing in the + package calls it. Combine the columns with :func:`pandas.to_datetime` + directly. """ - # create a datetime index from the columns in qwdata response + _deprecation.warn_deprecated( + "`utils.format_datetime`", + replacement="a direct `pandas.to_datetime` over the combined columns", + removal=_deprecation.REMOVALS["utils.format_datetime"], + ) df[tz_field] = df[tz_field].map(tz) df["datetime"] = pd.to_datetime( diff --git a/tests/deprecation_test.py b/tests/deprecation_test.py index 94a0b2b53..80a3d6caa 100644 --- a/tests/deprecation_test.py +++ b/tests/deprecation_test.py @@ -70,3 +70,14 @@ def test_detail_is_appended_not_interpolated(): assert message.endswith("Because reasons. And more.") assert "use 'b' instead." in message assert "in a future release" not in message + + +def test_format_datetime_is_deprecated_with_horizon(): + """The orphaned qw-era helper warns with the published removal date.""" + import pandas as pd + + from dataretrieval import utils + + df = pd.DataFrame({"d": ["2020-01-01"], "t": ["12:00"], "z": ["EST"]}) + with pytest.warns(DeprecationWarning, match=REMOVALS["utils.format_datetime"]): + utils.format_datetime(df, "d", "t", "z") From 6f35f6b5bacfcd942b761076c48e0f6e7543ba99 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 23 Aug 2026 12:36:02 -0500 Subject: [PATCH 7/8] refactor: build points through geopandas, gate the nested-value scan Second cleanup pass over the branch (reuse / simplification / efficiency / altitude), output-identical against the recorded API fixtures: - _point_geometries builds through gpd.points_from_xy over two flat x/y lists, the idiom nwis already uses, retiring the package's only direct shapely import (shapely is geopandas' own dependency, undeclared here). The pair form it replaces was ~1.6x slower; inlining the per-feature helper also retires the _NON_POINT sentinel and its three-way contract. - _properties_frame gates its Python dict scan behind infer_dtype, so only a genuinely mixed object column is walked. Under pandas 2 -- supported, and where every string column is object dtype -- that is 2.3x on the helper; spatial page build is now 4.9x the pre-branch path. - format_datetime documents its deprecation in prose: a bare `.. deprecated::` has a required version argument, so Sphinx was eating the first word of the body as the version on the published API page. - combining's docstring names its second responsibility (adjusting fetched responses) and why it sits below transport. - The deprecation test moves to utils_test, where per-surface deprecation tests live; deprecation_test keeps the mechanism claims. - Malformed-coordinate cases are parametrized, and a ragged pair now covers the array-build refusal that no test reached. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UKweWmHg8cu1WuJ17kiHdh --- dataretrieval/combining.py | 7 +++ dataretrieval/ogc/shaping.py | 86 ++++++++++++++++++----------------- dataretrieval/utils.py | 8 ++-- tests/deprecation_test.py | 11 ----- tests/utils_test.py | 9 ++++ tests/waterdata_utils_test.py | 40 +++++++++++++--- 6 files changed, 98 insertions(+), 63 deletions(-) diff --git a/dataretrieval/combining.py b/dataretrieval/combining.py index 07c9b2b86..94dd02f09 100644 --- a/dataretrieval/combining.py +++ b/dataretrieval/combining.py @@ -11,6 +11,13 @@ A top-level leaf rather than part of :mod:`dataretrieval.transport`: these are pandas transforms over already-fetched results, with no HTTP or event-loop concern, consumed by chunk planning and service fan-out as well as by pagination. + +The module's second half adjusts already-fetched :class:`httpx.Response` +objects in place of issuing any -- url, elapsed, headers, body release. Those +live here rather than in transport because both the merge and the page walk +need one spelling of each, and transport sits *above* this leaf in the layer +stack (see the import-linter contracts), so a shared helper can only live +below it. """ from __future__ import annotations diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index 77882e6f3..6a3e2e906 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -16,13 +16,13 @@ import httpx import pandas as pd +from pandas.api.types import infer_dtype from dataretrieval._response_metadata import BaseMetadata from dataretrieval.ogc.policy import DEFAULT_DIALECT, OgcDialect try: import geopandas as gpd - import shapely GEOPANDAS = True except ImportError: @@ -36,6 +36,10 @@ # (EPSG:4269). _CRS = "EPSG:4326" +# What ``infer_dtype`` reports for an object column that holds a dict among +# its values; a homogeneous column reports its own type and cannot hide one. +_MIXED_DTYPES = frozenset({"mixed", "mixed-integer"}) + # Whether geopandas is present is a static, environment-level fact, so warn # once here at import time rather than per query/chunk. if not GEOPANDAS: @@ -106,17 +110,28 @@ def _properties_frame(features: list[dict[str, Any]]) -> pd.DataFrame: Flat properties — every Water Data / NGWMN collection — build with the plain ``DataFrame`` constructor; a nested value in *any* feature routes the whole page through ``json_normalize`` so no row keeps a raw dict. - Dicts can only land in object-dtype columns, so scanning those after the - cheap build catches every one. + Dicts can only land in object-dtype columns, and only in one whose + values ``infer_dtype`` calls mixed, so the Python scan runs on those + alone — under pandas 2, where every string column is object dtype, that + is the difference between scanning one column and scanning all of them. """ properties = [feature.get("properties") or {} for feature in features] frame = pd.DataFrame(properties) - for _, column in frame.items(): - if column.dtype == object and any(isinstance(v, dict) for v in column): - return pd.json_normalize(properties, sep="_") + if any(_holds_nested_value(column) for _, column in frame.items()): + return pd.json_normalize(properties, sep="_") return frame +def _holds_nested_value(column: pd.Series) -> bool: + """Whether a built column carries a raw dict that needs flattening.""" + if column.dtype != object: + return False + values = column.to_numpy() + if infer_dtype(values, skipna=True) not in _MIXED_DTYPES: + return False + return any(isinstance(value, dict) for value in values) + + def _plain_feature_frame( features: list[dict[str, Any]], *, include_geometry: bool ) -> pd.DataFrame: @@ -128,57 +143,44 @@ def _plain_feature_frame( return df -#: Sentinel for a geometry the vectorized point build can't represent. -_NON_POINT = object() - - -def _point_xy(feature: dict[str, Any]) -> Any: - """One feature's 2-D point coordinates: ``None`` when it has no - geometry, :data:`_NON_POINT` for anything else the fast path can't build. - """ - geometry = feature.get("geometry") - if geometry is None: - return None - xy = geometry.get("coordinates") - if ( - geometry.get("type") == "Point" - and isinstance(xy, (list, tuple)) - and len(xy) == 2 - ): - return xy - return _NON_POINT - - def _point_geometries(features: list[dict[str, Any]]) -> Any: - """Build the geometry array for all-2D-point features in one vectorized - ``shapely.points`` call. + """Build the geometry array for an all-2D-point page in one vectorized + :func:`geopandas.points_from_xy` call. Returns ``None`` when any feature carries a non-point (or malformed) geometry, so the caller falls back to ``GeoDataFrame.from_features``. A feature with no geometry stays ``None`` in the result, matching the - fallback. + fallback. Two flat x/y lists rather than coordinate pairs: the paired + form is ~1.6x slower. """ nan = float("nan") - coords: list[Any] = [] + xs: list[Any] = [] + ys: list[Any] = [] missing: list[int] = [] for index, feature in enumerate(features): - xy = _point_xy(feature) - if xy is _NON_POINT: - return None - if xy is None: + geometry = feature.get("geometry") or {} + xy: Any = geometry.get("coordinates") + if not geometry: missing.append(index) xy = (nan, nan) - coords.append(xy) + elif geometry.get("type") != "Point" or not _is_pair(xy): + return None + xs.append(xy[0]) + ys.append(xy[1]) try: - points = shapely.points(coords) + points = gpd.points_from_xy(xs, ys) except (TypeError, ValueError): return None - if missing: - points[missing] = None + points[missing] = None return points -def _point_feature_frame(features: list[dict[str, Any]]) -> Any: +def _is_pair(value: Any) -> bool: + """Whether ``value`` is a two-element coordinate sequence.""" + return isinstance(value, (list, tuple)) and len(value) == 2 + + +def _point_feature_frame(features: list[dict[str, Any]]) -> pd.DataFrame | None: """Fast-path GeoDataFrame for an all-2D-point page, or ``None`` to fall back to :func:`_geo_feature_frame`. @@ -200,8 +202,10 @@ def _spatial_feature_frame(features: list[dict[str, Any]]) -> pd.DataFrame: df = _point_feature_frame(features) if df is None: df = _geo_feature_frame(features) - # A properties ``id`` column is overwritten by the feature-level id. df["id"] = [f.get("id") for f in features] + # Pin both names: the fast path appends ``geometry`` last and + # ``from_features`` emits it first, so only naming them keeps the two + # paths' column order identical across a chunked concat. ordered = ["id", "geometry"] return df[ordered + [col for col in df.columns if col not in ordered]] diff --git a/dataretrieval/utils.py b/dataretrieval/utils.py index c2ea393e4..ac22c489a 100644 --- a/dataretrieval/utils.py +++ b/dataretrieval/utils.py @@ -60,10 +60,10 @@ def format_datetime( df: ``pandas.DataFrame`` The data frame with a formatted 'datetime' column. - .. deprecated:: - The qw services this shaped responses for are retired; nothing in the - package calls it. Combine the columns with :func:`pandas.to_datetime` - directly. + Deprecated: the qw services this shaped responses for are retired and + nothing in the package calls it. Combine the columns with + :func:`pandas.to_datetime` directly. See + :data:`dataretrieval._deprecation.REMOVALS` for the removal horizon. """ _deprecation.warn_deprecated( "`utils.format_datetime`", diff --git a/tests/deprecation_test.py b/tests/deprecation_test.py index 80a3d6caa..94a0b2b53 100644 --- a/tests/deprecation_test.py +++ b/tests/deprecation_test.py @@ -70,14 +70,3 @@ def test_detail_is_appended_not_interpolated(): assert message.endswith("Because reasons. And more.") assert "use 'b' instead." in message assert "in a future release" not in message - - -def test_format_datetime_is_deprecated_with_horizon(): - """The orphaned qw-era helper warns with the published removal date.""" - import pandas as pd - - from dataretrieval import utils - - df = pd.DataFrame({"d": ["2020-01-01"], "t": ["12:00"], "z": ["EST"]}) - with pytest.warns(DeprecationWarning, match=REMOVALS["utils.format_datetime"]): - utils.format_datetime(df, "d", "t", "z") diff --git a/tests/utils_test.py b/tests/utils_test.py index b461b019e..50eb62b40 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -478,3 +478,12 @@ def test_retrying_get_maps_invalid_url(monkeypatch): with pytest.raises(exceptions.URLTooLong): _querying._get_with_retry("https://example.invalid") + + +def test_format_datetime_is_deprecated_with_horizon(): + """The orphaned qw-era helper warns with the published removal date.""" + from dataretrieval._deprecation import REMOVALS + + df = pd.DataFrame({"d": ["2020-01-01"], "t": ["12:00"], "z": ["EST"]}) + with pytest.warns(DeprecationWarning, match=REMOVALS["utils.format_datetime"]): + utils.format_datetime(df, "d", "t", "z") diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 20bf69817..e2839757d 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -1400,23 +1400,49 @@ def test_spatial_non_point_geometry_uses_from_features(): }, } ] - assert _shaping_module._point_geometries(features) is None df = _shaping_module._spatial_feature_frame(features) assert df.geometry.iloc[1].geom_type == "Polygon" @pytest.mark.skipif(not _shaping_module.GEOPANDAS, reason="requires geopandas") -def test_point_geometries_rejects_malformed_coordinates(): - """3-D or non-pair coordinates disable the fast path rather than - building a wrong geometry.""" - threed = [ +@pytest.mark.parametrize( + "coordinates", + [ + pytest.param([1.0, 2.0, 3.0], id="3d"), + pytest.param([1.0], id="single"), + pytest.param({"x": 1.0, "y": 2.0}, id="mapping"), + ], +) +def test_point_geometries_rejects_malformed_coordinates(coordinates): + """Coordinates that are not a 2-element sequence disable the fast path + rather than building a wrong geometry.""" + features = [ { "id": "f", "properties": {}, - "geometry": {"type": "Point", "coordinates": [1.0, 2.0, 3.0]}, + "geometry": {"type": "Point", "coordinates": coordinates}, } ] - assert _shaping_module._point_geometries(threed) is None + assert _shaping_module._point_geometries(features) is None + + +@pytest.mark.skipif(not _shaping_module.GEOPANDAS, reason="requires geopandas") +def test_ragged_coordinate_pairs_fall_back_to_from_features(): + """A pair whose members aren't scalars reaches the array build and is + refused there, so the page still shapes through ``from_features``.""" + features = [ + { + "id": "f-1", + "properties": {"value": "1"}, + "geometry": {"type": "Point", "coordinates": [[1.0, 2.0], [3.0, 4.0]]}, + }, + { + "id": "f-2", + "properties": {"value": "2"}, + "geometry": {"type": "Point", "coordinates": [5.0, 6.0]}, + }, + ] + assert _shaping_module._point_geometries(features) is None def test_properties_frame_flat_matches_normalize(): From 453927156d42e8b025f24011c5ecfe9f60aabba5 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sun, 23 Aug 2026 12:38:01 -0500 Subject: [PATCH 8/8] docs(news): refresh speedups after the second cleanup pass Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UKweWmHg8cu1WuJ17kiHdh --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 4a0678a5c..428ed9b8b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -**08/22/2026:** The OGC feature-frame builders take vectorized fast paths, halving the client-side CPU of a chunked call. Flat feature properties — every Water Data / NGWMN collection — build through a plain `DataFrame` constructor instead of `pd.json_normalize` (~2.5x), and all-point geometry pages build their `GeoDataFrame` through one vectorized `shapely.points` call instead of the per-feature Python walk in `GeoDataFrame.from_features` (~4.5x). Results are identical — a page with nested properties, or any non-point or malformed geometry, takes the previous path — and this CPU runs inside the fan-out's event loop, so trimming it also tightens chunk overlap (measured: a 93k-row, 12-chunk `get_daily` re-pull fell from 1.33 s to 0.95 s median wall; spatial frame construction alone is 4.2–4.6x faster at 10k–500k rows). Aggregated responses also no longer retain a page body: the response behind a getter's metadata and each per-chunk aggregate held for resume each carried one full JSON page's bytes, which on a many-chunk fan-out kept roughly the whole download resident until the call finished (~19% lower peak Python-heap in a 12-chunk replay, growing with chunk count and page size). **Behavior change:** the aggregated `httpx.Response` a completed call or `FanOutInterrupted.partial_response` carries now has an empty body — it was previously one arbitrary page's bytes, not the query's data; status, headers, URL, and elapsed are unchanged, and per-page responses are untouched. **Deprecation:** `utils.format_datetime` — orphaned since the qw services it shaped responses for were retired — now emits a `DeprecationWarning`; it will be removed on or after 2027-08-22. Combine the columns with `pandas.to_datetime` directly. +**08/22/2026:** The OGC feature-frame builders take vectorized fast paths, halving the client-side CPU of a chunked call. Flat feature properties — every Water Data / NGWMN collection — build through a plain `DataFrame` constructor instead of `pd.json_normalize` (~2.6x), and all-point geometry pages build their `GeoDataFrame` through one vectorized `geopandas.points_from_xy` call instead of the per-feature Python walk in `GeoDataFrame.from_features` (~4.8x). Results are identical — a page with nested properties, or any non-point or malformed geometry, takes the previous path — and this CPU runs inside the fan-out's event loop, so trimming it also tightens chunk overlap (measured: a 93k-row, 12-chunk `get_daily` re-pull fell from 1.33 s to 0.95 s median wall; spatial frame construction alone is 4.7–4.9x faster at 10k–500k rows). The gain is larger under pandas 2, where every string column is object dtype: the nested-value check that decides the fast path is gated on `infer_dtype`, so it walks only a genuinely mixed column rather than all of them. Aggregated responses also no longer retain a page body: the response behind a getter's metadata and each per-chunk aggregate held for resume each carried one full JSON page's bytes, which on a many-chunk fan-out kept roughly the whole download resident until the call finished (~19% lower peak Python-heap in a 12-chunk replay, growing with chunk count and page size). **Behavior change:** the aggregated `httpx.Response` a completed call or `FanOutInterrupted.partial_response` carries now has an empty body — it was previously one arbitrary page's bytes, not the query's data; status, headers, URL, and elapsed are unchanged, and per-page responses are untouched. **Deprecation:** `utils.format_datetime` — orphaned since the qw services it shaped responses for were retired — now emits a `DeprecationWarning`; it will be removed on or after 2027-08-22. Combine the columns with `pandas.to_datetime` directly. **08/13/2026:** Warning categories now say what they mean. Two advisories about *upstream data* were emitted as `DeprecationWarning` and as an uncategorized `warnings.warn` respectively; both are now `DataCurrencyWarning` (a `UserWarning` subclass, exported as `dataretrieval.DataCurrencyWarning`). **Behavior change:** WQP's legacy-WQX notice moves from `DeprecationWarning` to `DataCurrencyWarning`. Because `legacy=True` is the default on every WQP getter and the notice is unconditional, a downstream project running `-W error::DeprecationWarning` previously could not call any WQP getter with default arguments; it now can. The flip side is visibility — `DeprecationWarning` is silent by default outside `__main__`, so this notice will now print to stderr for library and notebook callers who never saw it. Silence it with `warnings.filterwarnings("ignore", category=dataretrieval.DataCurrencyWarning)`, or set `legacy=False` where a WQX3.0 profile exists. The NWIS qw-endpoint retirement notice gains the same category (it previously had none, arriving as a bare `UserWarning` that deprecation filters ignored). `DeprecationWarning` now means only that a name in this package is going away, always with a replacement and, where published, a removal date; those horizons are declared once in `dataretrieval._deprecation.REMOVALS` rather than spelled at each call site.