From 5d60db61016bf4a78d7eb3a2c08d6186e2472f47 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Mon, 14 Sep 2026 11:27:45 +0200 Subject: [PATCH 01/29] fix: complete imagecodecs codec package hints (#4351) * fix: complete imagecodecs codec package hints Assisted-by: Codex:GPT-6 * docs: associate codec changelog with PR 338 Assisted-by: Codex:GPT-6 * docs: associate codec changelog with upstream PR 4351 Assisted-by: Codex:GPT-6 --- changes/4351.bugfix.md | 1 + src/zarr/registry.py | 107 +++++++++++++++++++++++++++++++++-------- tests/test_registry.py | 46 +++++++++++++----- 3 files changed, 124 insertions(+), 30 deletions(-) create mode 100644 changes/4351.bugfix.md diff --git a/changes/4351.bugfix.md b/changes/4351.bugfix.md new file mode 100644 index 0000000000..515a244b46 --- /dev/null +++ b/changes/4351.bugfix.md @@ -0,0 +1 @@ +Missing Zarr format 3 imagecodecs now name `imagecodecs-zarr` as a known provider, alongside `virtual-tiff` where both packages register the codec. diff --git a/src/zarr/registry.py b/src/zarr/registry.py index a7537e5023..307c323153 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -53,31 +53,100 @@ # # The two Zarr formats resolve codecs through different registries, so they get different # tables: a name can mean one thing as a Zarr format 3 codec name and another as a Zarr -# format 2 codec id. `imagecodecs_*` is exactly that -- `virtual-tiff` declares 15 of those -# names under `zarr.codecs`, while `imagecodecs-numcodecs` declares all 81 under -# `numcodecs.codecs`, so the format 2 side can use a prefix and the format 3 side cannot. +# format 2 codec id. For example, `imagecodecs_*` names are registered by +# `imagecodecs-zarr` and `virtual-tiff` under `zarr.codecs`, and by +# `imagecodecs-numcodecs` under `numcodecs.codecs`. # Zarr format 3 codec names (entry point group "zarr.codecs"). _CODEC_PACKAGES: dict[str, tuple[str, ...]] = { "gribberish": ("gribberish",), - # `virtual-tiff` declares these 15 `imagecodecs_*` names, out of the 81 that exist as - # numcodecs ids. They are listed exactly rather than by prefix so that the other 66 get no - # hint instead of a hint pointing at a package that does not provide them. - "imagecodecs_deflate": ("virtual-tiff",), - "imagecodecs_delta": ("virtual-tiff",), - "imagecodecs_floatpred": ("virtual-tiff",), + # Verified against imagecodecs-zarr 2026.8.16's published zarr.codecs entry points: + # https://pypi.org/project/imagecodecs-zarr/2026.8.16/ + # virtual-tiff also provides 13 of these names, plus jpeg8 and jetraw. + # Keep exact names: neither package provides every possible imagecodecs_* name. + "imagecodecs_aec": ("imagecodecs-zarr",), + "imagecodecs_apng": ("imagecodecs-zarr",), + "imagecodecs_avif": ("imagecodecs-zarr",), + "imagecodecs_b2nd": ("imagecodecs-zarr",), + "imagecodecs_bfloat16": ("imagecodecs-zarr",), + "imagecodecs_bitorder": ("imagecodecs-zarr",), + "imagecodecs_bitshuffle": ("imagecodecs-zarr",), + "imagecodecs_blosc": ("imagecodecs-zarr",), + "imagecodecs_blosc2": ("imagecodecs-zarr",), + "imagecodecs_bmp": ("imagecodecs-zarr",), + "imagecodecs_brotli": ("imagecodecs-zarr",), + "imagecodecs_byteshuffle": ("imagecodecs-zarr",), + "imagecodecs_bz2": ("imagecodecs-zarr",), + "imagecodecs_ccittfax3": ("imagecodecs-zarr",), + "imagecodecs_ccittfax4": ("imagecodecs-zarr",), + "imagecodecs_ccittrle": ("imagecodecs-zarr",), + "imagecodecs_checksum": ("imagecodecs-zarr",), + "imagecodecs_chunked": ("imagecodecs-zarr",), + "imagecodecs_cms": ("imagecodecs-zarr",), + "imagecodecs_dds": ("imagecodecs-zarr",), + "imagecodecs_deflate": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_delta": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_dicomrle": ("imagecodecs-zarr",), + "imagecodecs_eer": ("imagecodecs-zarr",), + "imagecodecs_exr": ("imagecodecs-zarr",), + "imagecodecs_float24": ("imagecodecs-zarr",), + "imagecodecs_floatpred": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_gif": ("imagecodecs-zarr",), + "imagecodecs_hcomp": ("imagecodecs-zarr",), + "imagecodecs_heif": ("imagecodecs-zarr",), + "imagecodecs_htj2k": ("imagecodecs-zarr",), + "imagecodecs_isal": ("imagecodecs-zarr",), "imagecodecs_jetraw": ("virtual-tiff",), - "imagecodecs_jpeg": ("virtual-tiff",), - "imagecodecs_jpeg2k": ("virtual-tiff",), + "imagecodecs_jpeg": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_jpeg2k": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_jpeg8": ("virtual-tiff",), - "imagecodecs_jpegxl": ("virtual-tiff",), - "imagecodecs_jpegxr": ("virtual-tiff",), - "imagecodecs_lerc": ("virtual-tiff",), - "imagecodecs_lzw": ("virtual-tiff",), - "imagecodecs_packbits": ("virtual-tiff",), - "imagecodecs_png": ("virtual-tiff",), - "imagecodecs_webp": ("virtual-tiff",), - "imagecodecs_zstd": ("virtual-tiff",), + "imagecodecs_jpegls": ("imagecodecs-zarr",), + "imagecodecs_jpegxl": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_jpegxr": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_jpegxs": ("imagecodecs-zarr",), + "imagecodecs_lerc": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_ljpeg": ("imagecodecs-zarr",), + "imagecodecs_lz4": ("imagecodecs-zarr",), + "imagecodecs_lz4f": ("imagecodecs-zarr",), + "imagecodecs_lz4h5": ("imagecodecs-zarr",), + "imagecodecs_lzf": ("imagecodecs-zarr",), + "imagecodecs_lzfse": ("imagecodecs-zarr",), + "imagecodecs_lzham": ("imagecodecs-zarr",), + "imagecodecs_lzma": ("imagecodecs-zarr",), + "imagecodecs_lzo": ("imagecodecs-zarr",), + "imagecodecs_lzw": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_meshopt": ("imagecodecs-zarr",), + "imagecodecs_openzl": ("imagecodecs-zarr",), + "imagecodecs_packbits": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_packints": ("imagecodecs-zarr",), + "imagecodecs_pcodec": ("imagecodecs-zarr",), + "imagecodecs_pcx": ("imagecodecs-zarr",), + "imagecodecs_pglz": ("imagecodecs-zarr",), + "imagecodecs_pixarlog": ("imagecodecs-zarr",), + "imagecodecs_plio": ("imagecodecs-zarr",), + "imagecodecs_png": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_qoi": ("imagecodecs-zarr",), + "imagecodecs_quantize": ("imagecodecs-zarr",), + "imagecodecs_rcomp": ("imagecodecs-zarr",), + "imagecodecs_rgbe": ("imagecodecs-zarr",), + "imagecodecs_snappy": ("imagecodecs-zarr",), + "imagecodecs_sperr": ("imagecodecs-zarr",), + "imagecodecs_spng": ("imagecodecs-zarr",), + "imagecodecs_sz3": ("imagecodecs-zarr",), + "imagecodecs_szip": ("imagecodecs-zarr",), + "imagecodecs_tga": ("imagecodecs-zarr",), + "imagecodecs_tiff": ("imagecodecs-zarr",), + "imagecodecs_ultrahdr": ("imagecodecs-zarr",), + "imagecodecs_wavpack": ("imagecodecs-zarr",), + "imagecodecs_webp": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_wic": ("imagecodecs-zarr",), + "imagecodecs_xor": ("imagecodecs-zarr",), + "imagecodecs_zfp": ("imagecodecs-zarr",), + "imagecodecs_zlib": ("imagecodecs-zarr",), + "imagecodecs_zlibng": ("imagecodecs-zarr",), + "imagecodecs_zopfli": ("imagecodecs-zarr",), + "imagecodecs_zstd": ("imagecodecs-zarr", "virtual-tiff"), + "imagecodecs_zstd1": ("imagecodecs-zarr",), "n5_default": ("zarr-n5",), } diff --git a/tests/test_registry.py b/tests/test_registry.py index cae7a89d98..4344f5d72d 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -51,7 +51,7 @@ def unregistered_v2_codec(monkeypatch: pytest.MonkeyPatch) -> str: [ ("n5_default", ("zarr-n5",)), ("gribberish", ("gribberish",)), - ("imagecodecs_jpeg2k", ("virtual-tiff",)), + ("imagecodecs_jpeg2k", ("imagecodecs-zarr", "virtual-tiff")), ("omfiles.pfor", ("omfiles",)), ("any-numcodecs.array-array", ("zarr-any-numcodecs",)), ("totally-made-up", ()), @@ -81,7 +81,10 @@ def test_packages_for_numcodec_v2(name: str, expected: tuple[str, ...]) -> None: def test_packages_for_codec_is_format_specific() -> None: """The same name can mean different packages in each format's registry.""" - assert _packages_for_codec("imagecodecs_jpeg2k", zarr_format=3) == ("virtual-tiff",) + assert _packages_for_codec("imagecodecs_jpeg2k", zarr_format=3) == ( + "imagecodecs-zarr", + "virtual-tiff", + ) assert _packages_for_codec("imagecodecs_jpeg2k", zarr_format=2) == ("imagecodecs-numcodecs",) # `crc32c` is a codec zarr implements in format 3, so only format 2 gets a hint for it. assert _packages_for_codec("crc32c", zarr_format=3) == () @@ -270,15 +273,36 @@ def test_get_numcodec_non_mapping_input_still_raises_value_error(data: object) - get_numcodec(data) # type: ignore[arg-type] -def test_imagecodecs_prefix_does_not_over_match_in_zarr_format_3() -> None: - """virtual-tiff provides 15 of the 81 `imagecodecs_*` names; the rest must get no hint. - - Recommending virtual-tiff for a name it does not provide is worse than saying nothing. - """ - assert _packages_for_codec("imagecodecs_jpeg2k", zarr_format=3) == ("virtual-tiff",) - for name in ("imagecodecs_jpegls", "imagecodecs_avif", "imagecodecs_blosc"): - assert _packages_for_codec(name, zarr_format=3) == () - assert _packages_for_codec(name, zarr_format=2) == ("imagecodecs-numcodecs",) +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("imagecodecs_jpeg2k", ("imagecodecs-zarr", "virtual-tiff")), + ("imagecodecs_jpeg8", ("virtual-tiff",)), + ("imagecodecs_jetraw", ("virtual-tiff",)), + ("imagecodecs_jpegls", ("imagecodecs-zarr",)), + ("imagecodecs_avif", ("imagecodecs-zarr",)), + ("imagecodecs_blosc", ("imagecodecs-zarr",)), + ("imagecodecs_lzma", ("imagecodecs-zarr",)), + ("imagecodecs_wavpack", ("imagecodecs-zarr",)), + ("imagecodecs_zstd1", ("imagecodecs-zarr",)), + ("imagecodecs_not_a_codec", ()), + ], +) +def test_missing_imagecodec_reports_packages( + name: str, expected: tuple[str, ...], monkeypatch: pytest.MonkeyPatch +) -> None: + """Missing imagecodecs name their providers without guessing from the prefix.""" + monkeypatch.setitem(zarr.registry._codec_registries, name, zarr.registry.Registry()) + message = ( + f"An implementation for codec {name!r} is not available. Register one explicitly " + f"using the codec registry (see {_ZARR_CODEC_DOCS_URL}), or install a Python package " + "that registers a codec implementation with zarr." + ) + if expected: + message += f" Known packages supporting this codec: {', '.join(expected)}." + with pytest.raises(UnknownCodecError) as excinfo: + get_codec_class(name) + assert str(excinfo.value) == message def test_resolve_codec_reports_missing_codec() -> None: From 8ff7cb189f2c041969c391546705db5096aab559 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Mon, 14 Sep 2026 11:47:16 +0200 Subject: [PATCH 02/29] fix(indexing): audit factual contracts and close validation gaps (#4345) * docs(indexing): ground design and integration claims in current behavior Assisted-by: Codex:GPT-6 * docs(indexing): correct reader lazy-array and cache contracts Assisted-by: Codex:GPT-6 * fix(indexing): validate wire boundaries and clarify format contracts Assisted-by: Codex:GPT-6 * fix(indexing): validate selector bounds and shared dependencies Correct mathematical API documentation to match supported coordinate, grid, and chunk projection contracts. Assisted-by: Codex:GPT-6 * docs(indexing): reconcile reader contracts and record audit fixes Assisted-by: Codex:GPT-6 * docs(indexing): reconcile audit with current partition implementation Retain the existing unsigned selector fix and update the unsupported mixed-dependency error assertion for general intersection routing. Assisted-by: Codex:GPT-6 * docs(indexing): clarify planning coverage and benchmark measurement boundaries Assisted-by: Codex:GPT-6 * fix(indexing): group signed chunk coordinates without collisions Use lexicographic tuple grouping when chunk indices contain negative values. Cover shared one-axis and two-axis array dependencies, repeated points, and extreme signed coordinates. Assisted-by: Codex:GPT-6 * docs(indexing): state remaining planner limits precisely Assisted-by: Codex:GPT-6 * docs(indexing): number audit changelog entries for PR 4345 Assisted-by: Codex:GPT-6 * docs(indexing): describe current contracts in docstrings Remove implementation history and unsupported historical claims from source and test docstrings. Distinguish immutable coordinate mappings from mutable source values. Assisted-by: Codex:GPT-6 --- packages/zarr-indexing/CONTRIBUTING.md | 10 +- packages/zarr-indexing/README.md | 20 +- packages/zarr-indexing/benchmarks/README.md | 13 + .../benchmarks/chunk_planning.py | 7 + packages/zarr-indexing/changes/4345.bugfix.md | 1 + packages/zarr-indexing/changes/4345.doc.md | 3 + packages/zarr-indexing/docs/api/grid.md | 5 +- packages/zarr-indexing/docs/api/index.md | 7 +- packages/zarr-indexing/docs/api/lazy_array.md | 16 +- packages/zarr-indexing/docs/api/reader.md | 5 +- packages/zarr-indexing/docs/design-notes.md | 129 +++++----- packages/zarr-indexing/docs/guide/index.md | 16 +- .../zarr-indexing/docs/guide/integrations.md | 59 +++-- packages/zarr-indexing/docs/guide/patterns.md | 9 +- .../docs/guide/selection-flow.md | 5 +- packages/zarr-indexing/docs/index.md | 8 +- packages/zarr-indexing/docs/ndsel.md | 63 +++-- packages/zarr-indexing/docs/release-notes.md | 3 +- .../docs/snippets/grid_partition.py | 6 +- .../docs/snippets/output_maps.py | 8 +- .../examples/lazy_indexing_dask/README.md | 25 +- .../lazy_indexing_dask/lazy_indexing_dask.py | 26 +- .../examples/lazy_indexing_numpy/README.md | 5 +- .../lazy_indexing_numpy.py | 5 +- .../system_memory_chunk_cache/README.md | 8 +- .../system_memory_chunk_cache.py | 5 +- packages/zarr-indexing/justfile | 12 +- packages/zarr-indexing/mkdocs.yml | 2 +- packages/zarr-indexing/pyproject.toml | 39 ++- .../src/zarr_indexing/_composition.py | 8 +- .../zarr-indexing/src/zarr_indexing/_wire.py | 35 ++- .../src/zarr_indexing/boundary.py | 12 +- .../src/zarr_indexing/chunk_resolution.py | 24 +- .../zarr-indexing/src/zarr_indexing/domain.py | 25 +- .../zarr-indexing/src/zarr_indexing/errors.py | 4 +- .../zarr-indexing/src/zarr_indexing/grid.py | 17 +- .../zarr-indexing/src/zarr_indexing/json.py | 22 +- .../src/zarr_indexing/lazy_array.py | 234 ++++++++---------- .../src/zarr_indexing/messages.py | 43 ++-- .../src/zarr_indexing/output_map.py | 81 +++--- .../zarr-indexing/src/zarr_indexing/reader.py | 26 +- .../src/zarr_indexing/testing/stateful.py | 26 +- .../src/zarr_indexing/testing/strategies.py | 34 +-- .../src/zarr_indexing/transform.py | 62 +++-- packages/zarr-indexing/tests/test_boundary.py | 47 ++++ .../tests/test_chunk_resolution.py | 48 +++- .../zarr-indexing/tests/test_composition.py | 6 +- .../zarr-indexing/tests/test_doc_examples.py | 4 +- packages/zarr-indexing/tests/test_json.py | 20 +- .../zarr-indexing/tests/test_lazy_array.py | 72 +----- .../tests/test_lazy_array_stateful.py | 10 +- .../tests/test_ndsel_tensorstore.py | 8 +- packages/zarr-indexing/tests/test_reader.py | 9 +- .../zarr-indexing/tests/test_transform.py | 37 ++- .../tests/test_wire_boundaries.py | 63 +++++ 55 files changed, 818 insertions(+), 679 deletions(-) create mode 100644 packages/zarr-indexing/changes/4345.bugfix.md create mode 100644 packages/zarr-indexing/changes/4345.doc.md create mode 100644 packages/zarr-indexing/tests/test_boundary.py create mode 100644 packages/zarr-indexing/tests/test_wire_boundaries.py diff --git a/packages/zarr-indexing/CONTRIBUTING.md b/packages/zarr-indexing/CONTRIBUTING.md index 632e24929f..bcbe51c079 100644 --- a/packages/zarr-indexing/CONTRIBUTING.md +++ b/packages/zarr-indexing/CONTRIBUTING.md @@ -8,16 +8,16 @@ just test # run the test suite (extra args go to pytest) just lint # ruff, same invocation as CI just typecheck # pyright, same invocation as CI just docs-check # strict build of the docs site -just check # all of the above +just check # the checks above plus TensorStore parity just docs-serve # serve the docs site locally ``` -Run them from this directory, or from anywhere in the repository as +Run them from this directory, or from the repository root as `just packages/zarr-indexing/`. -The test recipe runs against the workspace-root environment, because the -chunk-resolution tests exercise this package against `zarr`'s chunk grids and -`zarr` is deliberately not a dependency of this package. +The test recipe layers this package into the repository-root environment. +Chunk-resolution tests use this package’s own grids; the Dask example also +uses `zarr`, which is not a dependency of the base indexing package. ## License diff --git a/packages/zarr-indexing/README.md b/packages/zarr-indexing/README.md index 4328de1cc1..2992dcf280 100644 --- a/packages/zarr-indexing/README.md +++ b/packages/zarr-indexing/README.md @@ -5,9 +5,9 @@ Composable, lazy coordinate transforms for Zarr array indexing. Documentation: This package implements TensorStore-inspired index transforms. The core idea: -every indexing operation (slicing, fancy indexing, etc.) produces a coordinate -mapping from user space to storage space. These mappings compose lazily — no -I/O until you explicitly read or write. +each supported indexing operation (slicing, fancy indexing, etc.) produces a coordinate +mapping from user space to storage space. These mappings compose without reading selected source values. `LazyArray` +materializes them on request; the transform algebra itself performs no source I/O. Key types: @@ -15,9 +15,9 @@ Key types: accessor: `LazyArray.from_numpy(numpy_array).lazy[10:50, ::2].lazy.oindex[[3, 1, 1], :]` composes a transform and returns a new view without reading data, and `result()` materializes it into owned system memory. `LazyArray(source)` uses - the conservative basic reader; `from_numpy` explicitly selects NumPy's - optimized reader. Device arrays require an explicit custom reader responsible - for transferring values into the supplied system-memory output buffer. + the basic reader; `from_numpy` selects `numpy_reader`, which currently uses + the same slab-and-gather implementation. Device sources that refuse NumPy + conversion need a custom reader to transfer values into the output buffer. - `Reader` — the explicit backend execution boundary: transforms say which values belong in the result, while readers say how a backend obtains them - `IndexDomain` — a rectangular region of integer coordinates @@ -32,9 +32,11 @@ Key types: dimension can depend on the input - `compose` — chain two transforms into one -The package depends only on NumPy and the standard library; it does not import -`zarr`. It is developed in the [zarr-python](https://github.com/zarr-developers/zarr-python) -repository and consumed by `zarr` to resolve array indexing operations. +The base package depends on NumPy and the standard library; its optional testing +module also requires Hypothesis. The package does not import `zarr`. It is developed +in the [zarr-python](https://github.com/zarr-developers/zarr-python) repository, +and its examples include reading Zarr arrays through Dask. Installing it does not +replace Zarr's indexing implementation. ## Installation diff --git a/packages/zarr-indexing/benchmarks/README.md b/packages/zarr-indexing/benchmarks/README.md index 28ef3e6a16..e5d39b1a89 100644 --- a/packages/zarr-indexing/benchmarks/README.md +++ b/packages/zarr-indexing/benchmarks/README.md @@ -22,6 +22,19 @@ whether input arrays and transform construction are included. Repeated local column access measures cache reuse, which trades allocation against retained memory. Bounded coordinate batches avoid constructing the full coordinate array. +The current script reports `peak_mib` as the incremental peak tracked by +`tracemalloc` during a separate invocation after the timed calls. It is not +process RSS or total retained memory. Inputs and most transforms are constructed +before measurement. The case walks construct plans and consume projections +without retaining them; `all_coordinates` materializes the complete coordinate +array. There is no bounded-batch coordinate workload in this script. + +`local_rows` reuses one prepared table: its first timed invocation populates the +local-coordinate cache, while later timed invocations and the allocation probe +reuse that cache. Its median mixes a cold first call with warm calls, and its +reported peak excludes the already-retained cache. A separate fresh-table +measurement would be needed to quantify cold cache construction. + These scripts measure planning rather than codec or storage throughput. Repeat measurements with alternating operation order before interpreting small timing differences. Preserve raw benchmark output as an experiment artifact rather than diff --git a/packages/zarr-indexing/benchmarks/chunk_planning.py b/packages/zarr-indexing/benchmarks/chunk_planning.py index 934d68dcbc..d7eadf248d 100644 --- a/packages/zarr-indexing/benchmarks/chunk_planning.py +++ b/packages/zarr-indexing/benchmarks/chunk_planning.py @@ -25,6 +25,11 @@ def measure(operation: Callable[[], Any], repeats: int) -> dict[str, float]: + """Time repeated calls, then trace one additional call's incremental peak. + + State retained by earlier calls can affect the extra call; this is not a + measurement of process RSS or all memory owned by the operation's inputs. + """ samples = [] for _ in range(repeats): start = time.perf_counter() @@ -84,6 +89,8 @@ def read_local_rows() -> None: for row in range(len(table)): table.local[table.run(row)] + # The first timed call fills table.local; later calls, including the traced + # allocation probe, reuse it. The table and retained cache are not re-created. results["local_rows"] = measure(read_local_rows, args.repeats) part = plan_chunks( IndexTransform.from_shape((100, 100, 100)), diff --git a/packages/zarr-indexing/changes/4345.bugfix.md b/packages/zarr-indexing/changes/4345.bugfix.md new file mode 100644 index 0000000000..928114728f --- /dev/null +++ b/packages/zarr-indexing/changes/4345.bugfix.md @@ -0,0 +1 @@ +Reject invalid wire index-array values and unrepresentable normalized bounds/ranks, and raise an explicit error for unsupported intersections sharing an affine and lookup input axis instead of returning incorrect coordinates. Group negative chunk-coordinate tuples without merging distinct chunks. diff --git a/packages/zarr-indexing/changes/4345.doc.md b/packages/zarr-indexing/changes/4345.doc.md new file mode 100644 index 0000000000..562e114e4e --- /dev/null +++ b/packages/zarr-indexing/changes/4345.doc.md @@ -0,0 +1,3 @@ +Correct indexing, reader, serialization, cache, and integration descriptions to match supported behavior; qualify NumPy/TensorStore compatibility and performance claims. + +Describe current contracts in source and test docstrings instead of narrating prior implementations. Clarify that immutable index coordinates do not snapshot source values. diff --git a/packages/zarr-indexing/docs/api/grid.md b/packages/zarr-indexing/docs/api/grid.md index b7c376eb85..012c9df6dc 100644 --- a/packages/zarr-indexing/docs/api/grid.md +++ b/packages/zarr-indexing/docs/api/grid.md @@ -11,8 +11,9 @@ objects whose `shape` is the valid data size and whose `codec_shape` preserves the full codec-buffer size at a regular-grid boundary. `dimension_grids_from_chunks` returns these compact dimensions: integer chunk -shapes become `FixedDimension` instances and explicit per-axis edge sequences -become `VaryingDimension` instances. `DimensionGridLike` remains the narrow +shapes become `FixedDimension` instances and positive per-axis edge sequences +become `VaryingDimension` instances. An empty-axis sequence of zeros (including +an empty sequence) becomes `FixedDimension(size=0, extent=0)`. `DimensionGridLike` remains the narrow protocol used by the chunk planner, while `EdgeDimensionGrid` is kept for explicit edge-based and coordinate-origin examples. diff --git a/packages/zarr-indexing/docs/api/index.md b/packages/zarr-indexing/docs/api/index.md index a5b7163bbd..8b7a5d6515 100644 --- a/packages/zarr-indexing/docs/api/index.md +++ b/packages/zarr-indexing/docs/api/index.md @@ -25,12 +25,11 @@ and the wire format built on top of it. - [`zarr_indexing.domain`](domain.md) — `IndexDomain`, a rectangular region of integer coordinates with an explicit (possibly non-zero) origin - [`zarr_indexing.output_map`](output_map.md) — `ConstantMap`, `DimensionMap`, - and `ArrayMap`: three representations of a set of integer coordinates, one - per storage dimension + and `ArrayMap`: three coordinate mappings that preserve order and duplicates, + one per storage dimension - [`zarr_indexing.transform`](transform.md) — `IndexTransform`, which pairs a domain with output maps, plus the indexing (`[...]`, `.oindex`, `.vindex`), - `intersect`, and `translate` operations, and `selection_to_transform` - transforms into one + `intersect`, `translate`, and `compose` operations, and `selection_to_transform` **Chunk resolution** diff --git a/packages/zarr-indexing/docs/api/lazy_array.md b/packages/zarr-indexing/docs/api/lazy_array.md index f855511d46..13740981cd 100644 --- a/packages/zarr-indexing/docs/api/lazy_array.md +++ b/packages/zarr-indexing/docs/api/lazy_array.md @@ -7,21 +7,21 @@ reader and composes its transform without reading data. `result()` allocates owned system memory, then calls that reader once for each projected part. Rectangular parts write directly into their final slices; advanced placement may first use an owned dense temporary. `LazyArray(source)` assumes only basic -indexing, while `LazyArray.from_numpy(array)` explicitly selects NumPy's -optimized reader. +indexing, while `LazyArray.from_numpy(array)` selects `numpy_reader`. Both +currently use the same slab-and-gather implementation. The built-in readers lower through NumPy system memory and support sources -whose basic reads can be converted there. They do not implicitly transfer -device arrays; a device source needs an explicit custom reader that transfers -into the supplied system-memory output. Derived views and parts share their +whose basic reads can be converted there. A device source that refuses NumPy +conversion needs a custom reader that transfers into the output buffer. Derived views and parts share their reader and part views may be materialized concurrently, so stateful readers must synchronize their own mutable state. Every public `Partition.view.transform` directly maps that view's zero-origin coordinates into its raw `Partition.view.array`, including for non-first partitions. `Partition.projection.chunk_transform` intentionally stays local to -the selected chunk. During materialization the reader receives both frames in -one `ReadContext`: the public global transform in `context.transform` and the -same local plan in `context.projection`. +the selected chunk. During parent materialization (`view.result(parts=parts)`) the reader receives +both frames in one `ReadContext`: the public global transform in `context.transform` and the +same local plan in `context.projection`. Direct `part.view.result()` calls +provide no projection; readers requiring it must use parent assembly. ::: zarr_indexing.lazy_array diff --git a/packages/zarr-indexing/docs/api/reader.md b/packages/zarr-indexing/docs/api/reader.md index 39129c4d47..3944336745 100644 --- a/packages/zarr-indexing/docs/api/reader.md +++ b/packages/zarr-indexing/docs/api/reader.md @@ -34,7 +34,7 @@ class RecordingReader: self.calls = [] def read_into(self, source, context, out, /): - self.calls.append((source, context, out)) + self.calls.append((source, context, out.shape, out.dtype)) self.inner.read_into(source, context, out) @@ -44,7 +44,8 @@ view = LazyArray.from_numpy(array).with_reader(outer) values = view.result() ``` -Both wrappers observe the same three objects, in outer-to-inner order. This +Both wrappers observe the same arguments, in outer-to-inner order, and log +output metadata without retaining the output buffer. This delegation pattern supports policies such as logging and caching without library-defined wrapper primitives. diff --git a/packages/zarr-indexing/docs/design-notes.md b/packages/zarr-indexing/docs/design-notes.md index 59eef4dacd..cb80e5ba19 100644 --- a/packages/zarr-indexing/docs/design-notes.md +++ b/packages/zarr-indexing/docs/design-notes.md @@ -16,7 +16,7 @@ visual guide owns the mechanics of ## Relationship to TensorStore The core is [TensorStore's](https://google.github.io/tensorstore/index_space.html) -index-transform model, reimplemented in Python against NumPy. The visual guide +index-transform model, implemented here in Python against NumPy. The visual guide introduces the shared model in [Coordinates are addresses](guide/index.md#coordinates-are-addresses) and [Lazy views compose](guide/index.md#lazy-views-compose); the comparison here is @@ -28,13 +28,14 @@ about the deliberately matching semantics: - **Slice semantics.** Slice bounds are literal domain coordinates: no clamping, no negative wrapping, non-empty intervals must be contained in the domain, and a strided slice's domain origin is `trunc(start/step)` rounded - toward zero. Every one of those rules was executed against tensorstore 0.1.84 - and is pinned in `tests/test_tensorstore_parity.py`. -- **The wire format.** A canonical [ndsel](ndsel.md) transform body is, - field-for-field, a TensorStore `IndexTransform` minus the `kind` - discriminator, and `tests/test_ndsel_tensorstore.py` loads our bodies into - `tensorstore.IndexTransform(json=...)` and round-trips them back through our - engine layer. + toward zero. `tests/test_tensorstore_parity.py` compares the enumerated cases with + TensorStore when that optional dependency is installed. +- **The wire format.** [ndsel](ndsel.md) uses TensorStore's domain and + output-map field names for transform bodies, with an additional `kind` + discriminator. `tests/test_ndsel_tensorstore.py` checks interoperability for + the tested cases. Their validation rules differ, and loading and + re-emitting a message through the engine can normalize or discard metadata; + see [lowering to a transform](ndsel.md#lowering-to-a-transform). - **Chunk partitioning.** Both factor a transform over a grid before visiting any cell, rather than intersecting the whole transform with each chunk. TensorStore's `IndexTransformGridPartition` holds strided sets and index @@ -47,17 +48,19 @@ about the deliberately matching semantics: components within a mixed request. Both derive the per-chunk transforms from the partition ([the guide](guide/index.md#a-plan-is-a-product-of-per-axis-tables) shows the tables). TensorStore keeps strided sets implicit, while this - library materializes their per-axis rows for vectorized consumers. Diagonals are rejected here; supporting them needs a strided set per - *input* dimension spanning every storage axis that reads it, TensorStore's - representation. + library materializes their per-axis rows for vectorized consumers. Pure + affine diagonals need grouping by input dimension; mixed affine/index-array + dependencies need joint partitioning. TensorStore classifies a connected + component containing index-array edges as an index-array set + ([source](https://github.com/google/tensorstore/blob/66b2ce5290fa2ec5c8019682391421062ce767a2/tensorstore/internal/grid_partition.h#L58-L67)). Four deliberate differences: | | TensorStore | `zarr-indexing` | | --- | --- | --- | -| Dialect | One strict dialect everywhere: literal coordinates, no negative wrapping | The algebra keeps that dialect; each public boundary picks its own. [`LazyArray`](api/lazy_array.md) speaks positional NumPy, `zarr.Array.lazy` speaks literal. [`zarr_indexing.boundary`](api/boundary.md) is the translation | +| Dialect | Coordinate indices are literal; negative coordinates do not wrap | The algebra keeps that dialect; each public boundary picks its own. [`LazyArray`](api/lazy_array.md) speaks positional NumPy, `IndexTransform` speaks literal. [`zarr_indexing.boundary`](api/boundary.md) is the translation | | Scheduling | An internal C++ scheduler owns concurrency and chunk ordering | [`parts()`](api/lazy_array.md) exposes the partition structure so the caller's own scheduler — dask, a thread pool, a task queue — drives it | -| Wire format | Implementation-defined JSON, specified by what the implementation accepts | [ndsel](ndsel.md) is spec-first, with a vendored language-agnostic conformance corpus every implementation runs | +| Wire format | [Documented JSON schema](https://google.github.io/tensorstore/index_space.html#index-transform) | [ndsel](ndsel.md) is spec-first, with a vendored conformance corpus exercised by this implementation | | Backends | A driver ecosystem (zarr, N5, neuroglancer, GCS, …) built into the library | No drivers. The default reader needs `shape`, `dtype`, basic integer/slice indexing, and selected slabs convertible to NumPy system memory; other backends use explicit custom readers. A device reader owns transfer into the supplied system-memory output | The mechanics of a @@ -74,19 +77,18 @@ and caller-supplied grid; it does not own reads, writes, buffers, locks, or scheduling. Zarr can therefore plan reads against an inner codec-chunk grid and writes against an atomic shard grid; napari or dask can turn the same projections into tasks without putting a dask dependency in this package. -`coverage` is relative to that selected grid: `full` proves a blind replacement -safe, `partial` proves it is not, and `unknown` conservatively covers fancy -selections whose duplicates would require additional work to classify. - -The comparison also runs the other way. TensorStore is a mature, heavily -optimized C++ system whose performance this library cannot approach. Independent strided planning -here is per axis, but each materialized `ChunkProjection` is -Python-level bookkeeping over NumPy — two domains, two transforms and the -projection itself — so the per-part overhead of the object view is -significant; a consumer that reads the partition's tables directly pays no -per-chunk object construction. This library is small and depends on nothing beyond -NumPy, so the algebra can be adopted by a Python project that wants the model -without the C++ runtime. +`coverage` describes selection coverage relative to that grid. A `full` +classification can help a writer avoid reading old values, but does not by +itself prove that a write is safe: encoding requirements, conflicts, duplicate +semantics, and concurrency remain consumer responsibilities. `unknown` means +the planner has not established complete or partial coverage. + +This implementation performs Python-level bookkeeping over NumPy. This page +provides no benchmark establishing a general performance ordering against +TensorStore; costs depend on the selection and execution backend. + +The partition tables can be consumed without constructing a `ChunkProjection` +for each chunk. Materializing projections adds Python object construction. ## Bounding-box selections vs query selections @@ -102,11 +104,11 @@ and the coordinates it touches form a regular lattice. Basic indexing produces one, and composing basic indexing with basic indexing keeps one. **A query** is a transform with at least one `ArrayMap` — an explicit lookup -table of coordinates. It costs `O(n)` to store, it has no locality (the -coordinates may repeat, reverse, or scatter arbitrarily), and intersecting it -with a region means scanning it. `oindex`, `vindex`, and boolean masks all -produce one, and once an axis is a query, subsequent basic indexing cannot make -it a box again. A second query composes onto any axis of an existing one — +table of coordinates. Its stored coordinate arrays cost space proportional to their stored size. +Coordinates may repeat or scatter, but can also be contiguous and local. +Current query-resolution paths inspect these arrays. Fancy indexing can produce +a query, but singleton or constant selections can collapse to `ConstantMap`; +subsequent indexing can therefore make a query affine again. A second query composes onto any axis of an existing one — including the axes it merely broadcasts along — by evaluating the existing lookup tables at the new coordinates. @@ -116,9 +118,10 @@ planning and materialization. [ndsel](ndsel.md) encodes the same split in its message kinds: `point`, `box`, and `slice` desugar to constant and affine output maps and are always boxes; -`points` desugars to `index_array` maps, and a `transform` body is a box -exactly when none of its output maps carries an `index_array`. A consumer can -therefore classify a selection off the wire without materializing anything: +`points` desugars to `index_array` maps. A transform without index-array maps +is a box in the engine’s structural classification. Loading can further +simplify degenerate index arrays to constants, so an arbitrary incoming body +with `index_array` fields need not remain a query. For example: ```python from zarr_indexing import IndexTransform @@ -134,16 +137,13 @@ gather.to_json()["output"][0] # 'index_array_bounds': ['-inf', '+inf']} ``` -The distinction matters to consumers of a selection. A box can be tiled into -rectangular dask chunks or passed to a viewer or tile server that only accepts -rectangles; a query cannot, and has to be resolved into a gather. A box can also -be served as a single strided slab read, but the read has to be strided: reading -its bounding box and discarding the rest transfers proportionally more data as -soon as any stride exceeds 1. The two also behave differently under -partitioning: a box touches a regularly-spaced run of parts, in increasing -order, each at most once — a stride larger than a part's extent skips parts -outright, so the run is not contiguous — while a query can touch any subset of -them, in any order, more than once. +The representation helps a consumer choose a lowering strategy. Independent +affine axes can often be read with slices plus reversal, permutation, or +broadcasting. Arbitrary affine maps can also express diagonals, so the absence +of `ArrayMap` alone is not proof of a rectangular slab. Queries may be lowered +through gathers or covers, and can sometimes simplify to slices. Chunk plans +group selected coordinates by chunk while preserving result placement; repeated +coordinates do not imply repeated visits to the same chunk. [`LazyArray`](api/lazy_array.md) exposes the category directly: @@ -173,16 +173,20 @@ gather.shape # (3, 80) `bounding_box()` is defined for both: it is the hull, the smallest interval per storage dimension containing every coordinate the selection reaches. `strides()` is defined only for a box and gives the step per dimension. -Together the two describe a box selection completely. - -Both are needed, because a box is dense in its hull only when every stride is -1. The slab above spans a 40x77 hull over the 40x20 cells it selects, so a +These summaries omit traversal direction, input-axis correspondence, and +result layout. For example, forward and reversed views have identical bounds +and stride magnitudes but different ordered results. Use the transform for the +complete selection. + +For independent axes with multiple selected coordinates, a stride magnitude +greater than one leaves gaps in the hull. Singleton axes are an exception, +and a query can also cover every cell of its hull. The slab above spans a 40x77 hull over the 40x20 cells it selects, so a consumer that issued one rectangular read of the hull and discarded the rest would transfer 3.85x the data. A query's hull is looser still and carries no stride at all: 88 rows of hull over three selected rows. An empty *box* touches no coordinate to report an interval around, so `bounding_box()` is `None` while `strides()` still answers — the step is a property of the selection's shape, not -of the region it reaches. Only a query returns `None` from both. +of the region it reaches. An empty query returns `None` from both. There is deliberately no separate `BoxView` type today. A statically-typed rectangular-only view is a plausible next step, but it should be introduced by @@ -248,8 +252,8 @@ could accept and finishing the rest elsewhere. A reader lowers the complete transform and can compose through delegation instead. This resembles [zarrita.js store extensions](https://zarrita.dev/packages/zarrita.html), where storage-specific behavior is an explicit extension point rather than an -inferred array capability. The implementation remains independently authored: -no code is shared with TensorStore, xarray, or zarrita.js. +inferred array capability. This is an architectural analogy, not a claim of API or implementation +compatibility. ## Current scope @@ -263,7 +267,7 @@ re-bases every view to origin 0, so the positional dialect never exposes it; a caller working with `IndexTransform` directly will see it, and re-bases explicitly with `translate_domain_to` for NumPy-shaped coordinates. -Fancy selections compose without restriction: a second `oindex`/`vindex`/mask +Supported fancy selections compose across already-fancy views: a second `oindex`/`vindex`/mask step may land on any axis of an already-fancy view, including axes an existing index array merely broadcasts along, so `lazy.oindex[[2, 0], :].lazy.oindex[:, [1, 3]]` selects the outer product it @@ -273,29 +277,26 @@ which evaluates the existing lookup tables at the new coordinates — rather tha rewritten in place. Resolution classifies the result by structure (`index_array_structure`): pure per-axis outer products keep the orthogonal resolvers, and everything else — correlated maps, mixtures, index arrays -sharing an input axis (a diagonal gather, reachable only by hand-building a -transform) — takes the general reader/intersection path. Chunk planning +sharing an input axis (as in paired vectorized coordinates) — takes the general reader/intersection path. Chunk planning factors index arrays into connected dependency components before flattening, so independent groups do not expand one another. Vectorized selection preserves broadcast singletons to retain those dependencies. -Three limits remain, all intentional and all expected to be lifted: +Some current limits are: -- **Affine diagonals.** A hand-built transform in which two output maps read - one input dimension — two slice maps, or a slice map and an orthogonal index - array — is rejected at planning with `ValueError`; a correlated index array - varying over a dimension a slice map also reads is rejected with - `NotImplementedError`. No selection dialect produces either. Supporting them - needs a strided set per *input* dimension spanning all dependent storage - axes, TensorStore's connected-component representation. *Planned.* +- **Shared affine dependencies.** Planning rejects two affine output maps + sharing an input axis with `ValueError`. An index array sharing a varying + input axis with an affine map takes the general classification and raises + `NotImplementedError`. Pure affine diagonals would need grouping dependent + storage axes by input dimension; mixed components need joint partitioning. - **Finite explicit bounds only.** `IndexDomain` has no implicit or unbounded dimensions; the message layer will normalize a body with `"-inf"`/`"+inf"` bounds, but the engine layer refuses to lower one into a transform. - TensorStore supports both. *Planned.* + TensorStore supports both. - **Labels are carried, not propagated.** `IndexDomain` holds optional dimension labels and the wire format round-trips them, but indexing operations build new domains without them, so a label does not survive a - slice. *Planned.* + slice. ## Selection to chunk operations diff --git a/packages/zarr-indexing/docs/guide/index.md b/packages/zarr-indexing/docs/guide/index.md index 68192f58cb..f55794c4c0 100644 --- a/packages/zarr-indexing/docs/guide/index.md +++ b/packages/zarr-indexing/docs/guide/index.md @@ -216,12 +216,17 @@ description; the assertion's call to `result()` is the first operation in the example that materializes the selected data. !!! warning "Stop here: the materialization boundary" - Indexing through `.lazy[...]` never reads. These do: + Indexing through `.lazy[...]` composes a selection without reading source values. + These operations request values: - `result()` - eager indexing of the wrapper: `view[...]` - - `numpy.asarray(view)`, or passing the view to any NumPy function - (`numpy.add(view, 1)` converts, and therefore materializes, the view) + - `numpy.asarray(view)` and NumPy operations that convert the view + (`numpy.add(view, 1)` does so; `numpy.shape(view)` and `numpy.ndim(view)` + can use metadata without reading values) + + Dask tokenization may also inspect values, depending on the wrapped source + and tokenization path. Python arithmetic such as `view + 1` raises `TypeError` instead: this wrapper defers indexing, not a general compute graph. @@ -384,8 +389,9 @@ each bundles a sub-view of the request (`.view`), that chunk's projection Within one `Partition`, the frames divide: `Partition.view.transform` is a different, global transform — it maps the part view directly into the raw wrapped source — while only `Partition.projection.chunk_transform` uses -zero-origin chunk-local coordinates. Readers receive both so the global -source address and the local planning frame cannot be confused. +zero-origin chunk-local coordinates. Parent assembly passes both frames to +the reader. Direct `part.view.result()` calls supply the global transform +with `projection=None`. | Projection field | What its output coordinates mean | | --- | --- | diff --git a/packages/zarr-indexing/docs/guide/integrations.md b/packages/zarr-indexing/docs/guide/integrations.md index f787d11f6c..a0bc721175 100644 --- a/packages/zarr-indexing/docs/guide/integrations.md +++ b/packages/zarr-indexing/docs/guide/integrations.md @@ -3,8 +3,9 @@ For the complete path from indexing syntax to chunk coordinates, local selectors, and result positions, start with [From a selection to chunk operations](selection-flow.md). -This package supplies indexing plans. It does **not** supply scheduling, -caching, codecs, or async orchestration. A consumer decides when projections +The core package supplies indexing plans and synchronous readers. It does not +provide a general scheduler, codec pipeline, or async execution engine. A +synchronous cache is included as an example. A consumer decides when projections run, how decoded chunks are obtained, and where completed values are retained. An `IndexTransform` says which source values belong in a result; a `Reader` lowers that complete transform for one backend. The reader does not choose @@ -61,21 +62,21 @@ afterward, so accessing a row does not recalculate every point. ## One slab read or many part reads -A backend with its own native subset read — a Rust or C zarr implementation, -a database, an HTTP range endpoint — resolves a **dense box** (`is_box` with -every stride 1) best as a single read: hand it the whole selection and let it -dispatch to chunks, decode in parallel, and partial-decode shards on its own -side of the boundary. Splitting that read along this library's partitioning -only adds round-trips. Every **other** selection — a strided box, an `oindex` -or `vindex` gather — is where the partitioning earns its keep. The **cover** -of a read is the smallest step-1 slab enclosing every coordinate it needs; -partitioned, each part's cover is bounded by that part's box, so a sparse -selection can never force one read of its whole bounding hull (the smallest -rectangle containing every selected coordinate — a thousand rows for the two -of `oindex[[0, 999]]`). +A backend with an efficient native subset operation may benefit from receiving +one complete dense selection so it can choose its own chunk dispatch. Other +backends may benefit from partitioning, including for strided or fancy +selections. The tradeoff depends on the backend, chunk layout, latency, memory, +and selection; a single read is not universally fastest. + +A read's **cover** is the smallest unit-step slab enclosing its coordinates. +With this package's basic readers, partitioning limits each source read to the +part's selected cover. This may reduce over-reading, but a partition that spans +the source can still require the entire hull. Custom readers choose their own +source operations under the reader contract. The composed view carries enough to make that call at materialization time, -and re-partitioning is a pure setter, so the policy is three lines: +and `with_parts()` returns a view with a new partitioning. This example uses +unit strides as a sufficient condition for its independently mapped selections: ```python --8<-- "snippets/integrations.py:dense-box-repartition" @@ -87,20 +88,19 @@ the dense box becomes exactly one backend call. Both regimes go through ### Sources that accept only unit-step slices -The default `basic_reader` pushes strided and descending selections down as -positive-step slices, which reads the minimum but assumes the source accepts -any step. Many backends do not: FFI bindings and range requests often -support nothing but `slice(start, stop, 1)`. Select -[`unit_step_reader`][zarr_indexing.reader.UnitStepReader] for such a source -and every key it receives is an ascending unit-step slice per axis, with +For affine selections, `basic_reader` uses positive-step slices and applies +reversal or layout changes in memory. Fancy selections can require reading a +cover containing unselected values. The source must accept the emitted steps. +Use [`unit_step_reader`][zarr_indexing.reader.UnitStepReader] for a source that +accepts only unit steps. Every key it receives is an ascending unit-step slice per axis, with strides, reversals, and gathers applied to the in-memory block instead: ```python view = LazyArray(source).with_reader(unit_step_reader) ``` -A strided selection then over-reads its cover by the stride factor, which the -partitioning above bounds by one part. +For strided selections, the ratio of cover cells to selected cells depends on +stride, length, and endpoint alignment. Partitioning can reduce that cover. ## napari-like consumer @@ -205,13 +205,12 @@ decoded chunks. Every read delta follows directly from the viewport request: | 4 | `image[1:5, 2]` | `(0, 0)` | `(0, 0)`, `(1, 0)` | The evicted chunk is reloaded while the required ready chunk is retained. | | 5 | `image[3:5, 4:6]` | `(1, 1)` fails; no repeated read; `(1, 1)` succeeds after retry | `(0, 0)`, `(1, 1)` | Failure is retained until explicit retry; the repaired source then returns `[[28, 29], [36, 37]]`. | -Chunks required by an active request are pinned through assembly, so a request -may temporarily span more chunks than the steady-state capacity. Capacity is -counted in decoded chunks—not records or bytes—and eviction occurs only after -all requested values have been placed. Because pinning and materialization use -the same prepared tuple, those lifecycle decisions cannot drift from the parts -that are actually read, and the cache never has to infer or reconstruct a -projection. +The example defers eviction until a successful outermost request finishes, so +it can temporarily exceed capacity during assembly. Capacity counts decoded +chunks, not bytes, event records, or temporary arrays. Failed requests skip +that eviction step. The example assumes an unchanged source and one source/grid +per reader; it does not implement invalidation or synchronization for concurrent +requests. The prepared tuple supplies the projections used for each read. The event log makes the failure boundary equally explicit: diff --git a/packages/zarr-indexing/docs/guide/patterns.md b/packages/zarr-indexing/docs/guide/patterns.md index 9d22c4eb4e..f2f2ddbe2a 100644 --- a/packages/zarr-indexing/docs/guide/patterns.md +++ b/packages/zarr-indexing/docs/guide/patterns.md @@ -302,10 +302,11 @@ so they equal the zero-origin models after `translate_domain_to`: --8<-- "snippets/indexing_patterns.py:indexing-patterns" ``` -`LazyArray` adds nothing to these semantics: it is a regular array-like API -whose `.lazy`, `.lazy.oindex`, and `.lazy.vindex` accessors compile the same -dialects to the same transforms — the only difference is the return type, a -view instead of an array. The test suite holds the wrapper to this matrix. +`LazyArray` exposes the transform machinery through a positional array-like +API. Its `.lazy`, `.lazy.oindex`, and `.lazy.vindex` accessors return views and +normalize positions before composition. This boundary differs from the literal +coordinate semantics of `IndexTransform`, as the following table shows. The +executable matrix checks the documented cases, not every possible NumPy expression. ## Positions vs literal coordinates diff --git a/packages/zarr-indexing/docs/guide/selection-flow.md b/packages/zarr-indexing/docs/guide/selection-flow.md index 571baa3869..749a9c36ae 100644 --- a/packages/zarr-indexing/docs/guide/selection-flow.md +++ b/packages/zarr-indexing/docs/guide/selection-flow.md @@ -171,7 +171,7 @@ pairs: | `(0,)` | `[1]` | `[1]` | | `(1,)` | `[2, 2, 0]` | `[0, 2, 3]` | -Chunk `(1,)` is fetched once, but its local value `2` contributes to two result +Chunk `(1,)` is fetched once, but the value at local coordinate `2` contributes to two result positions. Chunk visitation order need not be result order; `out_selection` restores the requested arrangement. @@ -188,6 +188,9 @@ than one. Repeated coverage is not full; fancy coverage remains conservative. Zarr's merge operation can skip a read for a full data-extent write and allocate a fill-valued codec buffer when the selected data is smaller than that buffer. +That shortcut also requires the consumer's expected value layout and order; +complete coverage by a reversal or reordered gather alone does not establish +that the supplied buffer can be copied directly into codec order. Touching every chunk alone does not prove full coverage of each chunk. Planning does not fetch bytes, decode buffers, choose concurrency, or define diff --git a/packages/zarr-indexing/docs/index.md b/packages/zarr-indexing/docs/index.md index 87e5a4b7a1..598282b698 100644 --- a/packages/zarr-indexing/docs/index.md +++ b/packages/zarr-indexing/docs/index.md @@ -5,8 +5,7 @@ the *declaration* of an array indexing expression from the result of that expres Developed for use in [`zarr`](https://zarr.readthedocs.io). -Inspired by [TensorStore](https://google.github.io/tensorstore/), which pioneered -the approach used here. +Inspired by [TensorStore's index-transform model](https://google.github.io/tensorstore/index_space.html). ## Install @@ -28,8 +27,9 @@ you want its values: --8<-- "snippets/canonical_slice.py:landing-quickstart" ``` -Nothing is read until the `result()` call, however many selections are -composed. [Lazy views compose](guide/index.md#lazy-views-compose) shows how +Composing these selections does not read source values; the example reads them +at `result()`. Construction inspects source metadata, and Dask tokenization can +inspect source values. [Lazy views compose](guide/index.md#lazy-views-compose) shows how the chain stays one description, and where the materialization boundary is. ## Learn more diff --git a/packages/zarr-indexing/docs/ndsel.md b/packages/zarr-indexing/docs/ndsel.md index 94971d49bf..090dce64c4 100644 --- a/packages/zarr-indexing/docs/ndsel.md +++ b/packages/zarr-indexing/docs/ndsel.md @@ -14,11 +14,13 @@ before a transform is serialized. `zarr-indexing` implements ndsel in two layers | Layer | Module | Depends on | Job | | --- | --- | --- | --- | -| Message | [`zarr_indexing.messages`](api/messages.md) | stdlib only | JSON in, canonical JSON out. Validates and desugars. Never rounds, clamps, or drops information. | +| Message | [`zarr_indexing.messages`](api/messages.md) | stdlib only | Validates and desugars JSON, removing redundant constant-map fields. | | Engine | [`zarr_indexing.json`](api/json.md) | NumPy | Lowers a *canonical* body into an in-memory [`IndexTransform`](api/transform.md), and back. | Constraints that only make sense for a real array — finite bounds, index -arrays as `ndarray`s — live in the engine layer and nowhere else. As a result, +arrays as `ndarray`s — are checked during engine lowering. The message layer +also limits input rank to 32 and checks affine input-dimension references. +These checks are stricter than the draft's required message validation. As a result, `messages` normalizes a message with `"-inf"` bounds that `IndexTransform.from_json` refuses to lower. @@ -47,9 +49,13 @@ normalize_ndsel({"kind": "box", "inclusive_min": [10, 5], "shape": [40, 1]}) ``` Normalization is idempotent: re-tag the output with `kind: "transform"` and -normalizing it again returns the same body. Because the canonical body is -field-for-field a TensorStore `IndexTransform` minus `kind`, a normalized -message loads directly into `tensorstore.IndexTransform(json=...)`. +normalizing it again returns the same body. The canonical body uses +TensorStore's `IndexTransform` field vocabulary, but normalization does not +guarantee acceptance by TensorStore. For example, the message layer leaves +index-array content unchecked, whereas TensorStore validates it; TensorStore +also requires unique nonempty labels and restricts finite index values to +`[-(2**62 - 2), 2**62 - 2]`. See +[TensorStore's index-space constraints](https://google.github.io/tensorstore/index_space.html). Both entry points raise [`NdselError`](api/messages.md#zarr_indexing.messages.NdselError), which @@ -64,12 +70,13 @@ Four are shorthands; the fifth is the canonical form itself. | `kind` | Fields | Selects | | --- | --- | --- | | `point` | `coords` | A single element. Normalizes to rank 0 with one `constant` output map per dimension. | -| `box` | `inclusive_min`, one of `exclusive_max` / `inclusive_max` / `shape`, `labels` | A rectangular region. Exactly one upper-bound spelling may appear. | +| `box` | `inclusive_min`, one of `exclusive_max` / `inclusive_max` / `shape`, `labels` | A rectangular region. At most one upper-bound spelling may appear; omission gives implicit positive infinity. | | `slice` | `start`, `stop`, `step`, `labels` | A strided region, one Python-style slice per dimension. | | `points` | `coords` (a list of coordinate rows) | An explicit list of points — the `vindex` case. Normalizes to one `index_array` output map per dimension over a shared rank-1 input domain. | | `transform` | `input_rank`, `input_inclusive_min`, one of the three `input_*` upper bounds, `input_labels`, `output` | The full canonical form. | -Value rules the message layer enforces throughout: every integer is a 64-bit +Value rules for validated fields (excluding verbatim `index_array` payloads +and discarded constant-map fields): every integer is a 64-bit signed value; JSON booleans are **not** integers (Python's `isinstance(True, int)` is guarded against explicitly); the `"-inf"` / `"+inf"` sentinels are legal only in bound positions; and an implicit bound is the @@ -81,17 +88,21 @@ normalization intact. The engine layer converts between canonical bodies and `IndexTransform`s: ```python -from zarr_indexing import IndexTransform +from zarr_indexing import IndexTransform, normalize_ndsel +canonical = normalize_ndsel({"kind": "box", "shape": [2, 3]}) t = IndexTransform.from_json(canonical) -t.to_json() == canonical +assert t.to_json() == canonical ``` `IndexDomain` carries the same pair for a bare domain body, and each output map kind has a `to_json`; `output_index_map_from_json` dispatches the wire's -tagged union back to the right kind. +structurally discriminated union back to the right kind. Exact JSON equality +in this example is not a general round-trip guarantee: implicit flags are +removed, finite `index_array_bounds` are not retained or enforced by the +engine, and degenerate array maps are collapsed. -Two engine constraints apply here and only here. A canonical body carrying a +A canonical body carrying a `"-inf"` or `"+inf"` bound cannot be lowered — an `IndexDomain` addresses a finite array — so `IndexTransform.from_json` raises. And implicit bounds lower *by value*: the `[n]`-bracket flag is a message-layer concern, and the engine @@ -102,21 +113,17 @@ keeps only the integer. ndsel and TensorStore both **reject** an output map that carries both `input_dimension` and `index_array`. The in-memory [`ArrayMap`](api/output_map.md#zarr_indexing.output_map.ArrayMap), though, -records an `input_dimension` to pin the axis an orthogonal (`oindex`) array -varies over. The serializer bridges that gap in both directions: +records its dependency axes in its full-rank array shape, with no +`input_dimension` field: - **On serialize**, a non-degenerate `index_array` map is emitted *without* `input_dimension`. -- **On load**, the in-memory `input_dimension` is reconstructed from the - full-rank array's dependency axes — its non-singleton axes. An array that - solely owns a single non-singleton axis is orthogonal; arrays that share - non-singleton axes, or vary over several, are correlated (`vindex`), and get - `input_dimension = None`. A single 1-D array over a rank-1 domain is - inherently ambiguous between the two flavors and reconstructs as - orthogonal, which is behaviorally identical in that case. - -There is one deliberate exception, and it is the only place a round trip changes -representation rather than preserving it. An all-singleton `index_array` — size +- **On load**, dependency axes are the full-rank array's non-singleton axes. + Maps sharing these axes describe correlated coordinates. The engine also + accepts lower-rank nonempty arrays by prepending singleton axes; that + convenience is not a guarantee of compatibility with other ndsel consumers. + +An all-singleton `index_array` — size 1 — selects the same coordinate regardless of the input, so it is collapsed to a `constant` map on serialize: @@ -137,6 +144,11 @@ The transform is still valid and the output shape is unchanged. A length-1 `oindex` selection therefore round-trips behaviorally (an `ArrayMap` comes back as a `ConstantMap`) rather than by object identity. +Empty index arrays also serialize as constant maps with offset zero: the +empty input domain carries the fact that no coordinates are selected. This +avoids losing trailing shape information in JSON when an array has a leading +zero-length axis. + ## Conformance The package is checked against the language-agnostic ndsel conformance corpus, @@ -145,8 +157,9 @@ vendored unmodified under — one JSON file per message kind plus `errors.json`, with the source commit recorded in `PROVENANCE.md`. Each fixture is either a *success* case (`input` + expected `normalized` body) or an *error* case (`input` + expected -reason code), and an implementation is conformant iff `normalize` reproduces -every one. `tests/test_conformance.py` runs the whole corpus as one +reason code). Matching every fixture establishes corpus conformance; the +fixtures do not prove correctness for every possible input or universal +TensorStore compatibility. `tests/test_conformance.py` runs the whole corpus as one parametrized test per fixture, so a corpus update reports failures fixture by fixture rather than as a single opaque assertion. diff --git a/packages/zarr-indexing/docs/release-notes.md b/packages/zarr-indexing/docs/release-notes.md index 767cd56a84..d0687d8758 100644 --- a/packages/zarr-indexing/docs/release-notes.md +++ b/packages/zarr-indexing/docs/release-notes.md @@ -1,5 +1,6 @@ +## 3.4.0 (2026-09-15) + +### Features + +- JSON metadata validation now delegates to ``msgspec.convert`` for the type + coercions it supports (``Literal`` membership, ``int`` / ``bool`` strictness, + list-to-tuple), replacing the per-field hand-written ``parse_*`` logic. + User-defined attributes retain their existing JSON handling. + A latent generator-exhaustion bug in + ``parse_storage_transformers`` is also fixed. See #3285. + + As a result some metadata inputs are now parsed more strictly. The previous + per-field checks compared values with ``==``, which accepts any numerically + equal object, so a float such as ``2.0`` was accepted as ``zarr_format``; it is + now rejected because it is not an ``int``. Booleans are likewise no longer + accepted where an ``int`` is expected, since ``bool`` is an ``int`` subclass. + Metadata that conforms to the Zarr specification is unaffected. ([#4063](https://github.com/zarr-developers/zarr-python/pull/4063)) + +- `zarr.registry.get_codec_class` now raises `zarr.errors.UnknownCodecError` instead of `KeyError` + when no implementation is registered for a codec, and `zarr.core.config.BadConfigError` instead of + `KeyError` when the implementation named in `config["codecs"][name]` is not registered. + `zarr.registry.get_numcodec` raises `UnknownCodecError` instead of the `ValueError` numcodecs + raises for an unregistered Zarr format 2 codec id (`numcodecs.errors.UnknownCodecError` on + numcodecs 0.15.1 and later). All of these are subclasses of `ValueError`, so `except ValueError` + is unaffected, but `except KeyError` and `except numcodecs.errors.UnknownCodecError` are. + + These errors now name Python packages known to provide the codec, so that a user who cannot read + an array learns what to install: + + ```text + An implementation for codec 'wavpack' is not available. Register one explicitly using the codec + registry (see ...), or install a Python package that registers a codec implementation with + numcodecs. Known packages supporting this codec: wavpack-numcodecs. + ``` + + The tables covering this live in `src/zarr/registry.py`, one per Zarr format, and include the + codecs `numcodecs` gates behind its own optional dependencies (`zfpy`, `pcodec`, `crc32c`, + `msgpack2`). Codec authors can add their published package to them. + + A codec whose `from_dict` raises `KeyError` on a malformed configuration now surfaces as + `zarr.errors.MetadataValidationError` naming the codec and the missing key. Previously it was + reported as `UnknownCodecError: Unknown codec: ''`, presenting a configuration + key as though it were a codec name, and on the `zarr.open` path a bare `KeyError` could be + swallowed by the array-then-group fallback and reported as an unrelated group error. + + `zarr.errors.UnknownCodecError` is now exported from `zarr.errors`. ([#4277](https://github.com/zarr-developers/zarr-python/pull/4277)) + +- `zarr.create_array`, `Group.create_array`, `zarr.from_array`, and the entry points built on them now accept a numpy array as the `chunks` or `shards` specification, alongside ints, tuples, and numpy integer scalars. This is new for that API: it has never accepted numpy arrays in any 3.x release, because each entry point compared the specification to the `"auto"` or `"keep"` sentinel string before normalizing it, and for a numpy array that comparison raised numpy's ambiguous-truth-value `ValueError`. Those sentinel checks are now guarded so array-like specifications reach the normalizer, bringing this API in line with the legacy `zarr.create` / `zarr.array` / `zarr.open_array` functions, which have accepted numpy arrays since 2.x. ([#4329](https://github.com/zarr-developers/zarr-python/pull/4329)) + +### Bugfixes + +- Array creation is now O(1) in the number of chunks per dimension. Chunk + normalization returns a `ChunkGrid` whose uniform dimensions are stored as a + size + extent pair (`FixedDimension`) instead of being expanded to one entry + per chunk, so creating arrays like + `zarr.create_array(store, shape=(2**62,), chunks=(1,), dtype='int32')` succeeds + instantly instead of raising `ValueError` or allocating gigabytes of memory. + The intermediate `ChunksTuple` representation was removed in the process, and + `ChunksLike` now admits per-dimension specs that mix a bare int (uniform chunk + size) with explicit edge-length sequences, matching what the normalizer and + the rectilinear grid spec already accepted. + This fixes the array-creation half of #4174; the coordinate-selection + allocation reported there is still tracked in that issue (#4172 fixed the + related case of sorted 1-D coordinate selections). ([#4218](https://github.com/zarr-developers/zarr-python/pull/4218)) +- Allow `Group.require_array` to accept a `ZDType` for `dtype`, matching the other array creation methods. Previously an existing array could only be required with a string or NumPy dtype. ([#4189](https://github.com/zarr-developers/zarr-python/pull/4189)) +- Consolidated metadata is now reconstructed independently of the order the keys appear in on disk. Previously, sibling subtrees whose keys were not adjacent in the persisted mapping lost their children, which made nodes unreachable through consolidated metadata -- most visibly for sibling groups whose names differ only by case. ([#4227](https://github.com/zarr-developers/zarr-python/pull/4227)) +- `FsspecStore.from_mapper` and `FsspecStore.from_url` no longer fail when converting a synchronous instance of an async-capable filesystem whose storage options contain objects that cannot be serialized to JSON (e.g. an `azure.identity.DefaultAzureCredential`). The async instance is now constructed from the original filesystem arguments instead of a JSON round-trip. ([#4239](https://github.com/zarr-developers/zarr-python/pull/4239)) +- Numpy integers are accepted as chunk sizes again. Since 3.3.0 a per-dimension chunk size that was a numpy integer (e.g. `chunks=(np.int64(2), np.int64(2))`, as produced by any computed chunk shape) raised `TypeError: 'numpy.int64' object is not iterable`, because the scalar chunk path narrowed on `int` while its caller dispatched on `numbers.Integral`. The same regression had broken numpy arrays as chunk specifications through the legacy `zarr.create` / `zarr.array` / `zarr.open_array` functions, which accepted them in 2.x and 3.2.x; those work again. (`zarr.create_array` and the functions built on it gain numpy-array support separately, in #4329.) A chunk specification that is neither an integer nor iterable now reports the offending value instead of failing with an opaque iteration error. ([#4257](https://github.com/zarr-developers/zarr-python/pull/4257)) +- The `cast_value` codec now requires `cast-value-rs>=0.4.2`. Earlier versions of that backend silently corrupted data when handed an array that was not row-major — the layout the `transpose` codec produces — so a `cast_value` codec next to a `transpose` codec would either write transposed values with no error or fail with `ValueError: Input array must be contiguous`. The minimum version is enforced at runtime as well as in the package metadata, so an environment that already has an older `cast-value-rs` installed now raises `ImportError` when the codec is used, instead of corrupting data. ([#4260](https://github.com/zarr-developers/zarr-python/pull/4260)) +- Accept [universal-pathlib](https://github.com/fsspec/universal_pathlib) `UPath` objects wherever + zarr accepts a `StoreLike` value. A remote `UPath` now creates an `FsspecStore` using the + filesystem and storage options the `UPath` already carries, and a local `UPath` creates a + `LocalStore`, so that `UPath('/data')` and `Path('/data')` behave the same. + + Previously this worked only by accident: in universal-pathlib < 0.3 every `UPath` subclassed + `pathlib.Path` and implemented `__fspath__`, so remote paths were either converted to a URI string + by the caller or wrapped in a `LocalStore` that happened to dispatch through fsspec. Since + universal-pathlib 0.3 remote paths do neither, and passing one raised + `TypeError: Unsupported type for store_like`. + + `FsspecStore.from_upath` also now converts the `UPath`'s filesystem to async mode, instead of + raising `TypeError` for synchronous filesystems and warning for sync-mode instances of async ones. ([#4265](https://github.com/zarr-developers/zarr-python/pull/4265)) + +- Explicit per-chunk size lists now always produce a rectilinear chunk grid, + even when the sizes happen to describe a regular grid (all equal, or all equal + with a smaller trailing chunk). Previously such input was silently collapsed to + a regular grid, which changed resize semantics: a regular grid grows by + extending the uniform pattern, while a rectilinear grid appends a new edge + chunk — the behavior an append-oriented layout like `(168,) * 13 + (24,)` + relies on. The grid kind now follows the input syntax, matching 3.2.x: + scalar chunk sizes (including numpy integers and the `-1` sentinel) produce a + regular grid, nested sequences produce a rectilinear grid. Rectilinear grids + remain gated behind `zarr.config.set({"array.rectilinear_chunks": True})`. + See #4174 for the accompanying O(1) chunk normalization change. + + One consequence for users who never enable rectilinear chunks: because a + nested sequence now always requests a rectilinear grid, a per-dimension + sequence of edge lengths that happens to be uniform — for example the + `((4,), (4,))` or `[[3, 3, 1]]` form that a dask array's `.chunks` attribute + produces — is no longer quietly accepted as a regular grid when the + `array.rectilinear_chunks` option is off. Such input raises + `ValueError: Rectilinear chunk grids are experimental and disabled by default`, + exactly as it did in 3.2.x; the silent acceptance existed only in 3.3.0. Pass + one integer per dimension (e.g. `chunks=(4, 4)`, or a dask array's + `.chunksize`) to request a regular grid. + + `zarr.from_array` with the default `chunks="keep"` / `shards="keep"` now + reproduces the source's stored grid exactly: a rectilinear grid is passed + through in O(number of dimensions), with uniform dimensions keeping their + bare-int shorthand; sharding under a rectilinear shard grid is preserved + instead of being silently dropped; and the default `write_data=True` copy + works for every grid kind. `Array.chunks` is now defined for any sharded array + (the inner chunks of a shard are always regular), and for sharded arrays with + a rectilinear shard grid `Array.info` no longer raises — it reports the shard + shape as `` — while `Array.nchunks_initialized` counts the chunks of + each initialized shard individually instead of raising. + + Apart from the nested-sequence input form noted above, everything described + here concerns rectilinear chunk grids, which remain an experimental feature + gated behind `zarr.config.set({"array.rectilinear_chunks": True})`; arrays + with regular chunk grids are unaffected. ([#4218](https://github.com/zarr-developers/zarr-python/pull/4218)) + +- A `scale_offset` codec configured with a string-valued zero scale is now rejected. `scale` accepts strings, and no string is ever equal to `0`, so `"0"`, `"0.0"` and the hex form `"0x0000000000000000"` skipped the "scale must be non-zero" check that the numeric `0` triggers. On float data types the array was created, every chunk was written as zero and read back as `nan` with no error, and the zero scale was persisted to the metadata so reopening the store reproduced it; on integer data types the codec raised `ZeroDivisionError` instead of `ValueError`. The check now runs on the parsed scalar rather than the value as supplied. ([#4279](https://github.com/zarr-developers/zarr-python/pull/4279)) +- Fixed a `ValueError` when setting an orthogonal selection on a sharded array where more than one dimension is indexed by an array. The sharding codec re-derives an indexer from the chunk selection it is handed, which turns such a selection into a coordinate selection addressing the value buffer flat, so the write failed on a shape mismatch. Both partial-encode paths are fixed, so the write works under either codec pipeline. ([#4284](https://github.com/zarr-developers/zarr-python/pull/4284)) +- Fixed integer array indexing with unsigned index dtypes. An unsorted index such as + `np.array([3, 0], dtype="uint8")` spanning more than one chunk raised `IndexError`, because + the order check used `np.diff`, which wraps on unsigned dtypes and misclassified a + descending selection as increasing. Separately, a `uint64` index raised `IndexError` on both + `array[...]` and `array.vindex[...]` — sorted or not — because `uint64` promotes to + `float64` against a signed chunk offset. Index arrays are now cast to `intp`. + + Unsigned indices are bounds-checked before this conversion, so values such as + `np.uint64(2**64 - 1)` are rejected rather than wrapping to a negative index and + reading or overwriting an element at the end of the array. + + Negative-index normalization copies indices before modifying them, preserving + caller-owned arrays and supporting read-only index arrays. Reusing one index + array across axes of different lengths now normalizes each axis independently. ([#4286](https://github.com/zarr-developers/zarr-python/pull/4286)) + +- `zarr.from_array` now defaults to the fill value and the attributes of the source array. Previously both were silently discarded: the array was created with the data type's default scalar and no attributes. + + An explicit `fill_value=None` now selects the data type's default scalar (Zarr format 3) or a null fill value (Zarr format 2), consistently with `create_array`, and an empty `attributes` dict creates the array with no attributes. ([#4288](https://github.com/zarr-developers/zarr-python/pull/4288)) + +- Fixed an infinite loop when creating a 0-dimensional array with `shards="auto"` while the `array.target_shard_size_bytes` config option is set. Such arrays now resolve to `shards=()`, matching the behavior when no shard size target is configured. ([#4305](https://github.com/zarr-developers/zarr-python/pull/4305)) +- Fixed `chunks=-1` on a zero-length axis resolving to an invalid chunk size of 0, which caused a `ValueError`, `ZeroDivisionError`, or infinite loop depending on the sharding configuration. Such axes now get chunk size 1, matching `chunks="auto"`. ([#4307](https://github.com/zarr-developers/zarr-python/pull/4307)) +- Fixed a `ValueError` when setting an orthogonal selection on a sharded array that mixes an integer index with two or more array indices, such as `a.oindex[[3, 1, 2], 1, [0, 2]] = value`. The fix for the array-only case in #4284 reshaped the value only when its shape matched the coordinate selection exactly; the sharding codec now also ravels a value that is the selection's shape minus the integer-indexed axes. Values of any other rank are left alone, so a write that is invalid on an unsharded array fails the same way on a sharded one. Both partial-encode paths share one helper for this. ([#4316](https://github.com/zarr-developers/zarr-python/pull/4316)) +- Opening a Zarr format 3 array whose codec's ``from_dict`` raised a ``KeyError`` with no arguments used to fail with an unrelated ``IndexError: tuple index out of range`` while formatting the error message. Because that ``IndexError`` is not a ``ValueError``, it also escaped the array-then-group fallback in ``zarr.open`` and broke group operations such as ``Group.members()`` and ``"child" in group`` when any child array used such a codec. The ``KeyError`` is now always reported as a ``MetadataValidationError`` naming the codec, with the offending key included only when the ``KeyError`` carried one. ([#4324](https://github.com/zarr-developers/zarr-python/pull/4324)) +- `zarr.from_array` now deep-copies the source array's attributes instead of sharing nested dicts and lists between the source and the new array. Previously, mutating a nested attribute on the new array (for example ``dst.attrs["meta"]["tags"].append(...)``) silently changed the source array's in-memory attributes too. Deeply nested attributes can raise `RecursionError` during the copy even if they can be stored and reopened; the threshold depends on Python's recursion limit and call stack. Pass `attributes={}` to omit inherited attributes. ([#4325](https://github.com/zarr-developers/zarr-python/pull/4325)) +- Fixed `chunks=False` on a zero-length axis resolving to a chunk size of 0, which raised a `ValueError` for Zarr format 3, raised a `ZeroDivisionError` with `shards="auto"`, and silently wrote invalid `chunks` metadata for Zarr format 2. `False` now takes the same path as `chunks=-1`, so such axes get chunk size 1, matching `chunks="auto"`. ([#4328](https://github.com/zarr-developers/zarr-python/pull/4328)) +- `from_array` preserves a Zarr source's explicit data type instead of trying to infer it from its NumPy dtype, allowing variable-length bytes arrays to be copied. ([#4335](https://github.com/zarr-developers/zarr-python/pull/4335)) +- The documentation build and the documentation test suite no longer delete a `data/` directory relative to the current working directory. Two executable docs sessions opened with `shutil.rmtree('data', ignore_errors=True)` to make their examples re-runnable; because executed docs blocks run in the process working directory rather than the docs tree, `mkdocs build -f /mkdocs.yml` or `pytest tests/test_docs.py` started from any directory containing a `data/` folder — a project checkout, or `/` — silently emptied it. The sdist ships `docs/` and `tests/` and `testpaths` collects `docs/user-guide`, so this reached anyone running the shipped test suite, not only contributors. The deletions are gone; the on-disk examples in the quick start, arrays, groups, storage and performance guides now create with `overwrite=True` (or `zarr.save_array(..., mode="w")`), which is also what a reader re-running an example needs, and a new docs test rejects any executed block that calls a filesystem deletion. ([#4339](https://github.com/zarr-developers/zarr-python/pull/4339)) +- Missing Zarr format 3 imagecodecs now name `imagecodecs-zarr` as a known provider, alongside `virtual-tiff` where both packages register the codec. ([#4351](https://github.com/zarr-developers/zarr-python/pull/4351)) +- `LocalStore` now retries the rename that publishes a written file when Windows + reports the destination as transiently busy (`ERROR_ACCESS_DENIED` or + `ERROR_SHARING_VIOLATION`). Replacing a name that was itself replaced moments + earlier intermittently fails this way in a single process, which aborted + otherwise ordinary writes. The retry is bounded to well under a second, is a + no-op off Windows, and never retries the `FileExistsError` that the `exclusive` + path uses to report an existing node. This mitigates but does not fully resolve + #3522: a second process holding the destination open for longer than the retry + budget will still fail. Zarr v2 had the equivalent retry from #698 and it was + not carried over when atomic writes arrived in #3412. ([#4358](https://github.com/zarr-developers/zarr-python/pull/4358)) + +### Improved Documentation + +- Added a Roadmap page to the documentation outlining future plans and intended changes to the library. ([#4149](https://github.com/zarr-developers/zarr-python/pull/4149)) +- Converted remaining reStructuredText-style double-backtick markup to Markdown + single backticks in the docstrings of `zarr.api.asynchronous`, + `zarr.api.synchronous`, `zarr.core.array`, `zarr.registry`, and + `zarr.storage._common`. No functional changes. ([#4193](https://github.com/zarr-developers/zarr-python/pull/4193)) +- Document how to reassign Read the Docs version slugs when publishing a subpackage release. ([#4236](https://github.com/zarr-developers/zarr-python/pull/4236)) +- Added a "Related Projects" page to the documentation listing the companion + packages developed in this repository — `zarr-metadata` and `zarr-indexing` — + and linked it from the landing page. Links to those packages now use the + canonical `https://zarr.readthedocs.io/projects/...` URLs, and each companion + package's documentation links back to the `zarr-python` docs. ([#4247](https://github.com/zarr-developers/zarr-python/pull/4247)) + +### Misc + +- [#4213](https://github.com/zarr-developers/zarr-python/pull/4213), [#4261](https://github.com/zarr-developers/zarr-python/pull/4261), [#4326](https://github.com/zarr-developers/zarr-python/pull/4326), [#4331](https://github.com/zarr-developers/zarr-python/pull/4331) + + ## 3.3.0 (2026-07-30) ### Features From e45e0758b62422c5e9e6895faa986b95d4466f4a Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 16 Sep 2026 00:41:54 +0200 Subject: [PATCH 14/29] ci: keep subpackage pull requests out of generated zarr release notes (#4361) GitHub's "Generate release notes" lists every pull request merged since the previous tag, which in this repository includes the zarr-metadata, zarr-indexing and zarr-http-server work that ships with those packages' own releases. Label pull requests confined to one packages/ directory with that package's name, and exclude those labels in .github/release.yml; dependabot updates get their own section. Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/labeler.yml | 18 ++++++++++++++++++ .github/release.yml | 22 ++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 .github/release.yml diff --git a/.github/labeler.yml b/.github/labeler.yml index 7eb74211ea..482207d6cb 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,3 +2,21 @@ needs release notes: - all: - changed-files: - all-globs-to-all-files: '!changes/*.md' + +# Subpackage labels: a pull request whose changes all live under one package +# directory gets that package's label, which .github/release.yml uses to keep +# subpackage work out of the generated `zarr` release notes. +zarr-metadata: + - all: + - changed-files: + - all-globs-to-all-files: 'packages/zarr-metadata/**' + +zarr-indexing: + - all: + - changed-files: + - all-globs-to-all-files: 'packages/zarr-indexing/**' + +zarr-http-server: + - all: + - changed-files: + - all-globs-to-all-files: 'packages/zarr-http-server/**' diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000000..a66182c50a --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,22 @@ +# Configuration for GitHub's "Generate release notes" button. +# +# The zarr-python repository also hosts the companion packages under packages/ +# (zarr-metadata, zarr-indexing, zarr-http-server), which have their own release +# cadence, tags and changelogs. Pull requests that only touch a subpackage are +# labelled by the labeler workflow (.github/labeler.yml) and excluded here, so a +# `zarr` release lists only the changes that ship in `zarr`. +changelog: + exclude: + labels: + - zarr-metadata + - zarr-indexing + - zarr-http-server + authors: + - pre-commit-ci[bot] + categories: + - title: Dependency updates + labels: + - dependencies + - title: What's Changed + labels: + - "*" From 2a704a72bb8825abe64b74931872a430cd1aa186 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 16 Sep 2026 08:48:47 +0200 Subject: [PATCH 15/29] fix(indexing): make every view executable with its source grid (#4349) * docs(indexing): ground design and integration claims in current behavior Assisted-by: Codex:GPT-6 * docs(indexing): correct reader lazy-array and cache contracts Assisted-by: Codex:GPT-6 * fix(indexing): validate wire boundaries and clarify format contracts Assisted-by: Codex:GPT-6 * fix(indexing): validate selector bounds and shared dependencies Correct mathematical API documentation to match supported coordinate, grid, and chunk projection contracts. Assisted-by: Codex:GPT-6 * docs(indexing): reconcile reader contracts and record audit fixes Assisted-by: Codex:GPT-6 * docs(indexing): reconcile audit with current partition implementation Retain the existing unsigned selector fix and update the unsupported mixed-dependency error assertion for general intersection routing. Assisted-by: Codex:GPT-6 * docs(indexing): clarify planning coverage and benchmark measurement boundaries Assisted-by: Codex:GPT-6 * fix(indexing): group signed chunk coordinates without collisions Use lexicographic tuple grouping when chunk indices contain negative values. Cover shared one-axis and two-axis array dependencies, repeated points, and extreme signed coordinates. Assisted-by: Codex:GPT-6 * docs(indexing): state remaining planner limits precisely Assisted-by: Codex:GPT-6 * feat(indexing): execute partitions with their reader context Assisted-by: Codex:GPT-6 * docs(indexing): number audit changelog entries for PR 4345 Assisted-by: Codex:GPT-6 * docs(indexing): describe current contracts in docstrings Remove implementation history and unsupported historical claims from source and test docstrings. Distinguish immutable coordinate mappings from mutable source values. Assisted-by: Codex:GPT-6 * fix(indexing): plan every view read with its source grid Remove Partition.result and partition-local source windows. Preserve the source grid in derived views and supply projections for every LazyArray reader call, including unpartitioned reads. Assisted-by: Codex:GPT-6 --- .../zarr-indexing/changes/4349.feature.md | 1 + packages/zarr-indexing/docs/api/lazy_array.md | 7 +- packages/zarr-indexing/docs/api/reader.md | 4 +- packages/zarr-indexing/docs/guide/index.md | 6 +- .../src/zarr_indexing/lazy_array.py | 95 +++++-------------- .../zarr-indexing/src/zarr_indexing/reader.py | 5 +- .../zarr-indexing/tests/test_lazy_array.py | 24 +++-- packages/zarr-indexing/tests/test_reader.py | 63 +++++++++++- 8 files changed, 122 insertions(+), 83 deletions(-) create mode 100644 packages/zarr-indexing/changes/4349.feature.md diff --git a/packages/zarr-indexing/changes/4349.feature.md b/packages/zarr-indexing/changes/4349.feature.md new file mode 100644 index 0000000000..7234374987 --- /dev/null +++ b/packages/zarr-indexing/changes/4349.feature.md @@ -0,0 +1 @@ +Make every `LazyArray` read supply a chunk projection, including unpartitioned reads and independently executed partition views. Partition views retain the source grid and full source base shape, so indexing and repartitioning use the same coordinate frame as other views. diff --git a/packages/zarr-indexing/docs/api/lazy_array.md b/packages/zarr-indexing/docs/api/lazy_array.md index 13740981cd..2441966a12 100644 --- a/packages/zarr-indexing/docs/api/lazy_array.md +++ b/packages/zarr-indexing/docs/api/lazy_array.md @@ -21,7 +21,10 @@ coordinates into its raw `Partition.view.array`, including for non-first partitions. `Partition.projection.chunk_transform` intentionally stays local to the selected chunk. During parent materialization (`view.result(parts=parts)`) the reader receives both frames in one `ReadContext`: the public global transform in `context.transform` and the -same local plan in `context.projection`. Direct `part.view.result()` calls -provide no projection; readers requiring it must use parent assembly. +local plan in `context.projection`. Every view retains the source grid and plans +its reads, so `part.view.result()` also supplies both frames. Its projection's +result placement is relative to that part view, rather than the parent output. +Further indexing and repartitioning use the same source-global coordinate frame. +Even `unpartitioned()` reads carry a projection for the single source-wide cell. ::: zarr_indexing.lazy_array diff --git a/packages/zarr-indexing/docs/api/reader.md b/packages/zarr-indexing/docs/api/reader.md index 3944336745..e0eb9d6a07 100644 --- a/packages/zarr-indexing/docs/api/reader.md +++ b/packages/zarr-indexing/docs/api/reader.md @@ -12,7 +12,9 @@ ownership. `Reader.read_into(source, context, out)` receives a `ReadContext` whose `transform` maps zero-origin output-buffer coordinates to global coordinates in `source`, with `context.transform.domain.shape == out.shape`. Its optional -`projection` is the existing plan for a partitioned read. The projection's +`projection` describes one planned read. `LazyArray.result()` always supplies +it, including for partition views and unpartitioned reads. Direct callers of +the reader protocol may omit it when their reader supports that. The projection's `chunk_transform` remains chunk-local, its `cell_transform` describes result placement, and its `chunk_domain` describes the grid cell. The global read transform and the projection's chunk transform deliberately use different diff --git a/packages/zarr-indexing/docs/guide/index.md b/packages/zarr-indexing/docs/guide/index.md index f55794c4c0..011f6f4426 100644 --- a/packages/zarr-indexing/docs/guide/index.md +++ b/packages/zarr-indexing/docs/guide/index.md @@ -390,8 +390,10 @@ Within one `Partition`, the frames divide: `Partition.view.transform` is a different, global transform — it maps the part view directly into the raw wrapped source — while only `Partition.projection.chunk_transform` uses zero-origin chunk-local coordinates. Parent assembly passes both frames to -the reader. Direct `part.view.result()` calls supply the global transform -with `projection=None`. +the reader. Independently scheduled `part.view.result()` calls plan against the +same source grid and supply both frames too. Their result placement is relative +to the part view being read. A partition view can be indexed or repartitioned +like any other view; its base shape remains the full source shape. | Projection field | What its output coordinates mean | | --- | --- | diff --git a/packages/zarr-indexing/src/zarr_indexing/lazy_array.py b/packages/zarr-indexing/src/zarr_indexing/lazy_array.py index 7326fc5256..4f71df35f8 100644 --- a/packages/zarr-indexing/src/zarr_indexing/lazy_array.py +++ b/packages/zarr-indexing/src/zarr_indexing/lazy_array.py @@ -30,9 +30,10 @@ The part view's transform directly addresses its raw wrapped array. The paired projection deliberately retains the chunk-local frame. Parent materialization -passes both in `ReadContext`; calling `part.view.result()` directly uses an -unpartitioned context with `projection=None`. Readers that require the projection -should be used through the parent `result(parts=parts)` path. +passes both in `ReadContext`. Every view retains the source's partitioning and +plans its own reads, including views obtained from partitions. Calling +`part.view.result()` therefore supplies the reader with the selected chunk's +projection as well. Output placement is relative to the view being executed. The partitioning is discovered from the wrapped array at construction — first `read_chunk_sizes` (zarr's clipped per-axis sizes, sharding-aware), then @@ -405,8 +406,8 @@ class Partition: Yielded by [`LazyArray.parts`][zarr_indexing.lazy_array.LazyArray.parts]. The parts of a view tile it exactly and disjointly: assembling every - `view.result()` at its `out_selection` reproduces the whole view's - `result()` for readers supporting contexts without projections. Parts can be + `part.view.result()` at its `out_selection` reproduces the whole view's + `result()`. Each view plans its reads using the retained source grid. Parts can be resolved concurrently when the source and reader permit it. Derived parts retain the same reader object; a shared stateful reader owns synchronization for concurrent calls. @@ -447,8 +448,8 @@ class Partition: view A `LazyArray` covering exactly the cells of the view that live in this box. Its transform directly addresses its raw wrapped `array`; only the - projection's `chunk_transform` is chunk-local. Resolving the view reads - the box once through its selected reader. Named `view` rather than + projection's `chunk_transform` is chunk-local. The view retains the source + grid and reader and can be read or indexed like any other view. Named `view` rather than `array` because `LazyArray.array` is the opposite thing — the raw wrapped source — and the two sat next to each other meaning inverses. out_selection @@ -569,7 +570,7 @@ class LazyArray: [ 8, 10]]) """ - __slots__ = ("_array", "_part_owner", "_parts", "_reader", "_transform", "_window") + __slots__ = ("_array", "_part_owner", "_parts", "_reader", "_transform") def __init__(self, array: _WrappedArray) -> None: """Wrap `array` without reading it; parameters are documented on the class. @@ -590,7 +591,6 @@ def __init__(self, array: _WrappedArray) -> None: ) shape = tuple(int(s) for s in array.shape) self._array = array - self._window: tuple[slice, ...] | None = None self._transform = IndexTransform.from_shape(shape) self._parts = _discover_parts(array, shape) self._reader = basic_reader @@ -611,7 +611,6 @@ def _derive( array: _WrappedArray, transform: IndexTransform, parts: tuple[DimensionGrid, ...] | None, - window: tuple[slice, ...] | None, reader: Reader, ) -> LazyArray: """Build a wrapper sharing `array` but carrying a new transform or partitioning.""" @@ -621,7 +620,6 @@ def _derive( # view's first element is at position 0 whatever it was sliced from. view._transform = transform.translate_domain_to((0,) * transform.input_rank) view._parts = parts - view._window = window view._reader = reader view._part_owner = _PartOwner() return view @@ -629,9 +627,7 @@ def _derive( @property def _base_shape(self) -> tuple[int, ...]: """The shape of what this wrapper treats as its base array.""" - if self._window is None: - return tuple(int(s) for s in self._array.shape) - return tuple(s.stop - s.start for s in self._window) + return tuple(int(s) for s in self._array.shape) # -- array-like surface ------------------------------------------------- @@ -645,11 +641,8 @@ def base_shape(self) -> tuple[int, ...]: """The shape the partitioning is expressed in — not this view's shape. `with_parts` and `with_parts_per_axis` describe boxes of the array being - read, not of the view reading it, so a narrowed view still partitions - the extents named here. For a part's own `array`, this is the part's - box, which is why the same call means different sizes there. Without - somewhere to read it, the frame in force could only be inferred from an - error message. + read, not of the view reading it. All derived views, including partition + views, retain the full source shape as their partitioning frame. """ return self._base_shape @@ -691,7 +684,6 @@ def with_reader(self, reader: Reader) -> LazyArray: self._array, self._transform, self._parts, - self._window, reader, ) @@ -920,9 +912,8 @@ def _part_entries(parts: Sequence[Any], method: str) -> tuple[Any, ...]: def unpartitioned(self) -> LazyArray: """Return the same view, read in one pass. - `result()` still allocates its owned output buffer first, then calls the - reader once with the whole projected transform. `parts()` still yields a - single part covering everything. + The source is treated as a single grid cell. Nonempty reads still pass + its projection to the reader; empty reads make no reader calls. Returns ------- @@ -932,7 +923,7 @@ def unpartitioned(self) -> LazyArray: return self._with_grids(None) def _with_grids(self, grids: tuple[DimensionGrid, ...] | None) -> LazyArray: - return LazyArray._derive(self._array, self._transform, grids, self._window, self._reader) + return LazyArray._derive(self._array, self._transform, grids, self._reader) def parts(self) -> Iterator[Partition]: """Iterate the base partitioning, projected through this view. @@ -943,8 +934,8 @@ def parts(self) -> Iterator[Partition]: Yields one [`Partition`][zarr_indexing.lazy_array.Partition] per box the view actually touches. The parts tile the view exactly and disjointly, - and each carries a `LazyArray` that can be resolved on its own: in - another thread, in another order, or not at all. Those views share this + and each view can be resolved with `part.view.result()`: in another thread, + in another order, or not at all. Those views share this view's reader, and `LazyArray` does not serialize calls, so a stateful reader must synchronize its own mutable state. @@ -964,48 +955,16 @@ def parts(self) -> Iterator[Partition]: >>> (part.base_coords, part.view.shape, part.is_complete) ((0, 0), (2, 1), False) """ - base_shape = self._base_shape - grids = self._parts if self._parts is not None else _whole_array_grids(base_shape) - rank = len(base_shape) - - if self._window is None: - plan_transform = self._transform - else: - plan_transform = self._transform.translate(tuple(-item.start for item in self._window)) - - for projection in plan_chunks(plan_transform, grids): - base_coords = projection.chunk_coords - local = projection.chunk_transform - origin = tuple(grid.chunk_offset(c) for grid, c in zip(grids, base_coords, strict=True)) - extent = tuple(grid.data_size(c) for grid, c in zip(grids, base_coords, strict=True)) - if origin == (0,) * rank and extent == base_shape: - # The part is the whole base: lowering directly against the - # source beats materializing a block that is the source. - window = self._window - elif self._window is None: - window = tuple(slice(o, o + e) for o, e in zip(origin, extent, strict=True)) - else: - window = tuple( - slice(w.start + o, w.start + o + e) - for w, o, e in zip(self._window, origin, extent, strict=True) - ) - # The global box, computed from the origin directly rather than from - # `window`: a part covering the whole base carries no window (so - # nothing is pre-materialized) but still sits somewhere concrete. - if self._window is None: - global_origin = origin - else: - global_origin = tuple( - w.start + o for w, o in zip(self._window, origin, strict=True) - ) + grids = self._parts if self._parts is not None else _whole_array_grids(self._base_shape) + for projection in plan_chunks(self._transform, grids): + domain = projection.chunk_domain yield Partition( projection=projection, - box=tuple((o, o + e) for o, e in zip(global_origin, extent, strict=True)), + box=tuple(zip(domain.inclusive_min, domain.exclusive_max, strict=True)), view=LazyArray._derive( self._array, - local.translate(global_origin), - None, - window, + projection.chunk_transform.translate(domain.inclusive_min), + self._parts, self._reader, ), out_selection=_partition_out_selection(projection.cell_transform), @@ -1039,7 +998,7 @@ def _select(self, selection: Any, mode: SelectionMode) -> LazyArray: composed = transform[literal] else: composed = transform.select(literal, mode) - return LazyArray._derive(self._array, composed, self._parts, self._window, self._reader) + return LazyArray._derive(self._array, composed, self._parts, self._reader) def __getitem__(self, selection: Any) -> Any: """Read a basic selection eagerly, like `numpy.ndarray.__getitem__`. @@ -1057,6 +1016,8 @@ def result(self, *, parts: Sequence[Partition] | None = None) -> Any: partition is read through the selected reader directly into its rectangular destination, or into an owned dense temporary before fancy placement. Empty views allocate without reading the source. + Every reader call includes a chunk projection, even when the source is + treated as a single grid cell by `unpartitioned()`. Parameters ---------- @@ -1099,10 +1060,6 @@ def result(self, *, parts: Sequence[Partition] | None = None) -> Any: if size == 0: return out - if prepared_parts is None and self._parts is None: - _invoke_reader(self._reader, self._array, ReadContext(self._transform), out) - return out - written = 0 selected_parts = self.parts() if prepared_parts is None else prepared_parts for part in selected_parts: diff --git a/packages/zarr-indexing/src/zarr_indexing/reader.py b/packages/zarr-indexing/src/zarr_indexing/reader.py index 2527972e02..bfb388b3e7 100644 --- a/packages/zarr-indexing/src/zarr_indexing/reader.py +++ b/packages/zarr-indexing/src/zarr_indexing/reader.py @@ -48,7 +48,10 @@ class ReadContext: """Maps zero-origin output-buffer coordinates to global coordinates in the source.""" projection: ChunkProjection | None = None - """The partition plan when this read is one part of a partitioned view, else `None`.""" + """The read plan, always supplied by `LazyArray` execution. + + Direct reader callers may omit it if their reader supports unplanned reads. + """ class Reader(Protocol): diff --git a/packages/zarr-indexing/tests/test_lazy_array.py b/packages/zarr-indexing/tests/test_lazy_array.py index 5bb55ae076..db4483f750 100644 --- a/packages/zarr-indexing/tests/test_lazy_array.py +++ b/packages/zarr-indexing/tests/test_lazy_array.py @@ -1212,7 +1212,8 @@ def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: outer_call, inner_call = events assert outer_call[1] is inner_call[1] is data assert outer_call[2] is inner_call[2] - assert outer_call[3] is inner_call[3] is result + assert outer_call[3] is inner_call[3] + assert np.shares_memory(outer_call[3], result) np.testing.assert_array_equal(result, data[1:6:2, ::-1, 1]) @@ -1259,10 +1260,12 @@ def read_into(self, source: Any, context: ReadContext, out: Any, /) -> Any: return np.empty(context.transform.domain.shape, dtype=source.dtype) -def test_result_rejects_a_reader_that_returns_a_value() -> None: +@pytest.mark.parametrize("independent", [False, True]) +def test_result_rejects_a_reader_that_returns_a_value(independent: bool) -> None: view = LazyArray(reference()).with_reader(ReturningReader()) + target = next(view.parts()).view if independent else view with pytest.raises(TypeError, match="must return None"): - view.result() + target.result() class BufferRecordingReader(RecordingReader): @@ -1325,15 +1328,18 @@ def test_fancy_part_placement_uses_owned_dense_temporaries() -> None: assert any(reader.owns_data) -def test_reader_exception_propagates_unchanged() -> None: +@pytest.mark.parametrize("independent", [False, True]) +def test_reader_exception_propagates_unchanged(independent: bool) -> None: error = RuntimeError("backend failed") class FailingReader: def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: raise error + view = LazyArray(reference()).with_reader(FailingReader()) + target = next(view.parts()).view if independent else view with pytest.raises(RuntimeError) as caught: - LazyArray(reference()).with_reader(FailingReader()).result() + target.result() assert caught.value is error @@ -2421,11 +2427,17 @@ def test_numpy_matrix_is_refused() -> None: @pytest.mark.parametrize("parts", [None, (2, 2), (1, 4), (3, 4)]) def test_a_masked_source_keeps_its_mask_under_every_partitioning(parts: Any) -> None: data = np.ma.masked_greater(np.arange(12).reshape(3, 4), 7) - got = repartition(LazyArray(data), parts).lazy[:, 1:].result() + view = repartition(LazyArray(data), parts).lazy[:, 1:] + got = view.result() expected = data[:, 1:] assert isinstance(got, np.ma.MaskedArray), parts np.testing.assert_array_equal(np.ma.getmaskarray(got), np.ma.getmaskarray(expected)) np.testing.assert_array_equal(np.ma.filled(got, 0), np.ma.filled(expected, 0)) + assembled = np.ma.masked_all(view.shape, dtype=view.dtype) + for part in view.parts(): + assembled[part.out_selection] = part.view.result() + np.testing.assert_array_equal(np.ma.getmaskarray(assembled), np.ma.getmaskarray(expected)) + np.testing.assert_array_equal(np.ma.filled(assembled, 0), np.ma.filled(expected, 0)) @pytest.mark.parametrize("parts", [None, (2, 2), (3, 4)]) diff --git a/packages/zarr-indexing/tests/test_reader.py b/packages/zarr-indexing/tests/test_reader.py index 48e642aeaf..2bf426753b 100644 --- a/packages/zarr-indexing/tests/test_reader.py +++ b/packages/zarr-indexing/tests/test_reader.py @@ -210,8 +210,67 @@ def test_successful_transform_contract_across_planning_readers_and_lazy_array( np.testing.assert_array_equal(out, expected_values) if lazy_selection is not None: - view = lazy_selection(LazyArray.from_numpy(source_data).with_parts(chunk_shape)) - np.testing.assert_array_equal(view.result(), expected_values) + reader = ProjectionRequiredReader() + view = lazy_selection( + LazyArray.from_numpy(source_data).with_parts(chunk_shape).with_reader(reader) + ) + parts = tuple(view.parts()) + np.testing.assert_array_equal(view.result(parts=parts), expected_values) + parent_contexts = tuple(reader.contexts) + reader.contexts.clear() + assembled = np.empty(view.shape, dtype=view.dtype) + for part in reversed(parts): + values = part.view.result() + assert not np.shares_memory(values, source_data) + assert values.dtype == expected_values.dtype + assembled[part.out_selection] = values + np.testing.assert_array_equal(assembled, expected_values) + for independent, parent in zip(reversed(reader.contexts), parent_contexts, strict=True): + assert independent.transform == parent.transform + assert independent.projection is not None + assert parent.projection is not None + assert independent.projection.chunk_coords == parent.projection.chunk_coords + assert independent.projection.chunk_domain == parent.projection.chunk_domain + assert independent.projection.chunk_transform == parent.projection.chunk_transform + for candidate in (view, view.unpartitioned()): + np.testing.assert_array_equal(candidate.result(), expected_values) + for part in parts: + assert part.view.base_shape == source_data.shape + for child in part.view.parts(): + np.testing.assert_array_equal(child.view.result(), part.view.result()) + reverse = (slice(None, None, -1),) * part.view.ndim + np.testing.assert_array_equal( + part.view.lazy[reverse].result(), part.view.result()[reverse] + ) + for candidate in ( + part.view.unpartitioned(), + part.view.with_parts((1,) * source_data.ndim), + ): + np.testing.assert_array_equal(candidate.result(), part.view.result()) + + +class ProjectionRequiredReader: + """Read through chunk-local coordinates so losing the projection breaks execution.""" + + def __init__(self) -> None: + self.contexts: list[ReadContext] = [] + + def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: + projection = context.projection + assert projection is not None + self.contexts.append(context) + for position, cell in zip( + np.ndindex(out.shape), + _domain_coordinates(projection.chunk_transform.domain), + strict=True, + ): + local = projection.chunk_transform.apply(cell) + global_position = tuple( + coord + origin + for coord, origin in zip(local, projection.chunk_domain.inclusive_min, strict=True) + ) + assert context.transform.apply(position) == global_position + out[position] = source[global_position] def _domain_coordinates(domain: IndexDomain) -> list[tuple[int, ...]]: From b6c592e3018468eff39a91f269ce17e2a65e0d55 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 16 Sep 2026 11:45:43 +0200 Subject: [PATCH 16/29] feat(indexing): make LazyArray indexing lazy by default (#4350) * feat(indexing): make LazyArray indexing lazy by default Add synchronous writes through composed selections and an explicit eager adapter for array consumers. Cover deferred reads, source mutation, masks, aliases, and repeated destinations. Assisted-by: Codex:GPT-6 Rebased-onto: upstream/main after #4345-#4349 merged; conflicts resolved with ClaudeCode:claude-fable-5-1 * fix(indexing): plan fancy writes against the source's write grid The non-affine write fallback assigned one element at a time, which on a chunked source is one chunk read-modify-write per element: a 50-row orthogonal write into a 1000x1000 zarr array with 100x100 chunks cost 49,999 chunk writes and 27 s against 100 writes and 10 ms natively. `write_into` now scatters in bulk. NumPy sources get one fancy assignment. A readable source with a write grid, which `LazyArray.write` discovers from `write_chunk_sizes` then `chunks`, is written one cell at a time: the cell's touched hull is read once, updated in memory with last-occurrence-wins semantics, and written back with one basic slice. The same write into zarr now costs 100 chunk writes. Sources with no grid, transforms the planner cannot factor, and sources that cannot be read keep the per-element path, which never reads. Also: order the rank check before the value copy while still validating values for an empty selection; keep the payload of a masked zero-rank affine write, which `np.flip` with no axes was turning into the `masked` singleton; note that writes bypass the reader and pin that with a test; note that iterated elements are views; add API pages for the writer and eager adapter; and describe the shipped indexing surface in the unreleased `LazyArray` changelog fragment instead of its removed `.lazy` accessor. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * feat(indexing): keep literal domains on views, positional keys unchanged `LazyArray` re-zeroed the domain of every derived view, so a view forgot which coordinates it was cut from and placement needed a side channel. TensorStore keeps literal coordinates on views; NumPy users expect positional keys. These are separable: the key dialect says how a key is read, the domain says what the view remembers. Views now keep their literal domain: `a[10:20]` has domain `[10, 20)` and `a[10:20][2:5]` has `[12, 15)`, while `a[10:20][0]` is still the first element because positional keys are normalized against the domain's origin, which `normalize_positional_selection` already did. Two literal keys join the NumPy ones, as in TensorStore's `__getitem__`: an `IndexDomain` restricts the view to coordinates of its own domain (empty intervals outside it are refused too) and an `IndexTransform` composes onto it. Box partitions keep the request's coordinates, so a part view's domain is a sub-domain of its parent's; a part placed by index arrays keeps a fresh zero-origin domain, and `out_selection` is the placement in both cases. A reversed view shows the negative origin the algebra already produced. Two frames stay zero-origin by construction so no consumer has to subtract an origin: `ReadContext` re-bases its transform, and `parts()` re-bases each projection's `cell_transform` from the request's literal domain to positions in the view's result buffer, which is what `Partition.projection` has always documented. A table-driven test states the expected literal domain for every indexing form and nested chain, checked against the transform algebra. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(indexing): state that a key's type picks relative or absolute indexing Make the one rule about indexing a LazyArray prominent: NumPy keys are positions relative to the view, an `IndexDomain` key names absolute coordinates of the view's domain, an `IndexTransform` key composes onto it, no key type has two readings, and every view keeps its absolute domain whichever key produced it. A guide section with a runnable snippet carries the table and the pandas `ix` / TensorStore comparison; the module docstring, README, and API page point at it. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- packages/zarr-indexing/README.md | 7 +- .../zarr-indexing/changes/4222.feature.2.md | 16 +- .../zarr-indexing/changes/4350.feature.md | 20 + packages/zarr-indexing/docs/api/eager.md | 12 + packages/zarr-indexing/docs/api/index.md | 2 +- packages/zarr-indexing/docs/api/lazy_array.md | 23 +- packages/zarr-indexing/docs/api/reader.md | 7 +- packages/zarr-indexing/docs/api/writer.md | 16 + packages/zarr-indexing/docs/design-notes.md | 13 +- packages/zarr-indexing/docs/guide/index.md | 61 ++- .../zarr-indexing/docs/guide/integrations.md | 6 + packages/zarr-indexing/docs/guide/patterns.md | 4 +- packages/zarr-indexing/docs/index.md | 2 +- .../docs/snippets/axis_manipulation.py | 6 +- .../docs/snippets/canonical_slice.py | 4 +- .../docs/snippets/chunk_projection.py | 2 +- .../docs/snippets/coordinate_origins.py | 2 +- .../docs/snippets/integrations.py | 8 +- .../zarr-indexing/docs/snippets/key_types.py | 30 ++ .../docs/snippets/lazy_composition.py | 4 +- .../examples/lazy_indexing_dask/README.md | 13 +- .../lazy_indexing_dask/lazy_indexing_dask.py | 28 +- .../examples/lazy_indexing_numpy/README.md | 6 +- .../lazy_indexing_numpy.py | 45 +- .../system_memory_chunk_cache.py | 2 +- packages/zarr-indexing/mkdocs.yml | 2 + .../src/zarr_indexing/__init__.py | 4 +- .../src/zarr_indexing/boundary.py | 8 +- .../zarr-indexing/src/zarr_indexing/eager.py | 57 +++ .../zarr-indexing/src/zarr_indexing/errors.py | 2 +- .../src/zarr_indexing/lazy_array.py | 296 +++++++---- .../zarr-indexing/src/zarr_indexing/reader.py | 17 +- .../src/zarr_indexing/testing/stateful.py | 6 +- .../src/zarr_indexing/transform.py | 2 +- .../zarr-indexing/src/zarr_indexing/writer.py | 225 +++++++++ .../tests/test_chunk_resolution.py | 2 +- .../zarr-indexing/tests/test_doc_examples.py | 18 +- .../zarr-indexing/tests/test_lazy_array.py | 447 +++++++++-------- .../zarr-indexing/tests/test_lazy_default.py | 213 ++++++++ packages/zarr-indexing/tests/test_reader.py | 34 +- .../zarr-indexing/tests/test_transform.py | 12 +- packages/zarr-indexing/tests/test_writer.py | 462 ++++++++++++++++++ 42 files changed, 1689 insertions(+), 457 deletions(-) create mode 100644 packages/zarr-indexing/changes/4350.feature.md create mode 100644 packages/zarr-indexing/docs/api/eager.md create mode 100644 packages/zarr-indexing/docs/api/writer.md create mode 100644 packages/zarr-indexing/docs/snippets/key_types.py create mode 100644 packages/zarr-indexing/src/zarr_indexing/eager.py create mode 100644 packages/zarr-indexing/src/zarr_indexing/writer.py create mode 100644 packages/zarr-indexing/tests/test_lazy_default.py create mode 100644 packages/zarr-indexing/tests/test_writer.py diff --git a/packages/zarr-indexing/README.md b/packages/zarr-indexing/README.md index 2992dcf280..2d75402664 100644 --- a/packages/zarr-indexing/README.md +++ b/packages/zarr-indexing/README.md @@ -11,8 +11,11 @@ materializes them on request; the transform algebra itself performs no source I/ Key types: -- `LazyArray` — wraps a system-memory/basic-indexing source and adds a `.lazy` - accessor: `LazyArray.from_numpy(numpy_array).lazy[10:50, ::2].lazy.oindex[[3, 1, 1], :]` +- `LazyArray` — wraps a system-memory/basic-indexing source with lazy indexing: + `LazyArray.from_numpy(numpy_array)[10:50, ::2].oindex[[3, 1, 1], :]`. + The key's type picks the frame: NumPy keys are positions relative to the + view, while an `IndexDomain` or `IndexTransform` key addresses the view's + absolute domain, which every view keeps. composes a transform and returns a new view without reading data, and `result()` materializes it into owned system memory. `LazyArray(source)` uses the basic reader; `from_numpy` selects `numpy_reader`, which currently uses diff --git a/packages/zarr-indexing/changes/4222.feature.2.md b/packages/zarr-indexing/changes/4222.feature.2.md index 722f56db99..d89fb98134 100644 --- a/packages/zarr-indexing/changes/4222.feature.2.md +++ b/packages/zarr-indexing/changes/4222.feature.2.md @@ -1,13 +1,13 @@ Added `LazyArray`, which grafts the full NumPy indexing dialect onto any source exposing `shape`, `dtype`, and basic integer/slice `__getitem__` — a chunked -store, an FFI binding, an HTTP endpoint. `view.lazy[...]`, `.lazy.oindex[...]` -and `.lazy.vindex[...]` each compose an `IndexTransform` and return a new view -without reading anything; `result()` materializes. Selections use positional -NumPy semantics (negatives wrap, scalars drop their axis, coordinate arrays keep -order and duplicates), which the new `zarr_indexing.boundary` module translates -into the algebra's literal coordinates. The wrapper describes reads only, and -behaves as a duck array: eager `__getitem__` and `__array__` make it a -`dask.array.from_array` source. +store, an FFI binding, an HTTP endpoint. `view[...]`, `view.oindex[...]` and +`view.vindex[...]` each compose an `IndexTransform` and return a new view +without reading anything; `result()` and `numpy.asarray(view)` materialize. +Selections use positional NumPy semantics (negatives wrap, scalars drop their +axis, coordinate arrays keep order and duplicates), which the new +`zarr_indexing.boundary` module translates into the algebra's literal +coordinates. Consumers that need indexing to produce data, such as +`dask.array.from_array`, wrap a view in `EagerArrayAdapter`. A read is divided along a **partitioning** — discovered from the wrapped array, or chosen with `with_parts` / `with_parts_per_axis` / `unpartitioned`. diff --git a/packages/zarr-indexing/changes/4350.feature.md b/packages/zarr-indexing/changes/4350.feature.md new file mode 100644 index 0000000000..7b740e9712 --- /dev/null +++ b/packages/zarr-indexing/changes/4350.feature.md @@ -0,0 +1,20 @@ +Make `LazyArray` indexing lazy by default: use `view[...]`, `view.oindex[...]`, +and `view.vindex[...]` directly, without a `.lazy` accessor. Iteration yields +lazy views, so arithmetic over iterated elements no longer works on values; +`result()` and NumPy conversion materialize. Add synchronous `write(values)` +and assignment through composed views, and an explicit `EagerArrayAdapter` for +consumers such as Dask that require eager indexing. + +Writes are planned against the source's write grid, discovered from +`write_chunk_sizes` or `chunks`: an affine selection is one basic assignment, +and any other selection reads, updates, and rewrites each touched cell once, +so storage round trips scale with touched chunks rather than selected +elements. A source with no advertised grid is written one element at a time. +Writes bypass the reader, so a caching reader is not invalidated. + +Views keep their literal domain instead of re-zeroing after every selection: +`a[10:20]` has domain `[10, 20)` and `a[10:20][2:5]` has domain `[12, 15)`, +as in TensorStore, while NumPy keys stay positional. An `IndexDomain` key +restricts a view to literal coordinates and an `IndexTransform` key composes +onto it. Box partitions keep the request's coordinates, so a part view's +domain is a sub-domain of its parent's. diff --git a/packages/zarr-indexing/docs/api/eager.md b/packages/zarr-indexing/docs/api/eager.md new file mode 100644 index 0000000000..e3b10998d8 --- /dev/null +++ b/packages/zarr-indexing/docs/api/eager.md @@ -0,0 +1,12 @@ +--- +title: Eager adapter +--- + +# Eager adapter + +`LazyArray` indexing returns views. Consumers that require indexing to +produce data, such as `dask.array.from_array`, wrap a view in +`EagerArrayAdapter`: the adapter reads each indexed block through the view's +reader and partitioning, while the view itself keeps lazy indexing. + +::: zarr_indexing.eager diff --git a/packages/zarr-indexing/docs/api/index.md b/packages/zarr-indexing/docs/api/index.md index 8b7a5d6515..a5c5498e8d 100644 --- a/packages/zarr-indexing/docs/api/index.md +++ b/packages/zarr-indexing/docs/api/index.md @@ -48,7 +48,7 @@ and the wire format built on top of it. **Lazy arrays** - [`zarr_indexing.lazy_array`](lazy_array.md) — `LazyArray`, a wrapper for - system-memory/basic-indexing sources that adds a `.lazy` accessor for + system-memory/basic-indexing sources with lazy indexing for TensorStore-style deferred indexing, plus `Partition` and `parts()` / `with_parts()`, which determine the boxes a read is broken into. Device sources require an explicit custom reader that transfers into the supplied diff --git a/packages/zarr-indexing/docs/api/lazy_array.md b/packages/zarr-indexing/docs/api/lazy_array.md index 2441966a12..44175476ab 100644 --- a/packages/zarr-indexing/docs/api/lazy_array.md +++ b/packages/zarr-indexing/docs/api/lazy_array.md @@ -2,7 +2,7 @@ title: lazy_array --- -`LazyArray.lazy[...]` is metadata-only: every derived view keeps the same +`LazyArray[...]` is metadata-only: every derived view keeps the same reader and composes its transform without reading data. `result()` allocates owned system memory, then calls that reader once for each projected part. Rectangular parts write directly into their final slices; advanced placement @@ -16,9 +16,14 @@ conversion needs a custom reader that transfers into the output buffer. Derived reader and part views may be materialized concurrently, so stateful readers must synchronize their own mutable state. -Every public `Partition.view.transform` directly maps that view's zero-origin +The key's type picks the frame: NumPy keys are relative positions, `IndexDomain` +and `IndexTransform` keys are absolute, and every view keeps its absolute +domain. See [the guide](../guide/index.md#the-keys-type-picks-the-frame). + +Every public `Partition.view.transform` directly maps that view's own domain coordinates into its raw `Partition.view.array`, including for non-first -partitions. `Partition.projection.chunk_transform` intentionally stays local to +partitions; a box part's domain is a sub-domain of the parent view's literal +domain. `Partition.projection.chunk_transform` intentionally stays local to the selected chunk. During parent materialization (`view.result(parts=parts)`) the reader receives both frames in one `ReadContext`: the public global transform in `context.transform` and the local plan in `context.projection`. Every view retains the source grid and plans @@ -27,4 +32,16 @@ result placement is relative to that part view, rather than the parent output. Further indexing and repartitioning use the same source-global coordinate frame. Even `unpartitioned()` reads carry a projection for the single source-wide cell. +`view[key]`, `view.oindex[key]`, and `view.vindex[key]` return lazy views, +and iteration yields lazy first-axis views. `result()` and `numpy.asarray(view)` +materialize values. `view.write(values)` synchronously writes to the original +source through the composed transform and returns `None`; `view[key] = values` +writes a selected sub-view. Writes require a writable source. + +For consumers requiring eager indexing, import `EagerArrayAdapter` from +`zarr_indexing` and wrap the view. The adapter delegates shape, rank, dtype, +NumPy conversion, and tokenization to the view, but materializes each +`adapter[key]`. Use it with `dask.array.from_array`; direct lazy indexing +is not a reliable Dask block-read interface. + ::: zarr_indexing.lazy_array diff --git a/packages/zarr-indexing/docs/api/reader.md b/packages/zarr-indexing/docs/api/reader.md index e0eb9d6a07..6cb49afacd 100644 --- a/packages/zarr-indexing/docs/api/reader.md +++ b/packages/zarr-indexing/docs/api/reader.md @@ -11,11 +11,14 @@ ownership. `Reader.read_into(source, context, out)` receives a `ReadContext` whose `transform` maps zero-origin output-buffer coordinates to global coordinates in -`source`, with `context.transform.domain.shape == out.shape`. Its optional +`source`, with `context.transform.domain.shape == out.shape`. A view's +transform keeps its literal domain; `ReadContext` re-bases it to origin zero on +construction, so readers never see a view's coordinates. Its optional `projection` describes one planned read. `LazyArray.result()` always supplies it, including for partition views and unpartitioned reads. Direct callers of the reader protocol may omit it when their reader supports that. The projection's -`chunk_transform` remains chunk-local, its `cell_transform` describes result +`chunk_transform` remains chunk-local, its `cell_transform` places cells in the +zero-origin result buffer of the view that planned the read, which is what result placement, and its `chunk_domain` describes the grid cell. The global read transform and the projection's chunk transform deliberately use different coordinate frames. diff --git a/packages/zarr-indexing/docs/api/writer.md b/packages/zarr-indexing/docs/api/writer.md new file mode 100644 index 0000000000..a8eb365954 --- /dev/null +++ b/packages/zarr-indexing/docs/api/writer.md @@ -0,0 +1,16 @@ +--- +title: Writers +--- + +# Writers + +`LazyArray.write(values)` and assignment through a view are implemented by +`write_into`, which writes through a transform using only basic integer/slice +assignment on the source. Independent affine selections become one basic +assignment. Other selections are scattered against the source's write grid: +each touched cell is read once, updated in memory, and written back, so +storage round trips are bounded by touched cells rather than selected +elements. NumPy sources receive one fancy assignment instead, and a source +that advertises no grid is written one element at a time without reading. + +::: zarr_indexing.writer diff --git a/packages/zarr-indexing/docs/design-notes.md b/packages/zarr-indexing/docs/design-notes.md index cb80e5ba19..9efed99769 100644 --- a/packages/zarr-indexing/docs/design-notes.md +++ b/packages/zarr-indexing/docs/design-notes.md @@ -157,13 +157,13 @@ arr = zarr.create_array({}, shape=(100, 80), chunks=(30, 40), dtype="int32") arr[:] = np.arange(8000).reshape(100, 80) lazy = LazyArray(arr) -slab = lazy.lazy[10:50, ::4] +slab = lazy[10:50, ::4] slab.is_box # True slab.bounding_box() # ((10, 50), (0, 77)) slab.strides() # (1, 4) slab.shape # (40, 20) -gather = lazy.lazy.oindex[[90, 3, 3], :] +gather = lazy.oindex[[90, 3, 3], :] gather.is_box # False gather.bounding_box() # ((3, 91), (0, 80)) gather.strides() # None @@ -263,14 +263,15 @@ than a silently empty selection. One consequence: a negative step normally produces a negative domain origin. Reversing a length-20 zero-origin axis gives the domain `[-19, 1)`, because the result stays anchored to the source coordinate frame and a reversing map traverses that frame backwards. `LazyArray` -re-bases every view to origin 0, so the positional dialect never exposes it; a -caller working with `IndexTransform` directly will see it, and re-bases -explicitly with `translate_domain_to` for NumPy-shaped coordinates. +views keep that literal domain, so `view.transform.domain` shows it; positional +keys are normalized against the domain's origin, so the NumPy dialect never +requires typing it. A caller wanting zero-origin coordinates re-bases +explicitly with `translate_domain_to`. Supported fancy selections compose across already-fancy views: a second `oindex`/`vindex`/mask step may land on any axis of an already-fancy view, including axes an existing index array merely broadcasts along, so -`lazy.oindex[[2, 0], :].lazy.oindex[:, [1, 3]]` selects the outer product it +`lazy.oindex[[2, 0], :].oindex[:, [1, 3]]` selects the outer product it spells. An array-carrying transform is composed — the new selection is applied to an identity transform over the current domain and chained on with `compose`, which evaluates the existing lookup tables at the new coordinates — rather than diff --git a/packages/zarr-indexing/docs/guide/index.md b/packages/zarr-indexing/docs/guide/index.md index 011f6f4426..f227ca1ac4 100644 --- a/packages/zarr-indexing/docs/guide/index.md +++ b/packages/zarr-indexing/docs/guide/index.md @@ -1,6 +1,6 @@ # Visual guide -The whole model in one sentence: indexing through `LazyArray.lazy` builds a +The whole model in one sentence: indexing through `LazyArray` builds a view, chunk planning partitions its coordinates, and `result()` materializes the view. This page follows one familiar NumPy selection, `source[2:5]`, through those stages. @@ -38,7 +38,7 @@ make the correspondence explicit: those result coordinates receive values `12`, `13`, and `14` from source coordinates `2`, `3`, and `4`. The wrapper below gives the same familiar selection a lazy spelling. Indexing -through `.lazy` creates `view`; the last line asks for its values and checks the +with `[...]` creates `view`; the last line asks for its values and checks the observable NumPy result. ```python @@ -105,11 +105,40 @@ different questions: | Surface | Meaning of an integer index | Meaning of `-1` | | --- | --- | --- | | `IndexDomain` and `IndexTransform` | A literal coordinate in the current domain | The actual address `-1`, if the domain contains it | -| `LazyArray.lazy` | A NumPy-style position in the current view | The last position, normalized before it reaches the transform algebra | +| `LazyArray` with a NumPy key | A NumPy-style position in the current view | The last position, normalized against the domain's origin before it reaches the transform algebra | +| `LazyArray` with an `IndexDomain` or `IndexTransform` key | A literal coordinate in the view's domain | The actual address `-1`, if the domain contains it | -`LazyArray` uses positions because it is an array-like wrapper: each derived -view starts at position zero and negative indices wrap exactly as they do in -NumPy. The lower-level domain and transform types keep literal coordinates. +`LazyArray` reads NumPy keys as positions because it is an array-like wrapper: +negative indices wrap exactly as they do in NumPy. The view itself keeps its +literal domain, as a TensorStore view does: `source[10:20]` has domain +`[10, 20)`, and a further `[2:5]` on it has domain `[12, 15)`. A domain or +transform key addresses those literal coordinates directly. + +### The key's type picks the frame {#the-keys-type-picks-the-frame} + +**The type of a key decides whether it is relative or absolute.** This is the +one rule to remember about indexing a `LazyArray`: + +| Key type | Read as | Example | +| --- | --- | --- | +| slice, integer, `...`, `None`, index array, mask | **Relative**: positions in the current view, NumPy-style | `view[2:5]`, `view[-1]`, `view.oindex[[3, 1]]` | +| `IndexDomain` | **Absolute**: coordinates of the view's domain, restricted to a box | `view[IndexDomain((12,), (15,))]` | +| `IndexTransform` | **Absolute**: composed onto the view; the key's domain becomes the new view's domain | `view[IndexTransform.identity(view.transform.domain)[12:15]]` | + +No key type is valid in both readings, so a key never has two meanings. That is +the property pandas lost with value-based `ix` dispatch and removed in favor of +`loc` and `iloc`; TensorStore keeps it by giving each key type one fixed +reading, and so does this wrapper. The only difference from TensorStore is +which reading the NumPy key gets: absolute there, relative here. + +Whichever key produced a view, **its domain is always absolute**: +`view.transform.domain` reports literal coordinates after `view[2:5]` exactly as +it does after `view[IndexDomain(...)]`. The relative reading exists only at the +moment a NumPy key is interpreted, so relative and absolute steps compose freely. + +```python +--8<-- "snippets/key_types.py:key-types" +``` ### A transform points from the request to the source @@ -209,18 +238,17 @@ metadata is ready to inspect: | Available without reading | Value in this example | | --- | --- | | `composed.shape` | `(2,)` | -| `composed.transform` | One transform mapping request `i` to source `3 - i` | +| `composed.transform` | One transform over the literal domain `[-3, -1)`, mapping request `i` to source `-i` | Neither property needs source values. Composition works only on the coordinate description; the assertion's call to `result()` is the first operation in the example that materializes the selected data. !!! warning "Stop here: the materialization boundary" - Indexing through `.lazy[...]` composes a selection without reading source values. + Indexing through `[...]` composes a selection without reading source values. These operations request values: - `result()` - - eager indexing of the wrapper: `view[...]` - `numpy.asarray(view)` and NumPy operations that convert the view (`numpy.add(view, 1)` does so; `numpy.shape(view)` and `numpy.ndim(view)` can use metadata without reading values) @@ -231,11 +259,14 @@ example that materializes the selected data. Python arithmetic such as `view + 1` raises `TypeError` instead: this wrapper defers indexing, not a general compute graph. - Nor does it write. There is no `__setitem__`, so `view[...] = values` - raises `TypeError` too, and a wrapped source needs no `__setitem__` of - its own. A consumer that writes plans the selection with `plan_chunks` - and performs its own read-modify-write, keeping chunk atomicity and - concurrent-writer policy on the backend's side of the boundary. + Iteration yields lazy first-axis views; call `result()` on each to read it. + + `view.write(values)` writes through the composed transform to the original + writable source, synchronously, and returns `None`. `view[key] = values` + writes the selected sub-view in the same way. These calls do not create + futures or transactions. Read-only sources remain usable for reads; + writes require source assignment support. Storage atomicity and concurrent + writer coordination remain the backend's responsibility. ## An index defines a result array {#an-index-defines-a-result-array} @@ -407,7 +438,7 @@ rank one and source rank two. ### Order and duplicates need the request-side projection -Orthogonal indexing (`.lazy.oindex`) applies each axis's indexer +Orthogonal indexing (`.oindex`) applies each axis's indexer independently, like `numpy.ix_` — an outer product; the [pattern reference](patterns.md) develops the dialects. It can visit source cells in an order that does not match chunk order, and it can visit one diff --git a/packages/zarr-indexing/docs/guide/integrations.md b/packages/zarr-indexing/docs/guide/integrations.md index f2399e1f16..c96dcf91a5 100644 --- a/packages/zarr-indexing/docs/guide/integrations.md +++ b/packages/zarr-indexing/docs/guide/integrations.md @@ -254,3 +254,9 @@ and reading, but required when requesting a Dask token. The reader and partitioning are omitted because they must preserve values. Changing a source after graph construction does not update existing Dask keys. + +For consumers that require eager indexing, such as `dask.array.from_array`, +wrap the view in `EagerArrayAdapter` (importable from `zarr_indexing`). The +adapter materializes each indexed block while the wrapped view keeps lazy +indexing, and its token derives from the view's. See Dask's [from_array +documentation](https://docs.dask.org/en/stable/generated/dask.array.from_array.html). diff --git a/packages/zarr-indexing/docs/guide/patterns.md b/packages/zarr-indexing/docs/guide/patterns.md index f2f2ddbe2a..df9a482c3e 100644 --- a/packages/zarr-indexing/docs/guide/patterns.md +++ b/packages/zarr-indexing/docs/guide/patterns.md @@ -303,7 +303,7 @@ so they equal the zero-origin models after `translate_domain_to`: ``` `LazyArray` exposes the transform machinery through a positional array-like -API. Its `.lazy`, `.lazy.oindex`, and `.lazy.vindex` accessors return views and +API. Its `[...]`, `.oindex[...]`, and `.vindex[...]` operations return views and normalize positions before composition. This boundary differs from the literal coordinate semantics of `IndexTransform`, as the following table shows. The executable matrix checks the documented cases, not every possible NumPy expression. @@ -313,7 +313,7 @@ executable matrix checks the documented cases, not every possible NumPy expressi | Surface | Meaning of an integer index | Meaning of `-1` | | --- | --- | --- | | `IndexDomain` and `IndexTransform` | A literal coordinate in the current domain | The address `-1`, when the domain contains it | -| `LazyArray.lazy` | A NumPy-style position in the current view | The last position, normalized before transform composition | +| `LazyArray` | A NumPy-style position in the current view | The last position, normalized before transform composition | The wrapper's three indexing modes all use positions in the current view. Each derived view begins at position zero, while the transform algebra underneath diff --git a/packages/zarr-indexing/docs/index.md b/packages/zarr-indexing/docs/index.md index 598282b698..03537493f2 100644 --- a/packages/zarr-indexing/docs/index.md +++ b/packages/zarr-indexing/docs/index.md @@ -20,7 +20,7 @@ pip install zarr-indexing ## Quickstart -Wrap an array, compose a lazy view through `.lazy`, and call `result()` when +Wrap an array, compose a lazy view with `view[...]`, and call `result()` when you want its values: ```python diff --git a/packages/zarr-indexing/docs/snippets/axis_manipulation.py b/packages/zarr-indexing/docs/snippets/axis_manipulation.py index 1d1369fd87..8f35b1718c 100644 --- a/packages/zarr-indexing/docs/snippets/axis_manipulation.py +++ b/packages/zarr-indexing/docs/snippets/axis_manipulation.py @@ -8,8 +8,8 @@ image = np.arange(12).reshape(3, 4) lazy = LazyArray.from_numpy(image) -integer_view = lazy.lazy[1, :] -slice_view = lazy.lazy[1:2, :] +integer_view = lazy[1, :] +slice_view = lazy[1:2, :] INTEGER_RESULT = integer_view.result() SLICE_RESULT = slice_view.result() @@ -21,7 +21,7 @@ # --8<-- [end:axis-shape-comparison] # --8<-- [start:axis-insertion] -inserted_view = lazy.lazy[1, :, None] +inserted_view = lazy[1, :, None] INSERTED_RESULT = inserted_view.result() assert INSERTED_RESULT.tolist() == [[4], [5], [6], [7]] diff --git a/packages/zarr-indexing/docs/snippets/canonical_slice.py b/packages/zarr-indexing/docs/snippets/canonical_slice.py index fc4417f641..8d9fa387a8 100644 --- a/packages/zarr-indexing/docs/snippets/canonical_slice.py +++ b/packages/zarr-indexing/docs/snippets/canonical_slice.py @@ -11,7 +11,7 @@ from zarr_indexing import LazyArray source = np.array([10, 11, 12, 13, 14, 15]) -view = LazyArray.from_numpy(source).lazy[2:5] +view = LazyArray.from_numpy(source)[2:5] view.result() # array([12, 13, 14]) @@ -24,7 +24,7 @@ # --8<-- [start:canonical-slice] source = np.array([10, 11, 12, 13, 14, 15]) lazy = LazyArray.from_numpy(source) -view = lazy.lazy[2:5] +view = lazy[2:5] assert view.result().tolist() == [12, 13, 14] # --8<-- [end:canonical-slice] diff --git a/packages/zarr-indexing/docs/snippets/chunk_projection.py b/packages/zarr-indexing/docs/snippets/chunk_projection.py index 5c56aafbbd..e6c7394540 100644 --- a/packages/zarr-indexing/docs/snippets/chunk_projection.py +++ b/packages/zarr-indexing/docs/snippets/chunk_projection.py @@ -46,7 +46,7 @@ def indices_to_chunks(self, indices: NDArray[np.intp]) -> NDArray[np.intp]: # --8<-- [start:advanced-projection] image = np.arange(48).reshape(6, 8) -advanced = LazyArray.from_numpy(image).with_parts((3, 4)).lazy.oindex[[4, 1, 1], 2:6] +advanced = LazyArray.from_numpy(image).with_parts((3, 4)).oindex[[4, 1, 1], 2:6] ADVANCED_EXPECTED = image[[4, 1, 1]][:, 2:6] ADVANCED_RESULT = np.empty_like(ADVANCED_EXPECTED) diff --git a/packages/zarr-indexing/docs/snippets/coordinate_origins.py b/packages/zarr-indexing/docs/snippets/coordinate_origins.py index 724312e5e8..aef3605379 100644 --- a/packages/zarr-indexing/docs/snippets/coordinate_origins.py +++ b/packages/zarr-indexing/docs/snippets/coordinate_origins.py @@ -19,7 +19,7 @@ assert domain.narrow(-1).inclusive_min == (-1,) values = np.array([10, 20, 30, 40, 50]) -assert LazyArray.from_numpy(values).lazy[-1:].result().tolist() == [50] +assert LazyArray.from_numpy(values)[-1:].result().tolist() == [50] # --8<-- [end:coordinate-origin] diff --git a/packages/zarr-indexing/docs/snippets/integrations.py b/packages/zarr-indexing/docs/snippets/integrations.py index 5699c0d319..8608c506e6 100644 --- a/packages/zarr-indexing/docs/snippets/integrations.py +++ b/packages/zarr-indexing/docs/snippets/integrations.py @@ -56,7 +56,7 @@ def _gather_and_scatter( for chunk_column in range(2) } zarr_source = RecordingChunkSource(zarr_chunks) -zarr_view = LazyArray.from_numpy(zarr_image).with_parts((2, 2)).lazy[1, 0:4] +zarr_view = LazyArray.from_numpy(zarr_image).with_parts((2, 2))[1, 0:4] ZARR_RESULT = np.empty(zarr_view.shape, dtype=zarr_image.dtype) shared_domains: list[tuple[IndexDomain, IndexDomain]] = [] chunk_local_coords: list[tuple[tuple[int, ...], ...]] = [] @@ -123,7 +123,7 @@ def __getitem__(self, key: tuple[slice, ...]) -> np.ndarray[Any, Any]: viewport_source = RecordingArray(np.arange(12).reshape(3, 4), chunks=(2, 2)) -viewport = LazyArray(viewport_source).lazy[1, 0:4] +viewport = LazyArray(viewport_source)[1, 0:4] VIEWPORT_READS_BEFORE_RESULT = tuple(viewport_source.keys) assert VIEWPORT_READS_BEFORE_RESULT == () assert viewport.result().tolist() == [4, 5, 6, 7] @@ -152,12 +152,12 @@ def materialize(view: LazyArray) -> Any: slab_source = RecordingArray(np.arange(100).reshape(10, 10), chunks=(4, 4)) slab = LazyArray(slab_source) -dense = slab.lazy[2:9, 1:8] # a dense box: every stride 1 +dense = slab[2:9, 1:8] # a dense box: every stride 1 assert materialize(dense).shape == (7, 7) assert len(slab_source.keys) == 1 # one slab read; the source dispatches slab_source.keys.clear() -gather = slab.lazy.oindex[[0, 9], [0, 9]] # a query: keep the chunk parts +gather = slab.oindex[[0, 9], [0, 9]] # a query: keep the chunk parts assert materialize(gather).tolist() == [[0, 9], [90, 99]] assert len(slab_source.keys) == 4 # four covers, each inside one chunk assert all( diff --git a/packages/zarr-indexing/docs/snippets/key_types.py b/packages/zarr-indexing/docs/snippets/key_types.py new file mode 100644 index 0000000000..7917fc5eab --- /dev/null +++ b/packages/zarr-indexing/docs/snippets/key_types.py @@ -0,0 +1,30 @@ +"""The type of a key decides whether it is read relative to the view or as absolute coordinates.""" + +import numpy as np + +from zarr_indexing import IndexDomain, IndexTransform, LazyArray + +source = np.arange(30) + +# --8<-- [start:key-types] +view = LazyArray(source)[10:20] +assert view.transform.domain == IndexDomain((10,), (20,)) + +# NumPy keys are positions relative to the view: 0 is its first element. +relative = view[2:5] +assert relative.result().tolist() == [12, 13, 14] +assert view[-1].result() == 19 + +# An IndexDomain key names absolute coordinates of the view's domain. +absolute = view[IndexDomain((12,), (15,))] +assert absolute.result().tolist() == [12, 13, 14] + +# An IndexTransform key composes onto the view; index the identity over the +# view's domain to spell any selection in absolute coordinates. +literal = IndexTransform.identity(view.transform.domain) +assert view[literal[12:15]].result().tolist() == [12, 13, 14] +assert view[literal[::-1]].result().tolist()[:2] == [19, 18] + +# Whichever key produced a view, its domain is absolute. +assert relative.transform.domain == absolute.transform.domain == IndexDomain((12,), (15,)) +# --8<-- [end:key-types] diff --git a/packages/zarr-indexing/docs/snippets/lazy_composition.py b/packages/zarr-indexing/docs/snippets/lazy_composition.py index a4d9884b2b..88551d0023 100644 --- a/packages/zarr-indexing/docs/snippets/lazy_composition.py +++ b/packages/zarr-indexing/docs/snippets/lazy_composition.py @@ -7,8 +7,8 @@ # --8<-- [start:lazy-composition] source = np.array([10, 11, 12, 13, 14, 15]) -view = LazyArray.from_numpy(source).lazy[2:5] -composed = view.lazy[::-1].lazy[1:] +view = LazyArray.from_numpy(source)[2:5] +composed = view[::-1][1:] assert composed.result().tolist() == source[2:5][::-1][1:].tolist() # --8<-- [end:lazy-composition] diff --git a/packages/zarr-indexing/examples/lazy_indexing_dask/README.md b/packages/zarr-indexing/examples/lazy_indexing_dask/README.md index bdf5220ae0..e8d5ab5d88 100644 --- a/packages/zarr-indexing/examples/lazy_indexing_dask/README.md +++ b/packages/zarr-indexing/examples/lazy_indexing_dask/README.md @@ -1,13 +1,13 @@ # Lazy Indexing with Dask This example demonstrates how to use `zarr_indexing.LazyArray` with Dask, both as -an array Dask can wrap and as a source of independent tasks, and compares the two +an explicitly adapted array Dask can wrap and as a source of independent tasks, and compares the two ways of deferring an indexing operation. The example shows how to: -- Pass a `LazyArray` — over a Zarr array or over a view of one — to - `dask.array.from_array` +- Wrap a `LazyArray` — over a Zarr array or over a view of one — in + `EagerArrayAdapter` and pass the adapter to `dask.array.from_array` - Build one Dask task per partition from `parts()`, compute them in parallel, and place each result with the partition's `out_selection` - Read `is_complete` to inspect coverage of a partition cell @@ -16,10 +16,15 @@ The example shows how to: - Measure what a task graph costs for indexing-only work, against composing the same selections into one transform -A `LazyArray` exposes no `chunks` attribute, so `dask.array.from_array` chooses +The adapter exposes no `chunks` attribute, so `dask.array.from_array` chooses its own block size unless one is given. The partitioning that `parts()` reports is discovered from the wrapped array and is independent of Dask's blocks. +`LazyArray[...]` returns another lazy view. `EagerArrayAdapter(view)[...]` +returns materialized values, providing the block reads Dask expects. Import +both classes from `zarr_indexing`. Use the adapter for `from_array`; passing +a lazy view directly is not a reliable integration. + ## Choosing Between Them If Dask is doing arithmetic across chunks, reductions, rechunking, or distributed diff --git a/packages/zarr-indexing/examples/lazy_indexing_dask/lazy_indexing_dask.py b/packages/zarr-indexing/examples/lazy_indexing_dask/lazy_indexing_dask.py index e814bc3c76..891bd44472 100644 --- a/packages/zarr-indexing/examples/lazy_indexing_dask/lazy_indexing_dask.py +++ b/packages/zarr-indexing/examples/lazy_indexing_dask/lazy_indexing_dask.py @@ -24,7 +24,7 @@ import zarr from dask.base import tokenize -from zarr_indexing import LazyArray +from zarr_indexing import EagerArrayAdapter, LazyArray @pytest.fixture @@ -36,25 +36,25 @@ def source() -> zarr.Array: def test_from_array(source: zarr.Array) -> None: - """Hand a LazyArray to `dask.array.from_array`.""" + """Adapt lazy selections for `dask.array.from_array`.""" lazy = LazyArray(source) - # `from_array` needs `shape`, `dtype`, and `__getitem__`, which the wrapper - # provides. Each Dask block reads its own region through the wrapper. - array = da.from_array(lazy, chunks=(10, 10)) + # `from_array` expects eager block indexing, which `EagerArrayAdapter` + # provides. Each Dask block reads its own region through the adapter. + array = da.from_array(EagerArrayAdapter(lazy), chunks=(10, 10)) print(array) assert np.array_equal(array.compute(scheduler="threads"), source[:]) # A view works the same way, and its shape is the shape of the selection. - view = LazyArray(source).lazy[5:35, 3:27] - array = da.from_array(view, chunks=(10, 10)) + view = LazyArray(source)[5:35, 3:27] + array = da.from_array(EagerArrayAdapter(view), chunks=(10, 10)) assert array.shape == (30, 24) assert np.array_equal(array.compute(scheduler="threads"), source[5:35, 3:27]) def test_parts_as_tasks(source: zarr.Array) -> None: """Build one task per partition and compute them in parallel.""" - view = LazyArray(source).lazy[5:35, 3:27] + view = LazyArray(source)[5:35, 3:27] # The partitioning is discovered from the wrapped array's chunks, so each # partition of the view lies within one stored chunk. @@ -89,13 +89,13 @@ def test_tokenize(source: zarr.Array) -> None: # These wrappers have equal tokens despite being different Python objects. # Source mutation and token hooks affect whether cached results remain valid. assert tokenize(lazy) == tokenize(LazyArray(source)) - assert tokenize(lazy.lazy[0:10]) == tokenize(LazyArray(source).lazy[0:10]) + assert tokenize(lazy[0:10]) == tokenize(LazyArray(source)[0:10]) # Different selections are different tasks. - assert tokenize(lazy.lazy[0:10]) != tokenize(lazy.lazy[10:20]) + assert tokenize(lazy[0:10]) != tokenize(lazy[10:20]) # These two slice chains serialize to the same transform and token. - assert tokenize(lazy.lazy[0:20].lazy[5:10]) == tokenize(lazy.lazy[5:10]) + assert tokenize(lazy[0:20][5:10]) == tokenize(lazy[5:10]) def test_indexing_only_workload() -> None: @@ -119,13 +119,13 @@ def dask_chain(depth: int) -> da.Array: def lazy_chain(depth: int) -> LazyArray: view = LazyArray.from_numpy(data) for _ in range(depth): - view = view.lazy[1:] + view = view[1:] return view # Read once through each path first, so the timings below exclude the cost # of importing and initializing the machinery. dask_chain(1)[:2].compute(scheduler="synchronous") - lazy_chain(1).lazy[:2].result() + lazy_chain(1)[:2].result() header = ( f"{'selections':>10} {'dask compose':>13} {'dask read':>10} {'layers':>7}" @@ -146,7 +146,7 @@ def lazy_chain(depth: int) -> LazyArray: lazy_compose = time.perf_counter() - start start = time.perf_counter() - from_lazy = view.lazy[:2].result() + from_lazy = view[:2].result() lazy_read = time.perf_counter() - start # Both paths describe the same selection, so they read the same data. diff --git a/packages/zarr-indexing/examples/lazy_indexing_numpy/README.md b/packages/zarr-indexing/examples/lazy_indexing_numpy/README.md index 26540f4d4f..5da2eebc0a 100644 --- a/packages/zarr-indexing/examples/lazy_indexing_numpy/README.md +++ b/packages/zarr-indexing/examples/lazy_indexing_numpy/README.md @@ -6,8 +6,10 @@ index it without reading data. The example shows how to: - Wrap a NumPy array and read the forwarded `shape`, `dtype`, and `ndim` -- Compose selections through `.lazy[...]`, `.lazy.oindex[...]`, and - `.lazy.vindex[...]`, and materialize the composed view once with `result()` +- Compose selections through `[...]`, `.oindex[...]`, and + `.vindex[...]`, and materialize the composed view once with `result()` +- Iterate lazy row views and write through a composed view with `write(values)` + or indexed assignment; writes finish synchronously on the original source - Tell a box selection (slices and integers, described by an interval and a step per dimension) from a query selection (points gathered through an index array) using `is_box`, `bounding_box()`, and `strides()` diff --git a/packages/zarr-indexing/examples/lazy_indexing_numpy/lazy_indexing_numpy.py b/packages/zarr-indexing/examples/lazy_indexing_numpy/lazy_indexing_numpy.py index a545be1227..7c734d66f4 100644 --- a/packages/zarr-indexing/examples/lazy_indexing_numpy/lazy_indexing_numpy.py +++ b/packages/zarr-indexing/examples/lazy_indexing_numpy/lazy_indexing_numpy.py @@ -30,34 +30,47 @@ def test_wrap_and_compose() -> None: assert lazy.dtype == data.dtype assert lazy.ndim == 2 - # `.lazy[...]` returns another LazyArray. No element of `data` is read. - view = lazy.lazy[2:10, ::2] + # `[...]` returns another LazyArray. No element of `data` is read. + view = lazy[2:10, ::2] print(view) assert view.shape == (8, 4) # Selections compose. Each step narrows the view; still nothing is read. - smaller = view.lazy[1:5, 1:3] + smaller = view[1:5, 1:3] # `result()` performs the read. NumPy is the reference for the whole chain. assert np.array_equal(smaller.result(), data[2:10, ::2][1:5, 1:3]) # Selections use positional NumPy semantics: indices count from zero within # the current view, and negative indices count from the end. - assert np.array_equal(lazy.lazy[-1].result(), data[-1]) - assert np.array_equal(lazy.lazy[::-1].result(), data[::-1]) + assert np.array_equal(lazy[-1].result(), data[-1]) + assert np.array_equal(lazy[::-1].result(), data[::-1]) - # Orthogonal and vectorized indexing are available under the same accessor. + # Orthogonal and vectorized indexing also return lazy views. rows = np.array([9, 1, 4]) - assert np.array_equal(lazy.lazy.oindex[rows, :].result(), data[rows, :]) + assert np.array_equal(lazy.oindex[rows, :].result(), data[rows, :]) cols = np.array([0, 3, 7]) - assert np.array_equal(lazy.lazy.vindex[rows, cols].result(), data[rows, cols]) + assert np.array_equal(lazy.vindex[rows, cols].result(), data[rows, cols]) - # A LazyArray is also an ordinary duck array: __getitem__ reads immediately, - # and np.asarray materializes the view. - assert np.array_equal(lazy[2:4, 0], data[2:4, 0]) + # Indexing remains lazy; np.asarray explicitly materializes the view. + assert isinstance(lazy[2:4, 0], LazyArray) assert np.array_equal(np.asarray(view), data[2:10, ::2]) +def test_iteration_and_writes() -> None: + """Iterate lazy rows and synchronously write through a composed view.""" + data = np.arange(24).reshape(6, 4) + view = LazyArray.from_numpy(data)[1:5][::2] + rows = list(view) + assert all(isinstance(row, LazyArray) for row in rows) + np.testing.assert_array_equal(rows[1].result(), data[3]) + + assert view[:, 1:3].write(-1) is None + np.testing.assert_array_equal(data[[1, 3], 1:3], np.full((2, 2), -1)) + view[1, 0] = 99 + assert data[3, 0] == 99 + + def test_box_and_query_selections() -> None: """Distinguish selections that describe a region from selections that gather points.""" data = np.arange(12 * 8).reshape(12, 8) @@ -66,7 +79,7 @@ def test_box_and_query_selections() -> None: # This box selection is built from slices and integers. It is described # completely by an interval and a step per dimension, so a consumer can # serve it as one strided read. - box = lazy.lazy[2:10, ::2] + box = lazy[2:10, ::2] print(f"box: is_box={box.is_box} bounding_box={box.bounding_box()} strides={box.strides()}") assert box.is_box assert box.bounding_box() == ((2, 10), (0, 7)) @@ -75,7 +88,7 @@ def test_box_and_query_selections() -> None: # A query selection gathers points through an index array. Its coordinates # are a lookup table, so `strides()` is undefined and `bounding_box()` is # the hull of the points rather than an exact description. - query = lazy.lazy.oindex[np.array([9, 1, 4]), :] + query = lazy.oindex[np.array([9, 1, 4]), :] print(f"query: is_box={query.is_box} bounding_box={query.bounding_box()}") assert not query.is_box assert query.strides() is None @@ -83,7 +96,7 @@ def test_box_and_query_selections() -> None: # This slice retains an index-array map, so this particular view stays a query. # Singleton gathers and later scalar indexing can instead collapse to a box. - assert not query.lazy[0:2, 0:2].is_box + assert not query[0:2, 0:2].is_box def test_parts() -> None: @@ -93,7 +106,7 @@ def test_parts() -> None: # A plain NumPy array declares no partitioning, so `with_parts` states one. # Partitioning changes the granularity of reads, never the result. lazy = LazyArray.from_numpy(data).with_parts((4, 4)) - view = lazy.lazy[2:10, ::2] + view = lazy[2:10, ::2] parts = list(view.parts()) print(f"{len(parts)} parts") @@ -110,7 +123,7 @@ def test_parts() -> None: # The partitioning is a read strategy, so a different one gives the same data. assert np.array_equal( - LazyArray.from_numpy(data).with_parts((5, 3)).lazy[2:10, ::2].result(), assembled + LazyArray.from_numpy(data).with_parts((5, 3))[2:10, ::2].result(), assembled ) diff --git a/packages/zarr-indexing/examples/system_memory_chunk_cache/system_memory_chunk_cache.py b/packages/zarr-indexing/examples/system_memory_chunk_cache/system_memory_chunk_cache.py index e3aad6669a..92a9417309 100644 --- a/packages/zarr-indexing/examples/system_memory_chunk_cache/system_memory_chunk_cache.py +++ b/packages/zarr-indexing/examples/system_memory_chunk_cache/system_memory_chunk_cache.py @@ -329,7 +329,7 @@ def __getitem__(self, key: Any) -> np.ndarray[Any, Any]: def _read(self, key: Any, *, orthogonal: bool) -> np.ndarray[Any, Any]: self.reader.projection_uses.clear() - lazy = self._lazy.lazy + lazy = self._lazy view = lazy.oindex[key] if orthogonal else lazy[key] # One prepared tuple is the request plan: queue its chunks and defer # eviction during the request, then hand the diff --git a/packages/zarr-indexing/mkdocs.yml b/packages/zarr-indexing/mkdocs.yml index df9104ad8b..0f8b8ffa4e 100644 --- a/packages/zarr-indexing/mkdocs.yml +++ b/packages/zarr-indexing/mkdocs.yml @@ -51,6 +51,8 @@ nav: - ' zarr_indexing.grid': api/grid.md - ' zarr_indexing.lazy_array': api/lazy_array.md - ' zarr_indexing.reader': api/reader.md + - ' zarr_indexing.writer': api/writer.md + - ' zarr_indexing.eager': api/eager.md - ' zarr_indexing.boundary': api/boundary.md - ' zarr_indexing.json': api/json.md - ' zarr_indexing.messages': api/messages.md diff --git a/packages/zarr-indexing/src/zarr_indexing/__init__.py b/packages/zarr-indexing/src/zarr_indexing/__init__.py index b44c6d2a9c..866630b5c0 100644 --- a/packages/zarr-indexing/src/zarr_indexing/__init__.py +++ b/packages/zarr-indexing/src/zarr_indexing/__init__.py @@ -14,7 +14,7 @@ - `IndexTransform.compose` — chain two transforms into one `LazyArray` wraps a system-memory/basic-indexing source and gives it deferred -indexing through `.lazy[...]`, yielding its reads as `Partition`s. Other +indexing through `view[...]`, yielding its reads as `Partition`s. Other backends use an explicit `Reader` adapter. `plan_chunks` projects a transform through a caller-selected chunk grid without @@ -39,6 +39,7 @@ plan_chunks, ) from zarr_indexing.domain import IndexDomain +from zarr_indexing.eager import EagerArrayAdapter from zarr_indexing.errors import BoundsCheckError, VindexInvalidSelectionError from zarr_indexing.grid import ( ChunkGrid, @@ -93,6 +94,7 @@ "DimensionGrid", "DimensionGridLike", "DimensionMap", + "EagerArrayAdapter", "EdgeDimensionGrid", "FixedDimension", "GridPartition", diff --git a/packages/zarr-indexing/src/zarr_indexing/boundary.py b/packages/zarr-indexing/src/zarr_indexing/boundary.py index aa8faf1264..6574bded3f 100644 --- a/packages/zarr-indexing/src/zarr_indexing/boundary.py +++ b/packages/zarr-indexing/src/zarr_indexing/boundary.py @@ -13,11 +13,9 @@ Note ---- -`zarr.Array` currently carries its own copy of this normalization, tuned to a -different boundary contract (`Array.lazy[...]` deliberately exposes the literal -dialect, so a view's coordinates keep their meaning across composition). This -module is the generic, zarr-free version used by `LazyArray`; consolidating -zarr's copy onto it is left to a follow-up. +This module provides the positional boundary used by `LazyArray`, independently +of Zarr's own array indexers. Direct `IndexTransform` indexing continues to use +literal coordinates. """ from __future__ import annotations diff --git a/packages/zarr-indexing/src/zarr_indexing/eager.py b/packages/zarr-indexing/src/zarr_indexing/eager.py new file mode 100644 index 0000000000..16aad1eb65 --- /dev/null +++ b/packages/zarr-indexing/src/zarr_indexing/eager.py @@ -0,0 +1,57 @@ +"""Explicit eager indexing for consumers such as Dask's ``from_array``.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from zarr_indexing.lazy_array import LazyArray + + +class EagerArrayAdapter: + """Expose an eager array interface over a lazy view. + + Constructing the adapter does not read source values. Indexing materializes + the selected view in fresh memory, using its existing reader and partitioning. + Tokenization delegates to the view's source-token contract and may read data. + + Examples + -------- + >>> import numpy as np + >>> from zarr_indexing import LazyArray + >>> view = LazyArray(np.arange(8))[1::2] + >>> EagerArrayAdapter(view)[1:3] + array([3, 5]) + """ + + __slots__ = ("_view",) + + def __init__(self, view: LazyArray) -> None: + self._view = view + + @property + def shape(self) -> tuple[int, ...]: + """Shape of the selected view.""" + return self._view.shape + + @property + def ndim(self) -> int: + """Number of dimensions of the selected view.""" + return self._view.ndim + + @property + def dtype(self) -> Any: + """Data type reported by the source.""" + return self._view.dtype + + def __getitem__(self, selection: Any) -> Any: + """Read a basic selection into fresh memory.""" + return self._view[selection].result() + + def __array__(self, dtype: Any = None, copy: bool | None = None) -> Any: + """Materialize using the view's NumPy conversion contract.""" + return self._view.__array__(dtype=dtype, copy=copy) + + def __dask_tokenize__(self) -> tuple[Any, ...]: + """Distinguish eager tasks while preserving the source identity contract.""" + return (type(self).__qualname__, self._view.__dask_tokenize__()) diff --git a/packages/zarr-indexing/src/zarr_indexing/errors.py b/packages/zarr-indexing/src/zarr_indexing/errors.py index 74780aa4fc..24fc6c6ba6 100644 --- a/packages/zarr-indexing/src/zarr_indexing/errors.py +++ b/packages/zarr-indexing/src/zarr_indexing/errors.py @@ -34,7 +34,7 @@ class VindexInvalidSelectionError(IndexError): >>> import numpy as np >>> from zarr_indexing import LazyArray >>> view = LazyArray.from_numpy(np.arange(12).reshape(3, 4)) - >>> view.lazy.vindex[np.array([0, 2]), :] + >>> view.vindex[np.array([0, 2]), :] Traceback (most recent call last): ... zarr_indexing.errors.VindexInvalidSelectionError: ... diff --git a/packages/zarr-indexing/src/zarr_indexing/lazy_array.py b/packages/zarr-indexing/src/zarr_indexing/lazy_array.py index 4f71df35f8..43ffd5625f 100644 --- a/packages/zarr-indexing/src/zarr_indexing/lazy_array.py +++ b/packages/zarr-indexing/src/zarr_indexing/lazy_array.py @@ -2,18 +2,18 @@ `LazyArray` wraps a source with `shape`, `dtype`, and basic integer/slice `__getitem__`, whose reads can be lowered through NumPy system memory. It adds -a `.lazy` accessor whose indexing operations build up an +indexing operations that build up an [`IndexTransform`](transform.md) instead of reading data: ```python -view = LazyArray(source).lazy[10:50, ::2].lazy.oindex[[3, 1, 1], :] +view = LazyArray(source)[10:50, ::2].oindex[[3, 1, 1], :] view.shape # known without touching the data values = view.result() ``` Selection construction does not read source values: `result()`, `__array__`, -and eager `__getitem__` perform reads. Source tokenization is delegated to -Dask and may inspect source values. `.lazy` operations inspect +and scalar conversions perform reads. Source tokenization is delegated to +Dask and may inspect source values. Indexing operations inspect selection metadata and may copy or process supplied index arrays. Composition does not accumulate wrapper layers: a view of a view is still a single transform and retains its reader. @@ -85,17 +85,30 @@ traversal order. A query can fill its hull, and a singleton box can fill its hull even when its recorded stride exceeds one. -The positional dialect ----------------------- -Selections on `LazyArray` are **positional, NumPy-style**: index 0 is the first +Relative keys, absolute domain +------------------------------ +**The type of a key decides whether it is relative or absolute.** NumPy keys +(slices, integers, `...`, `None`, index arrays, masks) are positions relative to +the current view. An `IndexDomain` key names absolute coordinates of the view's +domain, and an `IndexTransform` key composes onto the view. No key type has two +readings, and whichever key produced a view, `view.transform.domain` is +absolute. See the guide section "The key's type picks the frame". + +Selections with NumPy keys are **positional, NumPy-style**: index 0 is the first element of the current view, `-1` is the last, boolean masks must match the view's shape, integer coordinates are bounds-checked, and slices are clipped to the view's extent. -This differs from the low-level `IndexTransform` literal-coordinate dialect: -a transform can retain a nonzero domain origin, while `LazyArray` re-zeroes -positions on each derived view. The current main Zarr `Array` does not expose -this wrapper as an `Array.lazy` attribute; use `LazyArray(array)` explicitly. +The dialect governs how a *key* is read, not what a view remembers. A derived +view keeps its **literal domain**, as a TensorStore view does: `a[10:20]` has +domain `[10, 20)` and `a[10:20][2:5]` has domain `[12, 15)`, while `a[10:20][0]` +still names the first element because positional keys are normalized against +the domain's origin. The literal frame is reachable through `view.transform`, +and directly through the two literal keys: an `IndexDomain` restricts the view +to those coordinates, and an `IndexTransform` composes onto it. A reversed view +carries a negative origin, as in TensorStore, because a reversing map traverses +the source frame backwards. The current main Zarr `Array` does not expose this +wrapper as an `Array.lazy` attribute; use `LazyArray(array)` explicitly. Scalar integers drop axes. Non-boolean objects implementing `SupportsIndex` are accepted as scalar indices and in slice bounds; `__int__` alone is not enough. @@ -122,9 +135,9 @@ the wrapper defines no arithmetic dunders and an `int` has nothing to dispatch to. Both facts follow from the same intent — laziness here applies to indexing, not to building a deferred compute graph — and a `LazyArray` is not a drop-in -for arithmetic on a large array either way. Use `.lazy[...]` to narrow the view -first, or pass the wrapper to `dask.array.from_array` so that dask owns the -compute graph. +for arithmetic on a large array either way. Use `[...]` to narrow the view +first, or pass `EagerArrayAdapter(view)` to `dask.array.from_array` so that +Dask owns the compute graph. Ownership --------- @@ -140,7 +153,7 @@ import math import operator from collections.abc import Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any, Protocol, cast import numpy as np @@ -154,6 +167,8 @@ ChunkProjection, plan_chunks, ) +from zarr_indexing.domain import IndexDomain +from zarr_indexing.errors import BoundsCheckError from zarr_indexing.grid import DimensionGrid, FixedDimension, dimension_grids_from_chunks from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap from zarr_indexing.reader import ( @@ -250,6 +265,25 @@ def _discover_parts(array: Any, shape: tuple[int, ...]) -> tuple[DimensionGrid, return None +def _discover_write_grid(array: Any, shape: tuple[int, ...]) -> tuple[DimensionGrid, ...] | None: + """Resolve the grid `array` commits writes in, or None for one whole cell. + + The write grid can be coarser than the read grid: a sharded zarr array + decodes inner chunks but replaces whole shards, so `write_chunk_sizes` is + consulted before `chunks`. As with read discovery, anything unusable + means "no advertised grid" rather than an error. + """ + declared = _read_source_attribute(array, "write_chunk_sizes") + if declared is None: + declared = _read_source_attribute(array, "chunks") + if declared is None: + return None + try: + return dimension_grids_from_chunks(declared, shape) + except (ValueError, TypeError): + return None + + def _whole_array_grids(shape: tuple[int, ...]) -> tuple[DimensionGrid, ...]: """A partitioning with a single part covering the whole array.""" return tuple(FixedDimension(size=extent, extent=extent) for extent in shape) @@ -260,6 +294,20 @@ def _whole_array_grids(shape: tuple[int, ...]) -> tuple[DimensionGrid, ...]: # --------------------------------------------------------------------------- # +def _pure_translation(transform: IndexTransform) -> tuple[int, ...] | None: + """The per-axis shift when `transform` only translates its zero-origin domain, else None.""" + if transform.output_rank != transform.input_rank or any( + transform.domain.inclusive_min[i] != 0 for i in range(transform.input_rank) + ): + return None + shift: list[int] = [] + for axis, m in enumerate(transform.output): + if not (isinstance(m, DimensionMap) and m.input_dimension == axis and m.stride == 1): + return None + shift.append(m.offset) + return tuple(shift) + + def _is_identity_transform(transform: IndexTransform, shape: tuple[int, ...]) -> bool: """True when `transform` maps every coordinate of `shape` to itself. @@ -432,9 +480,11 @@ class Partition: projection The source-independent description of this part. Its paired `chunk_transform` and `cell_transform` share one compact synthetic - domain, mapping each selected cell to chunk-local storage and request - coordinates respectively. This is the authoritative placement model; - `base_coords` and `is_complete` are conveniences derived from it. + domain, mapping each selected cell to chunk-local storage coordinates + and to positions in this view's zero-origin result buffer respectively. + (The planner's own projections place cells in the request's literal + domain; `parts()` re-bases them.) This is the authoritative placement + model; `base_coords` and `is_complete` are conveniences derived from it. base_coords Which box of the base partitioning this is, one coordinate per dimension of the wrapped array. @@ -448,8 +498,12 @@ class Partition: view A `LazyArray` covering exactly the cells of the view that live in this box. Its transform directly addresses its raw wrapped `array`; only the - projection's `chunk_transform` is chunk-local. The view retains the source - grid and reader and can be read or indexed like any other view. Named `view` rather than + projection's `chunk_transform` is chunk-local. A box part keeps the + request's literal coordinates, so its domain is a sub-domain of the + parent view's; a part placed by index arrays has a fresh zero-origin + domain, and `out_selection` is the placement in both cases. The view + retains the source grid and reader and can be read or indexed like any + other view. Named `view` rather than `array` because `LazyArray.array` is the opposite thing — the raw wrapped source — and the two sat next to each other meaning inverses. out_selection @@ -530,7 +584,7 @@ class LazyArray: """A lazily-indexable view over a system-memory/basic-indexing source. Wrapping neither copies nor reads the wrapped array at construction time. - Indexing through `.lazy` composes an `IndexTransform` and returns another + Indexing composes an `IndexTransform` and returns another `LazyArray`; `result()` materializes. Selections use the **positional NumPy dialect** and reads are broken up @@ -540,12 +594,9 @@ class LazyArray: dialect differs from low-level literal transforms and which NumPy operations materialize the view. - This wrapper describes **reads**. It defines no `__setitem__`, so - assigning into a view raises `TypeError`. Writing belongs to the - consumer: plan the selection with - [`plan_chunks`][zarr_indexing.chunk_resolution.plan_chunks] and own the - read-modify-write, since chunk atomicity and concurrent-writer policy are - the backend's to decide, not an indexing plan's. + Selection is lazy; `result()` reads and `write(values)` writes synchronously. + Assignment is shorthand for selecting a view and calling its `write` method. + The source owns storage errors and concurrency; writes are not transactional. Parameters ---------- @@ -562,7 +613,7 @@ class LazyArray: -------- >>> import numpy as np >>> source = np.arange(12).reshape(3, 4) - >>> view = LazyArray.from_numpy(source).with_parts((2, 2)).lazy[1:, ::2] + >>> view = LazyArray.from_numpy(source).with_parts((2, 2))[1:, ::2] >>> view.shape (2, 2) >>> view.result() @@ -616,9 +667,10 @@ def _derive( """Build a wrapper sharing `array` but carrying a new transform or partitioning.""" view = cls.__new__(cls) view._array = array - # Views re-zero their coordinate system: the positional dialect means a - # view's first element is at position 0 whatever it was sliced from. - view._transform = transform.translate_domain_to((0,) * transform.input_rank) + # The domain keeps its literal coordinates, as TensorStore's does: a + # slice of `[10, 20)` remembers that it is `[10, 20)`. Positional keys + # are normalized against that origin at selection time instead. + view._transform = transform view._parts = parts view._reader = reader view._part_owner = _PartOwner() @@ -705,7 +757,7 @@ def is_box(self) -> bool: -------- >>> import numpy as np >>> array = LazyArray.from_numpy(np.arange(12).reshape(3, 4)) - >>> (array.lazy[1:, ::2].is_box, array.lazy.oindex[[2, 0], :].is_box) + >>> (array[1:, ::2].is_box, array.oindex[[2, 0], :].is_box) (True, False) """ return not any(isinstance(m, ArrayMap) for m in self._transform.output) @@ -741,11 +793,11 @@ def bounding_box(self) -> tuple[tuple[int, int], ...] | None: -------- >>> import numpy as np >>> array = LazyArray.from_numpy(np.arange(12).reshape(3, 4)) - >>> array.lazy[1:, ::2].bounding_box() + >>> array[1:, ::2].bounding_box() ((1, 3), (0, 3)) - >>> array.lazy.oindex[[2, 0], :].bounding_box() + >>> array.oindex[[2, 0], :].bounding_box() ((0, 3), (0, 4)) - >>> array.lazy[1:1].bounding_box() is None + >>> array[1:1].bounding_box() is None True """ if self.size == 0: @@ -798,11 +850,11 @@ def strides(self) -> tuple[int, ...] | None: -------- >>> import numpy as np >>> array = LazyArray.from_numpy(np.arange(24).reshape(4, 6)) - >>> (array.lazy[1:, ::2].bounding_box(), array.lazy[1:, ::2].strides()) + >>> (array[1:, ::2].bounding_box(), array[1:, ::2].strides()) (((1, 4), (0, 5)), (1, 2)) - >>> array.lazy[2, ::3].strides() + >>> array[2, ::3].strides() (1, 3) - >>> array.lazy.oindex[[2, 0], :].strides() is None + >>> array.oindex[[2, 0], :].strides() is None True """ if not self.is_box: @@ -951,22 +1003,30 @@ def parts(self) -> Iterator[Partition]: -------- >>> import numpy as np >>> view = LazyArray.from_numpy(np.arange(12).reshape(3, 4)).with_parts((2, 2)) - >>> part = next(view.lazy[:, 1:].parts()) + >>> part = next(view[:, 1:].parts()) >>> (part.base_coords, part.view.shape, part.is_complete) ((0, 0), (2, 1), False) """ grids = self._parts if self._parts is not None else _whole_array_grids(self._base_shape) - for projection in plan_chunks(self._transform, grids): - domain = projection.chunk_domain + to_buffer = tuple(-o for o in self._transform.domain.inclusive_min) + for planned in plan_chunks(self._transform, grids): + domain = planned.chunk_domain + part_transform = planned.chunk_transform.translate(domain.inclusive_min) + placement = _pure_translation(planned.cell_transform) + if placement is not None: + # A box part sits at a fixed offset in the request, so its view + # keeps the request's literal coordinates: its domain is a + # sub-domain of this view's, and placement is readable from it. + part_transform = part_transform.translate_domain_by(placement) + # The planner places cells in the request's literal domain; the + # partition places them in this view's zero-origin result buffer. + projection = replace( + planned, cell_transform=planned.cell_transform.translate(to_buffer) + ) yield Partition( projection=projection, box=tuple(zip(domain.inclusive_min, domain.exclusive_max, strict=True)), - view=LazyArray._derive( - self._array, - projection.chunk_transform.translate(domain.inclusive_min), - self._parts, - self._reader, - ), + view=LazyArray._derive(self._array, part_transform, self._parts, self._reader), out_selection=_partition_out_selection(projection.cell_transform), _owner=self._part_owner, ) @@ -974,12 +1034,14 @@ def parts(self) -> Iterator[Partition]: # -- indexing ----------------------------------------------------------- @property - def lazy(self) -> _LazyIndexer: - """Lazy indexing: `lazy[...]`, `lazy.oindex[...]`, `lazy.vindex[...]`. + def oindex(self) -> _LazyOIndex: + """Build a view using orthogonal (outer-product) indexing.""" + return _LazyOIndex(self._select) - Each returns a new `LazyArray` view; no data is read. - """ - return _LazyIndexer(self._select) + @property + def vindex(self) -> _LazyVIndex: + """Build a view using vectorized coordinate or mask indexing.""" + return _LazyVIndex(self._select) def _select(self, selection: Any, mode: SelectionMode) -> LazyArray: transform = self._transform @@ -989,7 +1051,6 @@ def _select(self, selection: Any, mode: SelectionMode) -> LazyArray: scalar_selection, selection = split_scalar_axes(selection, transform.domain, mode) if scalar_selection is not None: transform = transform.select(scalar_selection, "basic") - transform = transform.translate_domain_to((0,) * transform.input_rank) literal = normalize_positional_selection(selection, transform.domain, mode) if mode == "basic": # IndexTransform's basic path includes NumPy's `None`/newaxis. @@ -1000,14 +1061,87 @@ def _select(self, selection: Any, mode: SelectionMode) -> LazyArray: composed = transform.select(literal, mode) return LazyArray._derive(self._array, composed, self._parts, self._reader) - def __getitem__(self, selection: Any) -> Any: - """Read a basic selection eagerly, like `numpy.ndarray.__getitem__`. + def __getitem__(self, selection: Any) -> LazyArray: + """Build a view without reading source values. + + A NumPy key (integers, slices, ellipsis, `None`) is **positional**: + `view[0]` is the first element and `view[-1]` the last, whatever + literal coordinates the view's domain carries. An `IndexDomain` key + is **literal**, restricting the view to those coordinates of its + domain, and an `IndexTransform` key composes onto the view, mapping + the key's domain into the view's domain. These are the two ways + TensorStore's `__getitem__` addresses a view, offered here without + changing what a NumPy key means. + """ + if isinstance(selection, IndexDomain): + return self._select_domain(selection) + if isinstance(selection, IndexTransform): + return self._select_transform(selection) + return self._select(selection, "basic") - Reads here are eager, not lazy, so that a `LazyArray` works as a duck - array for consumers (dask's `from_array`, `numpy.asarray`) that expect - indexing to produce data. Use `.lazy[...]` for the lazy form. + def _select_domain(self, domain: IndexDomain) -> LazyArray: + """Restrict the view to a literal sub-domain of its own domain.""" + if domain.ndim != self._transform.input_rank: + raise ValueError( + f"domain rank {domain.ndim} does not match view rank {self._transform.input_rank}" + ) + own = self._transform.domain + for axis, (lo, hi) in enumerate( + zip(domain.inclusive_min, domain.exclusive_max, strict=True) + ): + # Stricter than a literal slice, which admits an empty interval + # anywhere: a domain key names coordinates of this view's domain. + if lo < own.inclusive_min[axis] or hi > own.exclusive_max[axis]: + raise BoundsCheckError( + f"domain [{lo}, {hi}) on dimension {axis} is outside the view's domain " + f"[{own.inclusive_min[axis]}, {own.exclusive_max[axis]})" + ) + literal = tuple( + slice(lo, hi) for lo, hi in zip(domain.inclusive_min, domain.exclusive_max, strict=True) + ) + return LazyArray._derive(self._array, self._transform[literal], self._parts, self._reader) + + def _select_transform(self, transform: IndexTransform) -> LazyArray: + """Compose a literal transform onto the view, as TensorStore's `store[transform]`.""" + return LazyArray._derive( + self._array, transform.compose(self._transform), self._parts, self._reader + ) + + def __setitem__(self, selection: Any, values: Any) -> None: + """Select a basic view and synchronously write its values.""" + self[selection].write(values) + + def write(self, values: Any) -> None: + """Synchronously write broadcastable values through this view. + + The source must support integer/slice assignment. Values are copied + before mutation, including when they alias the source. Writes follow + C-order view coordinates; the last occurrence wins for repeated source + coordinates. + + Affine selections use one basic assignment. Other selections are + scattered against the source's **write grid**, discovered from + `write_chunk_sizes` or `chunks`: each touched cell is read once, updated + in memory, and written back, so the number of storage round trips is + bounded by the number of touched cells rather than selected elements. + A NumPy source receives one fancy assignment instead, and a source with + no advertised grid is written one element at a time without reading. + The read-side partitioning (`with_parts`) does not affect writes. + + Writes go to the source directly and bypass the reader. A reader that + caches source data is not invalidated, so reading after writing through + such a reader may return stale values. Backend errors propagate and may + leave a partially written source. This method does not provide + transactions, concurrency control, or asynchronous execution. """ - return self._select(selection, "basic").result() + from zarr_indexing.writer import write_into + + write_into( + self._array, + self._transform, + values, + write_grid=_discover_write_grid(self._array, self._base_shape), + ) def result(self, *, parts: Sequence[Partition] | None = None) -> Any: """Materialize this view. @@ -1162,8 +1296,11 @@ def __len__(self) -> int: raise TypeError("len() of unsized object") return self.shape[0] - def __iter__(self) -> Iterator[Any]: - """Iterate eagerly over the first axis, like a NumPy array. + def __iter__(self) -> Iterator[LazyArray]: + """Iterate over lazy first-axis views without reading source values. + + Each element is a `LazyArray`, not a value: call `result()` or convert + with NumPy before doing arithmetic on it. The rank check happens in `__iter__` itself rather than in the generator, so `iter(view)` on a zero-rank view raises immediately as @@ -1196,35 +1333,8 @@ def __repr__(self) -> str: return f"" -class _LazyIndexer: - """The `.lazy` accessor: builds views instead of reading data. - - Holds the owning view's bound `_select` rather than the view itself, so the - accessor classes never reach into another object's internals. - """ - - __slots__ = ("_select",) - - def __init__(self, select: SelectFn) -> None: - self._select = select - - def __getitem__(self, selection: Any) -> LazyArray: - """Basic (integer / slice / ellipsis) indexing, lazily.""" - return self._select(selection, "basic") - - @property - def oindex(self) -> _LazyOIndex: - """Orthogonal (outer-product) indexing, lazily.""" - return _LazyOIndex(self._select) - - @property - def vindex(self) -> _LazyVIndex: - """Vectorized (coordinate / mask) indexing, lazily.""" - return _LazyVIndex(self._select) - - class _LazyOIndex: - """`lazy.oindex[...]` — one selection per axis, combined as an outer product.""" + """`view.oindex[...]` — one selection per axis, combined as an outer product.""" __slots__ = ("_select",) @@ -1234,9 +1344,12 @@ def __init__(self, select: SelectFn) -> None: def __getitem__(self, selection: Any) -> LazyArray: return self._select(selection, "orthogonal") + def __setitem__(self, selection: Any, values: Any) -> None: + self[selection].write(values) + class _LazyVIndex: - """`lazy.vindex[...]` — correlated coordinate arrays, or a single mask.""" + """`view.vindex[...]` — correlated coordinate arrays, or a single mask.""" __slots__ = ("_select",) @@ -1245,3 +1358,6 @@ def __init__(self, select: SelectFn) -> None: def __getitem__(self, selection: Any) -> LazyArray: return self._select(selection, "vectorized") + + def __setitem__(self, selection: Any, values: Any) -> None: + self[selection].write(values) diff --git a/packages/zarr-indexing/src/zarr_indexing/reader.py b/packages/zarr-indexing/src/zarr_indexing/reader.py index bfb388b3e7..3da7267335 100644 --- a/packages/zarr-indexing/src/zarr_indexing/reader.py +++ b/packages/zarr-indexing/src/zarr_indexing/reader.py @@ -45,7 +45,12 @@ class ReadContext: """ transform: IndexTransform - """Maps zero-origin output-buffer coordinates to global coordinates in the source.""" + """Maps zero-origin output-buffer coordinates to global coordinates in the source. + + A transform carrying a literal (nonzero-origin) domain, such as a view's, + is re-based to origin zero on construction: readers address the buffer + they fill, so the origin is fixed here rather than by every caller. + """ projection: ChunkProjection | None = None """The read plan, always supplied by `LazyArray` execution. @@ -53,6 +58,13 @@ class ReadContext: Direct reader callers may omit it if their reader supports unplanned reads. """ + def __post_init__(self) -> None: + origin = self.transform.domain.inclusive_min + if any(origin): + object.__setattr__( + self, "transform", self.transform.translate_domain_to((0,) * len(origin)) + ) + class Reader(Protocol): """Backend adapter that fills supplied system-memory result buffers. @@ -88,7 +100,8 @@ def read_into( to global coordinates in `source`, and its domain shape equals `out.shape`. `context.projection`, when present, is the corresponding partition plan: its `chunk_transform` is chunk-local, its - `cell_transform` describes result placement, and its `chunk_domain` + `cell_transform` places cells in the zero-origin result buffer of the + view that planned the read, and its `chunk_domain` describes the grid cell. Fill every cell in place, preserving the transform's exact values, order, and dtype, then return `None`. Do not replace or retain `out`; it may be a strided writable view rather than diff --git a/packages/zarr-indexing/src/zarr_indexing/testing/stateful.py b/packages/zarr-indexing/src/zarr_indexing/testing/stateful.py index b407f121e9..00dce58bb9 100644 --- a/packages/zarr-indexing/src/zarr_indexing/testing/stateful.py +++ b/packages/zarr-indexing/src/zarr_indexing/testing/stateful.py @@ -253,11 +253,11 @@ def _step(self, mode: SelectionMode, selection: tuple[Any, ...]) -> None: self.chain.append((mode, selection)) self.model = apply_selection(self.model, selection, mode) if mode == "basic": - self.view = self.view.lazy[selection] + self.view = self.view[selection] elif mode == "orthogonal": - self.view = self.view.lazy.oindex[selection] + self.view = self.view.oindex[selection] else: - self.view = self.view.lazy.vindex[selection] + self.view = self.view.vindex[selection] # -- rules -------------------------------------------------------------- diff --git a/packages/zarr-indexing/src/zarr_indexing/transform.py b/packages/zarr-indexing/src/zarr_indexing/transform.py index 39976edc90..3486060b20 100644 --- a/packages/zarr-indexing/src/zarr_indexing/transform.py +++ b/packages/zarr-indexing/src/zarr_indexing/transform.py @@ -23,7 +23,7 @@ The transform is the atomic unit that connects user-facing indexing to chunk-level I/O. A wrapper holds one — `LazyArray` starts from the identity — -and `.lazy[...]` composes a new transform lazily rather than reading. Reading +and `view[...]` composes a new transform lazily rather than reading. Reading resolves the transform against the chunk grid via intersect + translate. """ diff --git a/packages/zarr-indexing/src/zarr_indexing/writer.py b/packages/zarr-indexing/src/zarr_indexing/writer.py new file mode 100644 index 0000000000..3301e22574 --- /dev/null +++ b/packages/zarr-indexing/src/zarr_indexing/writer.py @@ -0,0 +1,225 @@ +"""Synchronous writes through coordinate transforms using basic source assignment.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import numpy as np + +from zarr_indexing.chunk_resolution import plan_chunks +from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap + +if TYPE_CHECKING: + from collections.abc import Sequence + + from zarr_indexing.chunk_resolution import ChunkProjection + from zarr_indexing.grid import DimensionGridLike + from zarr_indexing.transform import IndexTransform + + +def write_into( + source: Any, + transform: IndexTransform, + values: Any, + write_grid: Sequence[DimensionGridLike] | None = None, +) -> None: + """Write broadcast values to precisely the cells addressed by ``transform``. + + The source must expose ``shape``, a NumPy-compatible ``dtype``, and basic + integer/slice assignment. Values are snapshotted and converted before the + first mutation, so source aliases and conversion failures are safe. Extra + leading singleton value axes are accepted, as in NumPy assignment. Repeated + source coordinates receive the last value in C order of the view. + + Independent affine maps use one basic assignment. Other selections are + scattered in bulk: a NumPy source receives one fancy assignment, and a + readable source with a ``write_grid`` is written one grid cell at a time — + the cell's touched hull is read once, updated in memory, and written back + with one basic assignment, so storage round trips are bounded by touched + cells. Without a grid, or for the few transforms the planner cannot factor + (such as one input axis feeding two output maps), or for a source that + cannot be read, the fallback is one integer assignment per element, which + never reads. Backend assignment may itself read storage units as part of + updating them. + + Source assignment failures propagate and may leave preceding writes + applied; this operation is not transactional. + """ + from zarr_indexing.lazy_array import LazyArray + + shape = transform.domain.shape + source_shape = tuple(int(extent) for extent in source.shape) + if transform.output_rank != len(source_shape): + raise ValueError("transform output rank must match the source rank") + if isinstance(values, LazyArray): + # Its NumPy conversion protocol may discard MaskedArray metadata. + values = values.result() + if isinstance(source, np.ma.MaskedArray): + prepared = np.ma.array(values, dtype=source.dtype, copy=True) + else: + prepared = np.array(values, dtype=source.dtype, copy=True) + while prepared.ndim > len(shape) and prepared.shape[0] == 1: + prepared = prepared[0] + if isinstance(prepared, np.ma.MaskedArray): + broadcast = np.ma.array( + np.broadcast_to(prepared.data, shape), + mask=np.broadcast_to(np.ma.getmaskarray(prepared), shape), + copy=False, + ) + else: + broadcast = np.broadcast_to(prepared, shape) + if any(extent == 0 for extent in shape): + # Values were still converted and broadcast, so a mis-sized RHS is + # reported even when the selection happens to be empty. + return + + # Check every map's extremal outputs before any assignment. Python integer + # arithmetic avoids overflow while computing bounds of affine maps. + for extent, output in zip(source_shape, transform.output, strict=True): + if isinstance(output, ConstantMap): + low = high = output.offset + else: + if isinstance(output, DimensionMap): + axis = output.input_dimension + first = transform.domain.inclusive_min[axis] + last = transform.domain.exclusive_max[axis] - 1 + else: + first = int(output.index_array.min()) + last = int(output.index_array.max()) + endpoints = ( + output.offset + output.stride * first, + output.offset + output.stride * last, + ) + low, high = min(endpoints), max(endpoints) + if low < 0 or high >= extent: + raise IndexError("transform output coordinates are outside the source shape") + + if _write_affine(source, transform, broadcast): + return + if len(source_shape) == 0: + # A zero-rank source has one cell; no grid or scatter applies. + source[()] = broadcast[(0,) * len(shape)] + return + if isinstance(source, np.ndarray): + _scatter_numpy(source, transform, broadcast) + return + if write_grid is not None and hasattr(source, "__getitem__"): + try: + # Materialized before any write: an unfactorable transform is + # reported by planning, never after some cells were rewritten. + projections = list(plan_chunks(transform, tuple(write_grid))) + except (ValueError, NotImplementedError): + projections = None + if projections is not None: + _scatter_planned(source, transform, broadcast, projections) + return + _scatter_elementwise(source, transform, broadcast) + + +def _write_affine(source: Any, transform: IndexTransform, broadcast: Any) -> bool: + """Write with one basic assignment when every map is an independent affine map.""" + shape = transform.domain.shape + axes = [ + output.input_dimension for output in transform.output if isinstance(output, DimensionMap) + ] + if ( + any(isinstance(output, ArrayMap) for output in transform.output) + or len(set(axes)) != len(axes) + or any(shape[axis] != 1 for axis in range(len(shape)) if axis not in axes) + or any( + isinstance(output, DimensionMap) and output.stride == 0 for output in transform.output + ) + ): + return False + selection: list[int | slice] = [] + reverse_axes: list[int] = [] + for output in transform.output: + if isinstance(output, ConstantMap): + selection.append(output.offset) + else: + assert isinstance(output, DimensionMap) + axis = output.input_dimension + start = output.offset + output.stride * transform.domain.inclusive_min[axis] + if output.stride < 0: + reverse_axes.append(axes.index(axis)) + selection.append( + slice(start + output.stride * (shape[axis] - 1), start + 1, -output.stride) + ) + else: + selection.append(slice(start, start + output.stride * shape[axis], output.stride)) + unused = tuple(axis for axis in range(len(shape)) if axis not in axes) + squeezed = np.squeeze(broadcast, axis=unused) + remaining = sorted(axes) + ordered = np.transpose(squeezed, tuple(remaining.index(axis) for axis in axes)) + if reverse_axes: + # `np.flip` with no axes would hand a zero-rank masked value back as + # the shared `masked` singleton, discarding its payload. + ordered = np.flip(ordered, axis=tuple(reverse_axes)) + source[tuple(selection)] = ordered + return True + + +def _domain_cells(shape: tuple[int, ...]) -> np.ndarray[Any, np.dtype[np.intp]]: + """Every coordinate of a zero-origin domain, one row per cell in C order.""" + if len(shape) == 0: + return np.zeros((1, 0), dtype=np.intp) + return np.indices(shape, dtype=np.intp).reshape(len(shape), -1).T + + +def _last_occurrences(flat: np.ndarray[Any, Any]) -> np.ndarray[Any, np.dtype[np.intp]]: + """Positions to keep so that repeated flat indices take their last value.""" + _, first_in_reversed = np.unique(flat[::-1], return_index=True) + return np.asarray(len(flat) - 1 - first_in_reversed, dtype=np.intp) + + +def _scatter_numpy(source: np.ndarray[Any, Any], transform: IndexTransform, broadcast: Any) -> None: + """One fancy assignment: NumPy sources need no read-modify-write.""" + positions = _domain_cells(transform.domain.shape) + coords = transform.apply_many(positions + np.asarray(transform.domain.inclusive_min)) + flat = np.asarray(np.ravel_multi_index(tuple(coords.T), source.shape), dtype=np.intp) + keep = _last_occurrences(flat) + source[tuple(coords[keep].T)] = broadcast[tuple(positions[keep].T)] + + +def _scatter_planned( + source: Any, + transform: IndexTransform, + broadcast: Any, + projections: Sequence[ChunkProjection], +) -> None: + """One read-modify-write per touched grid cell, through basic slices only.""" + origin = np.asarray(transform.domain.inclusive_min, dtype=np.intp) + for projection in projections: + cells = _domain_cells(projection.chunk_transform.domain.shape) + local = projection.chunk_transform.apply_many(cells) + positions = projection.cell_transform.apply_many(cells) - origin + cell_origin = np.asarray(projection.chunk_domain.inclusive_min, dtype=np.intp) + hull_min = local.min(axis=0) + hull_shape = tuple(int(n) for n in local.max(axis=0) + 1 - hull_min) + box = tuple( + slice(int(lo), int(lo + n)) + for lo, n in zip(cell_origin + hull_min, hull_shape, strict=True) + ) + block = np.array(source[box], dtype=source.dtype, copy=True) + flat = np.asarray( + np.ravel_multi_index(tuple((local - hull_min).T), hull_shape), dtype=np.intp + ) + keep = _last_occurrences(flat) + np.put(block, flat[keep], np.asarray(broadcast[tuple(positions[keep].T)])) + source[box] = block + + +def _scatter_elementwise(source: Any, transform: IndexTransform, broadcast: Any) -> None: + """One integer assignment per element, for sources that cannot be read.""" + origin = transform.domain.inclusive_min + for position in np.ndindex(transform.domain.shape): + point = tuple(start + offset for start, offset in zip(origin, position, strict=True)) + if isinstance(broadcast, np.ma.MaskedArray): + # Masked scalar indexing returns the shared ``masked`` singleton, + # discarding the payload. A zero-dimensional array retains both. + value = np.ma.array( + broadcast.data[position], mask=np.ma.getmaskarray(broadcast)[position] + ) + else: + value = broadcast[position] + source[transform.apply(point)] = value diff --git a/packages/zarr-indexing/tests/test_chunk_resolution.py b/packages/zarr-indexing/tests/test_chunk_resolution.py index ba9c7cc336..8570fd34d2 100644 --- a/packages/zarr-indexing/tests/test_chunk_resolution.py +++ b/packages/zarr-indexing/tests/test_chunk_resolution.py @@ -943,7 +943,7 @@ def test_independent_components_scatter_through_lazy_array(reader_kind: str) -> b = np.array([5, 1, 3])[:, None] c = np.array([6, 0, 6, 2])[None, :] wrapped = LazyArray.from_numpy(source) if reader_kind == "numpy" else LazyArray(source) - view = wrapped.with_parts((2, 3, 2, 2)).lazy.vindex[a, b, c, ...] + view = wrapped.with_parts((2, 3, 2, 2)).vindex[a, b, c, ...] assert ( len( plan_chunks(view.transform, dimension_grids_from_chunks((2, 3, 2, 2), source.shape)) diff --git a/packages/zarr-indexing/tests/test_doc_examples.py b/packages/zarr-indexing/tests/test_doc_examples.py index 802e39e7f8..2af51f4a80 100644 --- a/packages/zarr-indexing/tests/test_doc_examples.py +++ b/packages/zarr-indexing/tests/test_doc_examples.py @@ -35,6 +35,7 @@ import zarr_indexing import zarr_indexing.lazy_array as lazy_array_module from zarr_indexing import IndexTransform, LazyArray, ReadContext +from zarr_indexing.domain import IndexDomain DOCS = Path(__file__).parents[1] / "docs" PACKAGE_ROOT = DOCS.parent @@ -203,9 +204,9 @@ def test_indexing_pattern_matrix_matches_numpy(case: dict[str, Any]) -> None: image = PATTERN_NAMESPACE["image"] lazy = LazyArray.from_numpy(image) accessor = { - "basic": lazy.lazy, - "oindex": lazy.lazy.oindex, - "vindex": lazy.lazy.vindex, + "basic": lazy, + "oindex": lazy.oindex, + "vindex": lazy.vindex, }[case["mode"]] view = accessor[case["selection"]] result = view.result() @@ -242,7 +243,7 @@ def __getitem__(self, key: tuple[Any, ...]) -> object: return self.data[key].tolist() source = ListSlabSource() - result = LazyArray(source).with_parts((2, 3)).lazy.oindex[[3, 1, 1], 1:5:2].result() + result = LazyArray(source).with_parts((2, 3)).oindex[[3, 1, 1], 1:5:2].result() np.testing.assert_array_equal(result, np.array([[16, 18], [6, 8], [6, 8]])) assert len(source.keys) > 0 @@ -251,7 +252,7 @@ def __getitem__(self, key: tuple[Any, ...]) -> object: def test_coordinate_array_example_preserves_order_and_duplicates() -> None: """Coordinate arrays are ordered sequences, not mathematical sets.""" - view = LazyArray.from_numpy(np.arange(6)).with_parts((2,)).lazy.oindex[[4, 1, 1, 3]] + view = LazyArray.from_numpy(np.arange(6)).with_parts((2,)).oindex[[4, 1, 1, 3]] np.testing.assert_array_equal(view.result(), np.array([4, 1, 1, 3])) assembled = np.empty(view.shape, dtype=view.dtype) @@ -264,8 +265,9 @@ def test_documented_partition_transform_is_global_and_projection_is_chunk_local( source = np.arange(8) part = tuple(LazyArray.from_numpy(source).with_parts((4,)).parts())[1] - assert part.view.transform.apply((0,)) == (4,) - assert part.view.array[part.view.transform.apply((0,))] == 4 + assert part.view.transform.domain == IndexDomain((4,), (8,)) + assert part.view.transform.apply((4,)) == (4,) + assert part.view.array[part.view.transform.apply((4,))] == 4 assert part.projection.chunk_transform.apply((0,)) == (0,) @@ -378,7 +380,7 @@ def test_chunk_cache_reader_resolves_a_transform_from_cached_chunks() -> None: reader_type = CACHE_NAMESPACE["SystemMemoryChunkReader"] source = source_type(np.arange(48).reshape(6, 8), chunks=(3, 4)) reader = reader_type(capacity=2) - view = LazyArray(source).with_reader(reader).lazy[1:5, 2] + view = LazyArray(source).with_reader(reader)[1:5, 2] parts = tuple(view.parts()) out = np.empty(view.shape, dtype=source.dtype) for part in parts: diff --git a/packages/zarr-indexing/tests/test_lazy_array.py b/packages/zarr-indexing/tests/test_lazy_array.py index db4483f750..787b408c26 100644 --- a/packages/zarr-indexing/tests/test_lazy_array.py +++ b/packages/zarr-indexing/tests/test_lazy_array.py @@ -26,8 +26,10 @@ ChunkProjection, ConstantMap, DimensionMap, + EagerArrayAdapter, EdgeDimensionGrid, FixedDimension, + IndexDomain, IndexTransform, LazyArray, ReadContext, @@ -341,27 +343,27 @@ def source(request: pytest.FixtureRequest) -> LazyArray: CASES: list[tuple[str, Callable[[LazyArray], LazyArray], Callable[[Any], Any]]] = [ ( "basic-strided-and-int-drop", - lambda a: a.lazy[1:6:2, :, -1], + lambda a: a[1:6:2, :, -1], lambda r: r[1:6:2, :, -1], ), - ("basic-ellipsis", lambda a: a.lazy[..., -2], lambda r: r[..., -2]), - ("basic-negative-scalar", lambda a: a.lazy[-3], lambda r: r[-3]), - ("basic-newaxis", lambda a: a.lazy[None, :, :, None], lambda r: r[None, :, :, None]), - ("basic-empty", lambda a: a.lazy[:, 2:2, :], lambda r: r[:, 2:2, :]), - ("basic-all-scalars", lambda a: a.lazy[-1, 0, 2], lambda r: r[-1, 0, 2]), + ("basic-ellipsis", lambda a: a[..., -2], lambda r: r[..., -2]), + ("basic-negative-scalar", lambda a: a[-3], lambda r: r[-3]), + ("basic-newaxis", lambda a: a[None, :, :, None], lambda r: r[None, :, :, None]), + ("basic-empty", lambda a: a[:, 2:2, :], lambda r: r[:, 2:2, :]), + ("basic-all-scalars", lambda a: a[-1, 0, 2], lambda r: r[-1, 0, 2]), ( "oindex-unsorted-duplicates-multi-axis", - lambda a: a.lazy.oindex[[4, 0, 0, 2], :, [3, 1]], + lambda a: a.oindex[[4, 0, 0, 2], :, [3, 1]], lambda r: outer(r, ([4, 0, 0, 2], slice(None), [3, 1])), ), ( "oindex-negative-and-slice", - lambda a: a.lazy.oindex[:, [-1, 0], 1:4], + lambda a: a.oindex[:, [-1, 0], 1:4], lambda r: outer(r, (slice(None), [-1, 0], slice(1, 4))), ), ( "oindex-boolean-axis", - lambda a: a.lazy.oindex[np.array([True, False, True, False, False, False, True]), :, :], + lambda a: a.oindex[np.array([True, False, True, False, False, False, True]), :, :], lambda r: outer( r, (np.array([True, False, True, False, False, False, True]), slice(None), slice(None)), @@ -369,115 +371,111 @@ def source(request: pytest.FixtureRequest) -> LazyArray: ), ( "vindex-coordinates", - lambda a: a.lazy.vindex[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])], + lambda a: a.vindex[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])], lambda r: r[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])], ), ( "vindex-broadcast-pair", - lambda a: a.lazy.vindex[np.array([[0], [6]]), np.array([1, 4]), np.array([2, 0])], + lambda a: a.vindex[np.array([[0], [6]]), np.array([1, 4]), np.array([2, 0])], lambda r: r[np.array([[0], [6]]), np.array([1, 4]), np.array([2, 0])], ), ( "vindex-negative-coordinates", - lambda a: a.lazy.vindex[np.array([-1, -7]), np.array([-2, 0]), np.array([0, -1])], + lambda a: a.vindex[np.array([-1, -7]), np.array([-2, 0]), np.array([0, -1])], lambda r: r[np.array([-1, -7]), np.array([-2, 0]), np.array([0, -1])], ), - ("vindex-mask", lambda a: a.lazy.vindex[MASK], lambda r: r[MASK]), + ("vindex-mask", lambda a: a.vindex[MASK], lambda r: r[MASK]), ( "compose-basic-then-oindex", - lambda a: a.lazy[1:6].lazy.oindex[[3, 0, 0], [4, 1], :], + lambda a: a[1:6].oindex[[3, 0, 0], [4, 1], :], lambda r: outer(r[1:6], ([3, 0, 0], [4, 1], slice(None))), ), ( "compose-oindex-then-basic-other-axis", - lambda a: a.lazy.oindex[[4, 0, 2], :, :].lazy[:, 1:4, ::2], + lambda a: a.oindex[[4, 0, 2], :, :][:, 1:4, ::2], lambda r: outer(r, ([4, 0, 2], slice(None), slice(None)))[:, 1:4, ::2], ), ( "compose-basic-then-basic", - lambda a: a.lazy[2:, 1:].lazy[::2, -1], + lambda a: a[2:, 1:][::2, -1], lambda r: r[2:, 1:][::2, -1], ), ( "compose-basic-then-vindex", - lambda a: a.lazy[1:6, :, 1:].lazy.vindex[ - np.array([0, 4]), np.array([2, 0]), np.array([1, 2]) - ], + lambda a: a[1:6, :, 1:].vindex[np.array([0, 4]), np.array([2, 0]), np.array([1, 2])], lambda r: r[1:6, :, 1:][np.array([0, 4]), np.array([2, 0]), np.array([1, 2])], ), # Scalar integers are basic indices in the positional dialect: they drop the # axis, in every mode, exactly as NumPy does. - ("oindex-scalar-drops-axis", lambda a: a.lazy.oindex[0], lambda r: r[0]), + ("oindex-scalar-drops-axis", lambda a: a.oindex[0], lambda r: r[0]), ( "oindex-scalar-with-arrays", - lambda a: a.lazy.oindex[0, [1, 2], :], + lambda a: a.oindex[0, [1, 2], :], lambda r: outer(r, (0, [1, 2], slice(None))), ), ( "oindex-scalar-middle-axis", - lambda a: a.lazy.oindex[[3, 1], -1, :], + lambda a: a.oindex[[3, 1], -1, :], lambda r: outer(r, ([3, 1], -1, slice(None))), ), - ("oindex-all-scalars", lambda a: a.lazy.oindex[0, 1, 2], lambda r: r[0, 1, 2]), - ("vindex-all-scalars", lambda a: a.lazy.vindex[0, 1, 2], lambda r: r[0, 1, 2]), + ("oindex-all-scalars", lambda a: a.oindex[0, 1, 2], lambda r: r[0, 1, 2]), + ("vindex-all-scalars", lambda a: a.vindex[0, 1, 2], lambda r: r[0, 1, 2]), ( "vindex-scalar-with-arrays", - lambda a: a.lazy.vindex[0, [1, 2], [3, 0]], + lambda a: a.vindex[0, [1, 2], [3, 0]], lambda r: r[0, [1, 2], [3, 0]], ), ( "vindex-scalar-on-middle-axis", - lambda a: a.lazy.vindex[[1, 2], 0, [3, 0]], + lambda a: a.vindex[[1, 2], 0, [3, 0]], lambda r: r[[1, 2], 0, [3, 0]], ), # Scalar applied to a previously fancy-indexed axis, both orders. ( "compose-oindex-then-scalar", - lambda a: a.lazy.oindex[[3, 1], :, :].lazy[0], + lambda a: a.oindex[[3, 1], :, :][0], lambda r: outer(r, ([3, 1], slice(None), slice(None)))[0], ), ( "compose-oindex-then-scalar-negative", - lambda a: a.lazy.oindex[[3, 1, 1], :, :].lazy[-1, 2], + lambda a: a.oindex[[3, 1, 1], :, :][-1, 2], lambda r: outer(r, ([3, 1, 1], slice(None), slice(None)))[-1, 2], ), ( "compose-scalar-then-oindex", - lambda a: a.lazy[0].lazy.oindex[[3, 1], :], + lambda a: a[0].oindex[[3, 1], :], lambda r: outer(r[0], ([3, 1], slice(None))), ), ( "compose-vindex-then-scalar", - lambda a: a.lazy.vindex[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])].lazy[ - 1 - ], + lambda a: a.vindex[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])][1], lambda r: r[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])][1], ), # Partial vindex whose coordinate arrays are NOT on the leading axes: NumPy # inserts the gathered axis where the (adjacent) advanced indices sat. ( "vindex-trailing-arrays", - lambda a: a.lazy.vindex[..., np.array([1, 4, 0]), np.array([2, 0, 1])], + lambda a: a.vindex[..., np.array([1, 4, 0]), np.array([2, 0, 1])], lambda r: r[..., np.array([1, 4, 0]), np.array([2, 0, 1])], ), ( "vindex-single-trailing-array", - lambda a: a.lazy.vindex[..., np.array([3, 0, 1])], + lambda a: a.vindex[..., np.array([3, 0, 1])], lambda r: r[..., np.array([3, 0, 1])], ), ( "vindex-trailing-mask", - lambda a: a.lazy.vindex[..., TRAILING_MASK], + lambda a: a.vindex[..., TRAILING_MASK], lambda r: r[..., TRAILING_MASK], ), ( "vindex-leading-partial", - lambda a: a.lazy.vindex[np.array([1, 2, 2])], + lambda a: a.vindex[np.array([1, 2, 2])], lambda r: r[np.array([1, 2, 2])], ), ( "compose-basic-then-vindex-trailing", - lambda a: a.lazy[2:, 1:].lazy.vindex[..., np.array([1, 3, 0])], + lambda a: a[2:, 1:].vindex[..., np.array([1, 3, 0])], lambda r: r[2:, 1:][..., np.array([1, 3, 0])], ), # A ConstantMap sitting between a slice and the coordinate arrays: NumPy @@ -485,7 +483,7 @@ def source(request: pytest.FixtureRequest) -> LazyArray: # front of the chunk block even though the arrays are trailing. ( "compose-vindex-trailing-then-scalar", - lambda a: a.lazy.vindex[..., np.array([3, 0])].lazy[2], + lambda a: a.vindex[..., np.array([3, 0])][2], lambda r: r[..., np.array([3, 0])][2], ), # Two fancy axes, then a scalar on the first: the surviving ArrayMap ends up @@ -493,57 +491,57 @@ def source(request: pytest.FixtureRequest) -> LazyArray: # advanced rule bites. ( "compose-oindex-two-axes-then-scalar", - lambda a: a.lazy.oindex[[3, 1, 0], 3:5, [2, 0, 2]].lazy[0], + lambda a: a.oindex[[3, 1, 0], 3:5, [2, 0, 2]][0], lambda r: outer(r, ([3, 1, 0], slice(3, 5), [2, 0, 2]))[0], ), # Negative steps: the positional dialect is NumPy's, including the empty # cases NumPy allows where the transform algebra alone would object. - ("reverse", lambda a: a.lazy[::-1], lambda r: r[::-1]), - ("reverse-every-axis", lambda a: a.lazy[::-1, ::-1, ::-1], lambda r: r[::-1, ::-1, ::-1]), - ("reverse-strided", lambda a: a.lazy[::-2], lambda r: r[::-2]), - ("reverse-nondivisible", lambda a: a.lazy[::-3], lambda r: r[::-3]), - ("reverse-bounded", lambda a: a.lazy[5:1:-1], lambda r: r[5:1:-1]), - ("reverse-negative-start", lambda a: a.lazy[-1:None:-1], lambda r: r[-1:None:-1]), - ("reverse-past-the-start", lambda a: a.lazy[:-8:-1], lambda r: r[:-8:-1]), - ("reverse-empty", lambda a: a.lazy[2:2:-1], lambda r: r[2:2:-1]), + ("reverse", lambda a: a[::-1], lambda r: r[::-1]), + ("reverse-every-axis", lambda a: a[::-1, ::-1, ::-1], lambda r: r[::-1, ::-1, ::-1]), + ("reverse-strided", lambda a: a[::-2], lambda r: r[::-2]), + ("reverse-nondivisible", lambda a: a[::-3], lambda r: r[::-3]), + ("reverse-bounded", lambda a: a[5:1:-1], lambda r: r[5:1:-1]), + ("reverse-negative-start", lambda a: a[-1:None:-1], lambda r: r[-1:None:-1]), + ("reverse-past-the-start", lambda a: a[:-8:-1], lambda r: r[:-8:-1]), + ("reverse-empty", lambda a: a[2:2:-1], lambda r: r[2:2:-1]), # NumPy reads a reversed *positional* interval as empty; only the literal # layer calls it a direction error. - ("reverse-inverted-is-empty", lambda a: a.lazy[2:5:-1], lambda r: r[2:5:-1]), - ("reverse-with-int-drop", lambda a: a.lazy[::-2, 2, ::-1], lambda r: r[::-2, 2, ::-1]), - ("reverse-trailing-axis", lambda a: a.lazy[..., ::-1], lambda r: r[..., ::-1]), + ("reverse-inverted-is-empty", lambda a: a[2:5:-1], lambda r: r[2:5:-1]), + ("reverse-with-int-drop", lambda a: a[::-2, 2, ::-1], lambda r: r[::-2, 2, ::-1]), + ("reverse-trailing-axis", lambda a: a[..., ::-1], lambda r: r[..., ::-1]), ( "compose-reverse-then-reverse", - lambda a: a.lazy[::-1].lazy[::-1], + lambda a: a[::-1][::-1], lambda r: r[::-1][::-1], ), ( "compose-strided-then-reverse", - lambda a: a.lazy[::2].lazy[::-1], + lambda a: a[::2][::-1], lambda r: r[::2][::-1], ), ( "compose-reverse-then-strided", - lambda a: a.lazy[::-1].lazy[::2], + lambda a: a[::-1][::2], lambda r: r[::-1][::2], ), ( "compose-reverse-then-oindex", - lambda a: a.lazy[::-1].lazy.oindex[[3, 0, 0], :, :], + lambda a: a[::-1].oindex[[3, 0, 0], :, :], lambda r: outer(r[::-1], ([3, 0, 0], slice(None), slice(None))), ), ( "compose-oindex-then-reverse", - lambda a: a.lazy.oindex[[3, 1, 2], :, :].lazy[::-1], + lambda a: a.oindex[[3, 1, 2], :, :][::-1], lambda r: outer(r, ([3, 1, 2], slice(None), slice(None)))[::-1], ), ( "compose-reverse-then-vindex", - lambda a: a.lazy[::-1].lazy.vindex[..., np.array([1, 3, 0])], + lambda a: a[::-1].vindex[..., np.array([1, 3, 0])], lambda r: r[::-1][..., np.array([1, 3, 0])], ), ( "compose-vindex-trailing-then-scalar-and-slice", - lambda a: a.lazy.vindex[..., np.array([3, 0, 1])].lazy[-1, 1:4], + lambda a: a.vindex[..., np.array([3, 0, 1])][-1, 1:4], lambda r: r[..., np.array([3, 0, 1])][-1, 1:4], ), # A downward walk that begins off the front of the axis selects nothing. @@ -551,17 +549,17 @@ def source(request: pytest.FixtureRequest) -> LazyArray: # carried by an index array rather than by the domain. ( "compose-oindex-then-empty-downward-walk", - lambda a: a.lazy.oindex[np.array([3, 1, 4]), :, :].lazy[-8::-1], + lambda a: a.oindex[np.array([3, 1, 4]), :, :][-8::-1], lambda r: r[np.array([3, 1, 4])][-8::-1], ), ( "compose-vindex-then-empty-downward-walk", - lambda a: a.lazy.vindex[np.array([3, 1]), np.array([2, 0])].lazy[-9::-2], + lambda a: a.vindex[np.array([3, 1]), np.array([2, 0])][-9::-2], lambda r: r[np.array([3, 1]), np.array([2, 0])][-9::-2], ), ( "empty-downward-walk-on-a-plain-axis", - lambda a: a.lazy[-11::-1], + lambda a: a[-11::-1], lambda r: r[-11::-1], ), # A fancy *spelling* whose entries are all slices is not a fancy selection: @@ -570,22 +568,22 @@ def source(request: pytest.FixtureRequest) -> LazyArray: # broadcast (singleton) axes of the existing index array would truncate it. ( "compose-oindex-then-oindex-slices-only", - lambda a: a.lazy.oindex[[4, 0, 0], :, :].lazy.oindex[:, 2:5, 1:], + lambda a: a.oindex[[4, 0, 0], :, :].oindex[:, 2:5, 1:], lambda r: outer(r, ([4, 0, 0], slice(None), slice(None)))[:, 2:5, 1:], ), ( "compose-oindex-then-oindex-slices-only-strided", - lambda a: a.lazy.oindex[:, [3, 1, 1], :].lazy.oindex[1::2, :, ::-1], + lambda a: a.oindex[:, [3, 1, 1], :].oindex[1::2, :, ::-1], lambda r: outer(r, (slice(None), [3, 1, 1], slice(None)))[1::2, :, ::-1], ), ( "compose-vindex-then-oindex-slices-only", - lambda a: a.lazy.vindex[np.array([4, 0, 2]), np.array([1, 3, 0])].lazy.oindex[1:, 2:], + lambda a: a.vindex[np.array([4, 0, 2]), np.array([1, 3, 0])].oindex[1:, 2:], lambda r: r[np.array([4, 0, 2]), np.array([1, 3, 0])][1:, 2:], ), ( "compose-oindex-then-oindex-array-on-its-own-axis", - lambda a: a.lazy.oindex[[4, 0, 0], :, :].lazy.oindex[[2, 0], 3:, :], + lambda a: a.oindex[[4, 0, 0], :, :].oindex[[2, 0], 3:, :], lambda r: outer( outer(r, ([4, 0, 0], slice(None), slice(None))), ([2, 0], slice(3, None), slice(None)), @@ -607,7 +605,7 @@ def test_selection_matches_numpy( assert view.shape == expected.shape assert view.ndim == expected.ndim np.testing.assert_array_equal(np.asarray(view.result()), expected) - # `__getitem__` is eager, and `__array__` routes through `result()`. + # NumPy conversion routes through `result()`. np.testing.assert_array_equal(np.asarray(view), expected) @@ -720,10 +718,10 @@ def _apply_oracle( def _apply_view(view: LazyArray, mode: str, selection: tuple[Any, ...]) -> LazyArray: if mode == "basic": - return view.lazy[selection] + return view[selection] if mode == "orthogonal": - return view.lazy.oindex[selection] - return view.lazy.vindex[selection] + return view.oindex[selection] + return view.vindex[selection] def _random_slices_only(rng: np.random.Generator, shape: tuple[int, ...]) -> tuple[Any, ...]: @@ -847,9 +845,7 @@ def test_a_slice_only_fancy_step_after_a_fancy_step_reads_real_data() -> None: expected = base[np.ix_([0, 2], range(8))][:, 2:8] for parts in (None, (2, 4), (1, 8), (3, 3)): - view = ( - repartition(LazyArray(base), parts).lazy.oindex[np.array([0, 2]), :].lazy.oindex[:, 2:8] - ) + view = repartition(LazyArray(base), parts).oindex[np.array([0, 2]), :].oindex[:, 2:8] assert view.shape == expected.shape, f"parts={parts}" np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=f"{parts}") @@ -865,28 +861,28 @@ def test_a_fancy_step_composes_onto_any_axis_of_a_fancy_view() -> None: rows, cols = np.array([0, 2]), np.array([1, 3, 3]) for parts in (None, (2, 4), (1, 8), (3, 3)): - view = repartition(LazyArray(base), parts).lazy.oindex[rows, :] + view = repartition(LazyArray(base), parts).oindex[rows, :] - composed = view.lazy.oindex[:, cols] + composed = view.oindex[:, cols] expected = base[np.ix_(rows, cols)] assert composed.shape == expected.shape, f"parts={parts}" np.testing.assert_array_equal(np.asarray(composed.result()), expected, err_msg=f"{parts}") - gathered = view.lazy.vindex[np.array([0, 1]), np.array([7, 0])] + gathered = view.vindex[np.array([0, 1]), np.array([7, 0])] np.testing.assert_array_equal( np.asarray(gathered.result()), base[np.ix_(rows, range(8))][[0, 1], [7, 0]], err_msg=f"{parts}", ) - pointwise = repartition(LazyArray(base), parts).lazy.vindex[[0, 1, 2], [0, 2, 3]] + pointwise = repartition(LazyArray(base), parts).vindex[[0, 1, 2], [0, 2, 3]] np.testing.assert_array_equal( - np.asarray(pointwise.lazy.oindex[[2, 0]].result()), + np.asarray(pointwise.oindex[[2, 0]].result()), base[[0, 1, 2], [0, 2, 3]][[2, 0]], err_msg=f"{parts}", ) np.testing.assert_array_equal( - np.asarray(pointwise.lazy.vindex[[1, 1, 0]].result()), + np.asarray(pointwise.vindex[[1, 1, 0]].result()), base[[0, 1, 2], [0, 2, 3]][[1, 1, 0]], err_msg=f"{parts}", ) @@ -903,8 +899,8 @@ def test_a_partial_vindex_after_an_oindex_resolves_the_mixed_transform() -> None expected = base[:, :, [4, 0]][[0, 2], [1, 3]] for parts in (None, (2, 2, 2), (3, 4, 5), (1, 1, 1)): - view = repartition(LazyArray(base), parts).lazy.oindex[:, :, [4, 0]] - composed = view.lazy.vindex[np.array([0, 2]), np.array([1, 3])] + view = repartition(LazyArray(base), parts).oindex[:, :, [4, 0]] + composed = view.vindex[np.array([0, 2]), np.array([1, 3])] assert composed.shape == expected.shape, f"parts={parts}" np.testing.assert_array_equal(np.asarray(composed.result()), expected, err_msg=f"{parts}") @@ -915,9 +911,9 @@ def test_a_boolean_mask_composes_onto_a_fancy_view() -> None: expected = base[[0, 1, 2]][mask] for parts in (None, (2, 2, 2), (1, 4, 5)): - view = repartition(LazyArray(base), parts).lazy.oindex[[0, 1, 2], :, :] + view = repartition(LazyArray(base), parts).oindex[[0, 1, 2], :, :] np.testing.assert_array_equal( - np.asarray(view.lazy.oindex[mask].result()), expected, err_msg=f"{parts}" + np.asarray(view.oindex[mask].result()), expected, err_msg=f"{parts}" ) @@ -929,18 +925,18 @@ def test_an_ellipsis_only_vindex_step_preserves_a_correlated_gather() -> None: base = np.arange(16).reshape(4, 4) for parts in (None, (2, 2), (4, 4), (1, 3)): - pointwise = repartition(LazyArray(base), parts).lazy.vindex[[0, 1, 2], [0, 2, 3]] + pointwise = repartition(LazyArray(base), parts).vindex[[0, 1, 2], [0, 2, 3]] np.testing.assert_array_equal( - np.asarray(pointwise.lazy.vindex[...].result()), + np.asarray(pointwise.vindex[...].result()), base[[0, 1, 2], [0, 2, 3]], err_msg=f"{parts}", ) - planar = repartition(LazyArray(base), parts).lazy.vindex[ + planar = repartition(LazyArray(base), parts).vindex[ np.array([[0], [1]]), np.array([[1], [3]]) ] np.testing.assert_array_equal( - np.asarray(planar.lazy.vindex[..., np.array(0)].result()), + np.asarray(planar.vindex[..., np.array(0)].result()), base[[0, 1], [1, 3]], err_msg=f"{parts}", ) @@ -965,27 +961,27 @@ def test_an_ellipsis_only_vindex_step_preserves_a_correlated_gather() -> None: ] = [ ( "leading-singleton-row", - lambda a: a.lazy.vindex[np.array([[2, 0]])].lazy[:, 0], + lambda a: a.vindex[np.array([[2, 0]])][:, 0], lambda r: r[np.array([[2, 0]])][:, 0], ), ( "trailing-singleton-column", - lambda a: a.lazy.vindex[np.array([[2], [0]])].lazy[0], + lambda a: a.vindex[np.array([[2], [0]])][0], lambda r: r[np.array([[2], [0]])][0], ), ( "repeated-coordinates-over-a-singleton", - lambda a: a.lazy.vindex[np.array([[1, 1]]), np.array([[3, 3]])].lazy[:, 0], + lambda a: a.vindex[np.array([[1, 1]]), np.array([[3, 3]])][:, 0], lambda r: r[np.array([[1, 1]]), np.array([[3, 3]])][:, 0], ), ( "unreferenced-axis-emptied", - lambda a: a.lazy.vindex[np.array([[2, 0]])].lazy[0:0, 0], + lambda a: a.vindex[np.array([[2, 0]])][0:0, 0], lambda r: r[np.array([[2, 0]])][0:0, 0], ), ( "partial-vindex-with-a-residual-slice", - lambda a: a.lazy.vindex[np.array([[2, 0]]), np.array([[1, 3]])].lazy[:, 0, 1:4], + lambda a: a.vindex[np.array([[2, 0]]), np.array([[1, 3]])][:, 0, 1:4], lambda r: r[np.array([[2, 0]]), np.array([[1, 3]])][:, 0, 1:4], ), ] @@ -1033,7 +1029,7 @@ def test_an_unreferenced_domain_axis_of_extent_zero_stays_empty() -> None: data = np.arange(140, dtype=np.int64).reshape(7, 5, 4) coords = np.array([[6], [3], [0]]) for parts in (None, (1, 1, 1), (3, 2, 3), (7, 5, 4)): - view = repartition(LazyArray(data), parts).lazy.vindex[coords, -4].lazy[0, 0:0] + view = repartition(LazyArray(data), parts).vindex[coords, -4][0, 0:0] expected = data[coords, -4][0, 0:0] assert view.shape == expected.shape == (0, 4), f"parts={parts}" np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=f"{parts}") @@ -1046,11 +1042,7 @@ def test_a_zero_length_axis_resolves_the_same_way_under_every_partitioning() -> expected = base[np.ix_([1, 0, 0], np.arange(0, dtype=int))] for parts in (None, ((1, 1, 1), ()), ((3,), ())): - view = ( - repartition(LazyArray(base), parts) - .lazy.oindex[np.array([1, -3, -3]), :] - .lazy.oindex[:, :] - ) + view = repartition(LazyArray(base), parts).oindex[np.array([1, -3, -3]), :].oindex[:, :] assert view.shape == expected.shape, f"parts={parts}" np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=f"{parts}") assert list(view.parts()) == [] @@ -1067,7 +1059,7 @@ def test_an_empty_slice_of_a_length_one_correlated_axis_has_no_parts() -> None: mask = np.array([False, True, False, False, False]) for parts in PARTITIONINGS_1D: - view = repartition(LazyArray(base), parts).lazy.vindex[mask].lazy[1:-2] + view = repartition(LazyArray(base), parts).vindex[mask][1:-2] assert view.shape == (0,), f"parts={parts}" assert list(view.parts()) == [], f"parts={parts}" np.testing.assert_array_equal(np.asarray(view.result()), base[mask][1:-2]) @@ -1078,9 +1070,7 @@ def test_an_empty_slice_of_a_length_one_vindex_pair_has_no_parts() -> None: base = np.arange(35).reshape(7, 5) for parts in (None, (2, 2), (7, 5)): - view = ( - repartition(LazyArray(base), parts).lazy.vindex[np.array([6]), np.array([0])].lazy[0:0] - ) + view = repartition(LazyArray(base), parts).vindex[np.array([6]), np.array([0])][0:0] assert view.shape == (0,), f"parts={parts}" assert list(view.parts()) == [], f"parts={parts}" np.testing.assert_array_equal( @@ -1106,7 +1096,7 @@ def test_a_correlated_view_narrowed_to_one_point_has_parts_of_the_views_rank() - for parts in (None, (2, 2, 2), (3, 3, 3), (7, 5, 4)): for selection, tail in cases: expected = base[selection][tail] - view = repartition(LazyArray(base), parts).lazy.vindex[selection].lazy[tail] + view = repartition(LazyArray(base), parts).vindex[selection][tail] assert view.shape == expected.shape, f"parts={parts}, {selection}" assembled = np.zeros(view.shape, dtype=view.dtype) @@ -1121,8 +1111,9 @@ def test_a_correlated_view_narrowed_to_one_point_has_parts_of_the_views_rank() - np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=err) -def test_eager_getitem_returns_data(source: LazyArray) -> None: - """`arr[...]` reads immediately, like `numpy.ndarray.__getitem__`.""" +def test_getitem_returns_a_view(source: LazyArray) -> None: + """`arr[...]` returns a view that materializes to the selected values.""" + assert isinstance(source[1:3, ::2, -1], LazyArray) np.testing.assert_array_equal(np.asarray(source[1:3, ::2, -1]), reference()[1:3, ::2, -1]) @@ -1142,7 +1133,7 @@ def test_forwards_array_attributes(source: LazyArray) -> None: def test_view_shape_comes_from_the_transform(source: LazyArray) -> None: """A view reports its own shape, not the wrapped array's.""" - view = source.lazy[1:6:2, :, -1] + view = source[1:6:2, :, -1] assert view.shape == (3, 5) assert view.ndim == 2 assert view.size == 15 @@ -1169,7 +1160,7 @@ def test_scalar_is_basic_even_when_a_slice_separates_it_from_the_arrays() -> Non selection reads as `x[0][:, i]`. """ data = reference() - view = make_source("numpy-uniform-parts").lazy.vindex[0, ..., np.array([3, 0])] + view = make_source("numpy-uniform-parts").vindex[0, ..., np.array([3, 0])] np.testing.assert_array_equal(np.asarray(view.result()), data[0][..., np.array([3, 0])]) assert view.shape == data[0][..., np.array([3, 0])].shape assert view.shape != data[0, ..., np.array([3, 0])].shape @@ -1177,7 +1168,7 @@ def test_scalar_is_basic_even_when_a_slice_separates_it_from_the_arrays() -> Non def test_zero_dimensional_result_is_an_array(source: LazyArray) -> None: """Both resolvers agree on the kind of a zero-rank result.""" - result = source.lazy[0, 1, 2].result() + result = source[0, 1, 2].result() assert isinstance(result, np.ndarray) assert result.ndim == 0 assert result[()] == reference()[0, 1, 2] @@ -1204,7 +1195,7 @@ def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: data = reference() inner = RecordingDelegatingReader("inner", numpy_reader) outer = RecordingDelegatingReader("outer", inner) - view = LazyArray(data).with_reader(outer).lazy[1:6:2, ::-1, 1].unpartitioned() + view = LazyArray(data).with_reader(outer)[1:6:2, ::-1, 1].unpartitioned() result = view.result() @@ -1239,7 +1230,7 @@ def test_view_operations_preserve_the_reader_without_reading() -> None: reader = RecordingReader() base = LazyArray(data).with_reader(reader) views = ( - base.lazy[1:5], + base[1:5], base.with_parts((2, 2, 2)), base.with_parts_per_axis(((3, 3, 1), (2, 3), (1, 3))), base.unpartitioned(), @@ -1281,7 +1272,7 @@ def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: def test_reader_is_called_once_per_touched_part() -> None: data = np.arange(48).reshape(6, 8) reader = RecordingReader() - view = LazyArray(data).with_reader(reader).with_parts((3, 4)).lazy[1:5, 2] + view = LazyArray(data).with_reader(reader).with_parts((3, 4))[1:5, 2] np.testing.assert_array_equal(view.result(), data[1:5, 2]) assert len(reader.calls) == len(tuple(view.parts())) == 2 assert all( @@ -1303,14 +1294,14 @@ def test_partition_reader_receives_global_transform_and_existing_projection() -> def test_an_empty_result_does_not_call_the_reader() -> None: reader = RecordingReader() - result = LazyArray(reference()).with_reader(reader).lazy[:, 0:0, :].result() + result = LazyArray(reference()).with_reader(reader)[:, 0:0, :].result() assert result.shape == (7, 0, 4) assert reader.calls == [] def test_rectangular_parts_write_into_result_views() -> None: reader = BufferRecordingReader() - view = LazyArray(reference()).with_reader(reader).with_parts((2, 2, 2)).lazy[1:6, 1:4] + view = LazyArray(reference()).with_reader(reader).with_parts((2, 2, 2))[1:6, 1:4] view.result() assert reader.owns_data assert not any(reader.owns_data) @@ -1318,12 +1309,7 @@ def test_rectangular_parts_write_into_result_views() -> None: def test_fancy_part_placement_uses_owned_dense_temporaries() -> None: reader = BufferRecordingReader() - view = ( - LazyArray(reference()) - .with_reader(reader) - .with_parts((2, 2, 2)) - .lazy.oindex[[6, 1, 1], :, :] - ) + view = LazyArray(reference()).with_reader(reader).with_parts((2, 2, 2)).oindex[[6, 1, 1], :, :] np.testing.assert_array_equal(view.result(), reference()[np.ix_([6, 1, 1], range(5), range(4))]) assert any(reader.owns_data) @@ -1372,7 +1358,7 @@ def test_malformed_discovered_parts_are_ignored() -> None: for bogus in (((3, 3), (5,), (4,)), (3, 2), "nope", (3.5, 2, 2)): wrapped = LazyArray(ForeignArray(data, bogus)) assert len(list(wrapped.parts())) == 1 - np.testing.assert_array_equal(np.asarray(wrapped.lazy[1:3].result()), data[1:3]) + np.testing.assert_array_equal(np.asarray(wrapped[1:3].result()), data[1:3]) def test_discovered_parts_come_from_the_source_vocabulary() -> None: @@ -1391,37 +1377,37 @@ def test_discovered_parts_come_from_the_source_vocabulary() -> None: # intervals of the wrapped (7, 5, 4) array, computed by hand. BOX_CASES: list[tuple[str, Callable[[LazyArray], LazyArray], bool, Any]] = [ ("identity", lambda a: a, True, ((0, 7), (0, 5), (0, 4))), - ("basic-slice", lambda a: a.lazy[1:6, :, 1:3], True, ((1, 6), (0, 5), (1, 3))), + ("basic-slice", lambda a: a[1:6, :, 1:3], True, ((1, 6), (0, 5), (1, 3))), # Stride 2 over axis 1 touches 0, 2, 4; the hull is the closed span. - ("strided", lambda a: a.lazy[::3, ::2, :], True, ((0, 7), (0, 5), (0, 4))), - ("int-drop", lambda a: a.lazy[2, :, -1], True, ((2, 3), (0, 5), (3, 4))), - ("all-scalars", lambda a: a.lazy[0, 1, 2], True, ((0, 1), (1, 2), (2, 3))), - ("negative-and-open", lambda a: a.lazy[-2:], True, ((5, 7), (0, 5), (0, 4))), + ("strided", lambda a: a[::3, ::2, :], True, ((0, 7), (0, 5), (0, 4))), + ("int-drop", lambda a: a[2, :, -1], True, ((2, 3), (0, 5), (3, 4))), + ("all-scalars", lambda a: a[0, 1, 2], True, ((0, 1), (1, 2), (2, 3))), + ("negative-and-open", lambda a: a[-2:], True, ((5, 7), (0, 5), (0, 4))), ( "oindex", - lambda a: a.lazy.oindex[[4, 0, 0], :, [3, 1]], + lambda a: a.oindex[[4, 0, 0], :, [3, 1]], False, ((0, 5), (0, 5), (1, 4)), ), ( "vindex", - lambda a: a.lazy.vindex[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])], + lambda a: a.vindex[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])], False, ((0, 7), (0, 5), (0, 3)), ), - ("mask", lambda a: a.lazy.vindex[MASK], False, ((0, 7), (0, 5), (0, 4))), + ("mask", lambda a: a.vindex[MASK], False, ((0, 7), (0, 5), (0, 4))), # Composition preserves the category in both directions. - ("box-of-box", lambda a: a.lazy[1:6].lazy[:, 1:3], True, ((1, 6), (1, 3), (0, 4))), + ("box-of-box", lambda a: a[1:6][:, 1:3], True, ((1, 6), (1, 3), (0, 4))), ( "box-after-fancy", - lambda a: a.lazy.oindex[[4, 0, 2], :, :].lazy[0:2], + lambda a: a.oindex[[4, 0, 2], :, :][0:2], False, ((0, 5), (0, 5), (0, 4)), ), - ("empty", lambda a: a.lazy[2:2], True, None), + ("empty", lambda a: a[2:2], True, None), ( "empty-fancy", - lambda a: a.lazy.oindex[np.array([], dtype=np.intp), :, :], + lambda a: a.oindex[np.array([], dtype=np.intp), :, :], False, None, ), @@ -1448,7 +1434,7 @@ def test_is_box_and_bounding_box( def test_a_unit_stride_box_is_dense_in_its_bounding_box() -> None: """Stride 1 everywhere: the hull is exactly what the view selects.""" data = reference() - view = make_source("numpy-uniform-parts").lazy[1:6, :, 1:3] + view = make_source("numpy-uniform-parts")[1:6, :, 1:3] assert view.is_box assert view.strides() == (1, 1, 1) bounds = view.bounding_box() @@ -1462,7 +1448,7 @@ def test_a_unit_stride_box_is_dense_in_its_bounding_box() -> None: def test_a_strided_box_is_sparse_in_its_bounding_box() -> None: """Stride > 1: the hull is a superset, and `strides()` is what says by how much.""" data = reference() - view = make_source("numpy-uniform-parts").lazy[1:6, ::2, :] + view = make_source("numpy-uniform-parts")[1:6, ::2, :] assert view.is_box assert view.strides() == (1, 2, 1) bounds = view.bounding_box() @@ -1482,20 +1468,20 @@ def test_a_strided_box_is_sparse_in_its_bounding_box() -> None: def test_strides_are_none_for_a_query() -> None: - view = make_source("numpy-uniform-parts").lazy.oindex[[5, 1], :, :] + view = make_source("numpy-uniform-parts").oindex[[5, 1], :, :] assert not view.is_box assert view.strides() is None def test_an_integer_indexed_dimension_has_stride_one() -> None: - view = make_source("numpy-uniform-parts").lazy[2, ::3, :] + view = make_source("numpy-uniform-parts")[2, ::3, :] assert view.strides() == (1, 3, 1) assert view.bounding_box() == ((2, 3), (0, 4), (0, 4)) def test_a_query_bounding_box_is_only_a_hull() -> None: """For a fancy selection the box is a superset, and `is_box` says so.""" - view = make_source("numpy-uniform-parts").lazy.oindex[[5, 1], :, :] + view = make_source("numpy-uniform-parts").oindex[[5, 1], :, :] assert not view.is_box assert view.bounding_box() == ((1, 6), (0, 5), (0, 4)) # The hull spans 5 rows; the selection touches 2 of them. @@ -1515,7 +1501,7 @@ def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: def test_prepared_part_validation_allocates_boolean_coverage_bitmap( monkeypatch: pytest.MonkeyPatch, ) -> None: - view = LazyArray.from_numpy(reference()).with_parts(PART_SHAPE).lazy[1:6, ::2, 1:] + view = LazyArray.from_numpy(reference()).with_parts(PART_SHAPE)[1:6, ::2, 1:] parts = tuple(view.parts()) allocations: list[tuple[tuple[int, ...], np.dtype[Any]]] = [] real_zeros = np.zeros @@ -1538,35 +1524,35 @@ def recording_zeros(shape: Any, *args: Any, **kwargs: Any) -> np.ndarray[Any, An pytest.param( np.arange(8), (3,), - lambda array: array.lazy[1:7:2], + lambda array: array[1:7:2], np.array([1, 3, 5]), id="basic-reordered-parts", ), pytest.param( np.arange(8), (3,), - lambda array: array.lazy[3], + lambda array: array[3], np.array(3), id="scalar", ), pytest.param( np.arange(20).reshape(4, 5), (2, 3), - lambda array: array.lazy.oindex[[3, 1, 1], [4, 0]], + lambda array: array.oindex[[3, 1, 1], [4, 0]], np.array([[19, 15], [9, 5], [9, 5]]), id="orthogonal-fancy", ), pytest.param( np.arange(20).reshape(4, 5), (2, 3), - lambda array: array.lazy.vindex[[3, 1, 1], [4, 0, 4]], + lambda array: array.vindex[[3, 1, 1], [4, 0, 4]], np.array([19, 5, 9]), id="correlated-fancy", ), pytest.param( np.arange(8), (3,), - lambda array: array.lazy[2:2], + lambda array: array[2:2], np.array([], dtype=np.int64), id="empty", ), @@ -1593,9 +1579,9 @@ def unexpected_replan(self: LazyArray) -> Any: def test_result_rejects_prepared_parts_owned_by_another_view() -> None: data = reference() base = LazyArray.from_numpy(data).with_reader(_ReadMustNotRun()).with_parts(PART_SHAPE) - view = base.lazy[1:6, ::2, 1:] + view = base[1:6, ::2, 1:] owned_parts = tuple(view.parts()) - foreign_parts = tuple(base.lazy[1:6, ::2, 1:].parts()) + foreign_parts = tuple(base[1:6, ::2, 1:].parts()) mixed_parts = (owned_parts[0], *foreign_parts[1:]) with pytest.raises(ValueError, match="prepared parts do not belong to this view"): @@ -1607,8 +1593,7 @@ def test_result_rejects_prepared_parts_that_do_not_tile_the_view() -> None: view = ( LazyArray.from_numpy(data) .with_reader(_ReadMustNotRun()) - .with_parts(PART_SHAPE) - .lazy[1:6, ::2, 1:] + .with_parts(PART_SHAPE)[1:6, ::2, 1:] ) parts = tuple(view.parts()) @@ -1658,9 +1643,7 @@ def test_result_rejects_empty_prepared_parts_for_a_nonempty_view() -> None: def test_result_accepts_empty_prepared_parts_for_an_empty_view() -> None: - view = ( - LazyArray.from_numpy(np.arange(8)).with_reader(_ReadMustNotRun()).with_parts((4,)).lazy[0:0] - ) + view = LazyArray.from_numpy(np.arange(8)).with_reader(_ReadMustNotRun()).with_parts((4,))[0:0] np.testing.assert_array_equal(view.result(parts=()), np.array([], dtype=np.int64)) @@ -1670,14 +1653,14 @@ def test_result_accepts_empty_prepared_parts_for_an_empty_view() -> None: "build", [ lambda a: a, - lambda a: a.lazy[1:6, :, 1:], - lambda a: a.lazy.oindex[[4, 0, 0], :, [3, 1]], - lambda a: a.lazy.vindex[..., np.array([1, 4, 0]), np.array([2, 0, 1])], + lambda a: a[1:6, :, 1:], + lambda a: a.oindex[[4, 0, 0], :, [3, 1]], + lambda a: a.vindex[..., np.array([1, 4, 0]), np.array([2, 0, 1])], # A reversing view drives the negative-stride branches of # `_intersect_dimension_map` and chunk projection, which were # written defensively long before anything could reach them. - lambda a: a.lazy[::-1, ::-2, :], - lambda a: a.lazy[5:1:-1, :, ::-1], + lambda a: a[::-1, ::-2, :], + lambda a: a[5:1:-1, :, ::-1], ], ids=["identity", "basic", "oindex", "vindex", "reversed", "reversed-bounded"], ) @@ -1724,8 +1707,8 @@ def _transform_point(transform: IndexTransform, point: tuple[int, ...]) -> tuple @pytest.mark.parametrize( "build", [ - lambda array: array.lazy.oindex[[6, 0, 2], :, [3, 1]], - lambda array: array.lazy.vindex[..., np.array([4, 0, 4]), np.array([3, 1, 1])], + lambda array: array.oindex[[6, 0, 2], :, [3, 1]], + lambda array: array.vindex[..., np.array([4, 0, 4]), np.array([3, 1, 1])], ], ids=["orthogonal", "vectorized"], ) @@ -1775,9 +1758,13 @@ def test_nonfirst_partition_transform_directly_addresses_its_array() -> None: part = list(LazyArray.from_numpy(source).with_parts((4,)).parts())[1] assert part.box == ((4, 8),) - assert part.view.transform.apply((0,)) == (4,) - assert part.view.array[part.view.transform.apply((0,))] == 4 + # A box part keeps the request's literal coordinates, so its domain says + # where it sits; the transform addresses the raw source from there. + assert part.view.transform.domain == IndexDomain((4,), (8,)) + assert part.view.transform.apply((4,)) == (4,) + assert part.view.array[part.view.transform.apply((4,))] == 4 assert part.view.result()[0] == 4 + assert part.view[0].result() == 4 assert part.projection.chunk_transform.apply((0,)) == (0,) @@ -1786,14 +1773,14 @@ def test_partition_token_encodes_its_public_global_transform() -> None: source = np.arange(8) base = LazyArray.from_numpy(source) partition_view = list(base.with_parts((4,)).parts())[1].view - direct_view = base.lazy[4:8] + direct_view = base[4:8] assert partition_view.__dask_tokenize__() == direct_view.__dask_tokenize__() def test_parts_resolve_independently_and_concurrently() -> None: """Each part's `array` is a standalone `LazyArray` with no shared mutable state.""" - view = make_source("zarr").lazy[1:7, :, 1:].with_parts((2, 2, 2)) + view = make_source("zarr")[1:7, :, 1:].with_parts((2, 2, 2)) parts = list(view.parts()) assert len(parts) > 1 @@ -1812,15 +1799,15 @@ def test_parts_report_completeness() -> None: assert all(part.is_complete for part in array.parts()) # Boxes along axis 2 are [0, 3) and [3, 4); dropping column 0 leaves the # first partially covered and the second whole. - trimmed = {part.base_coords[2]: part.is_complete for part in array.lazy[:, :, 1:].parts()} + trimmed = {part.base_coords[2]: part.is_complete for part in array[:, :, 1:].parts()} assert trimmed == {0: False, 1: True} # A fancy axis is always reported incomplete. - assert not any(part.is_complete for part in array.lazy.oindex[[4, 0, 0], :, :].parts()) + assert not any(part.is_complete for part in array.oindex[[4, 0, 0], :, :].parts()) def test_partition_boxes_are_global_and_tile_the_base() -> None: """Both the selected hull and the whole partition box use global coordinates.""" - view = make_source("numpy-uniform-parts").lazy[1:6, :, 1:] + view = make_source("numpy-uniform-parts")[1:6, :, 1:] parts = {part.base_coords: part for part in view.parts()} # The reviewer's repro: two parts of the same view now expose distinct @@ -1867,7 +1854,7 @@ def test_an_unpartitioned_wrapper_has_a_single_whole_array_part() -> None: def test_with_parts_keeps_the_view_and_the_base() -> None: - view = make_source("zarr").lazy[1:6, ::2] + view = make_source("zarr")[1:6, ::2] repartitioned = view.with_parts((2, 1, 4)) assert repartitioned.shape == view.shape assert repartitioned.array is view.array @@ -1880,7 +1867,7 @@ def test_with_parts_keeps_the_view_and_the_base() -> None: def test_with_parts_none_forces_one_shot_resolution() -> None: - view = make_source("zarr").lazy.oindex[[4, 0, 0], :, :] + view = make_source("zarr").oindex[[4, 0, 0], :, :] whole = view.unpartitioned() assert len(list(whole.parts())) == 1 np.testing.assert_array_equal(np.asarray(whole.result()), np.asarray(view.result())) @@ -1921,13 +1908,13 @@ def test_dask_token_is_deterministic_and_discriminating() -> None: tokens = { "base": base.__dask_tokenize__(), - "view": base.lazy[1:3].__dask_tokenize__(), - "other view": base.lazy[2:4].__dask_tokenize__(), + "view": base[1:3].__dask_tokenize__(), + "other view": base[2:4].__dask_tokenize__(), "other data": LazyArray(data + 1).__dask_tokenize__(), } assert len({repr(token) for token in tokens.values()}) == len(tokens) # Equivalent transforms reached different ways still token alike. - assert base.lazy[1:5].lazy[0:2].__dask_tokenize__() == base.lazy[1:3].__dask_tokenize__() + assert base[1:5][0:2].__dask_tokenize__() == base[1:3].__dask_tokenize__() def test_reader_and_partitioning_do_not_change_dask_identity() -> None: @@ -1941,27 +1928,28 @@ def test_reader_and_partitioning_do_not_change_dask_identity() -> None: @pytest.mark.parametrize("reader", [basic_reader, numpy_reader, DelegatingReader(numpy_reader)]) def test_reader_survives_pickle(reader: Reader) -> None: - view = LazyArray(reference()).with_reader(reader).lazy[1:5, ::2] + view = LazyArray(reference()).with_reader(reader)[1:5, ::2] restored = pickle.loads(pickle.dumps(view)) assert type(restored.reader) is type(reader) np.testing.assert_array_equal(restored.result(), view.result()) -def test_iteration_yields_eager_slices(source: LazyArray) -> None: - rows = list(source.lazy[2:5]) +def test_iteration_yields_lazy_slices(source: LazyArray) -> None: + rows = list(source[2:5]) assert len(rows) == 3 for row, expected in zip(rows, reference()[2:5], strict=True): + assert isinstance(row, LazyArray) np.testing.assert_array_equal(np.asarray(row), expected) def test_iteration_over_a_zero_dimensional_view_is_rejected() -> None: with pytest.raises(TypeError, match="iteration over a 0-d array"): - iter(make_source("numpy-uniform-parts").lazy[0, 0, 0]) + iter(make_source("numpy-uniform-parts")[0, 0, 0]) def test_len_of_a_zero_dimensional_view_is_rejected() -> None: with pytest.raises(TypeError, match="len\\(\\) of unsized object"): - len(make_source("numpy-uniform-parts").lazy[0, 0, 0]) + len(make_source("numpy-uniform-parts")[0, 0, 0]) @pytest.mark.parametrize( @@ -1978,7 +1966,7 @@ def test_len_of_a_zero_dimensional_view_is_rejected() -> None: def test_scalar_conversions_match_numpy(convert: Any, selection: Any) -> None: """Size-1 conversions delegate to NumPy, values and all.""" data = reference() - view = make_source("numpy-uniform-parts").lazy[selection] + view = make_source("numpy-uniform-parts")[selection] assert convert(view) == convert(data[selection]) @@ -1997,7 +1985,7 @@ def test_scalar_conversions_raise_what_numpy_raises( convert: Any, selection: Any, error: type[Exception] ) -> None: data = reference() - view = make_source("numpy-uniform-parts").lazy[selection] + view = make_source("numpy-uniform-parts")[selection] with pytest.raises(error): convert(view) with pytest.raises(error): @@ -2006,7 +1994,7 @@ def test_scalar_conversions_raise_what_numpy_raises( def test_pickle_round_trip() -> None: """A wrapper over a picklable base survives a round trip, view and parts intact.""" - view = LazyArray(reference()).with_parts((2, 2, 2)).lazy[1:6, ::2].lazy.oindex[[3, 0, 0], :, :] + view = LazyArray(reference()).with_parts((2, 2, 2))[1:6, ::2].oindex[[3, 0, 0], :, :] restored = pickle.loads(pickle.dumps(view)) assert restored.shape == view.shape np.testing.assert_array_equal(np.asarray(restored.result()), np.asarray(view.result())) @@ -2022,12 +2010,12 @@ def test_dask_from_array_roundtrip() -> None: da = pytest.importorskip("dask.array") source = make_source("zarr") - lazy = da.from_array(source) + lazy = da.from_array(EagerArrayAdapter(source)) np.testing.assert_array_equal(lazy.compute(), reference()) # dask chooses its own blocks; the wrapper reads each of them through its # own parts, so the two partitionings need not agree. - blocked = da.from_array(source, chunks=(4, 3, 3)) + blocked = da.from_array(EagerArrayAdapter(source), chunks=(4, 3, 3)) assert blocked.chunks == ((4, 3), (3, 2), (3, 1)) np.testing.assert_array_equal(blocked[2:, ::2].compute(), reference()[2:, ::2]) @@ -2039,7 +2027,7 @@ def test_dask_from_array_roundtrip() -> None: def test_boolean_scalar_is_rejected() -> None: with pytest.raises(IndexError, match="boolean scalars are not valid indices"): - make_source("numpy-uniform-parts").lazy[True] + make_source("numpy-uniform-parts")[True] @pytest.mark.parametrize( @@ -2061,9 +2049,9 @@ def test_positional_selectors_support_the_index_protocol( ) -> None: source = LazyArray.from_numpy(np.arange(8)) if mode == "basic": - view = source.lazy[selection] + view = source[selection] else: - view = getattr(source.lazy, "oindex" if mode == "orthogonal" else "vindex")[selection] + view = getattr(source, "oindex" if mode == "orthogonal" else "vindex")[selection] result = np.asarray(view.result()) assert result.shape == expected.shape @@ -2072,47 +2060,47 @@ def test_positional_selectors_support_the_index_protocol( def test_positional_selector_rejects_int_only_objects() -> None: with pytest.raises(IndexError, match="unsupported selection type"): - LazyArray.from_numpy(np.arange(8)).lazy[IntOnly()] + LazyArray.from_numpy(np.arange(8))[IntOnly()] def test_positional_selector_propagates_malformed_index_protocol() -> None: with pytest.raises(TypeError, match="__index__ returned non-int"): - LazyArray.from_numpy(np.arange(8)).lazy[BadIndex()] + LazyArray.from_numpy(np.arange(8))[BadIndex()] def test_positional_slice_propagates_malformed_index_protocol() -> None: with pytest.raises(TypeError, match="__index__ returned non-int"): - LazyArray.from_numpy(np.arange(8)).lazy[:: BadIndex()] + LazyArray.from_numpy(np.arange(8))[:: BadIndex()] def test_protocol_objects_inside_an_index_array_remain_invalid() -> None: selection = np.array([IndexLike(2)], dtype=object) with pytest.raises(IndexError, match="integer or boolean"): - LazyArray.from_numpy(np.arange(8)).lazy.oindex[selection] + LazyArray.from_numpy(np.arange(8)).oindex[selection] def test_mask_shape_must_match_the_view() -> None: array = make_source("numpy-uniform-parts") with pytest.raises(IndexError, match="boolean index has shape"): - array.lazy.vindex[np.ones((2, 2, 2), dtype=bool)] + array.vindex[np.ones((2, 2, 2), dtype=bool)] def test_scalar_index_out_of_bounds() -> None: with pytest.raises(IndexError, match="index 7 is out of bounds for axis 0 with size 7"): - make_source("numpy-uniform-parts").lazy[7] + make_source("numpy-uniform-parts")[7] def test_scalar_index_out_of_bounds_in_a_view() -> None: """Bounds are the *view's*, not the wrapped array's.""" - array = make_source("numpy-uniform-parts").lazy[1:4] + array = make_source("numpy-uniform-parts")[1:4] with pytest.raises(IndexError, match="index 3 is out of bounds for axis 0 with size 3"): - array.lazy[3] + array[3] def test_index_array_out_of_bounds() -> None: array = make_source("numpy-uniform-parts") with pytest.raises(IndexError, match="index 99 is out of bounds for axis 0 with size 7"): - array.lazy.oindex[[0, 99], :, :] + array.oindex[[0, 99], :, :] @pytest.mark.parametrize("mode", ["oindex", "vindex"]) @@ -2122,12 +2110,12 @@ def test_unsigned_index_array_beyond_intp_is_out_of_bounds(mode: str) -> None: index = np.array([2**64 - 1], dtype=np.uint64) selection = (index, 0, 0) if mode == "vindex" else (index, slice(None), slice(None)) with pytest.raises(IndexError, match=f"index {2**64 - 1} is out of bounds for axis 0"): - getattr(array.lazy, mode)[selection] + getattr(array, mode)[selection] def test_too_many_indices() -> None: with pytest.raises(IndexError, match="too many indices"): - make_source("numpy-uniform-parts").lazy[0, 0, 0, 0] + make_source("numpy-uniform-parts")[0, 0, 0, 0] def test_copy_false_conversion_is_rejected() -> None: @@ -2143,32 +2131,33 @@ def test_copy_false_conversion_is_rejected() -> None: def test_reversed_box_reports_a_positive_stride(source: LazyArray) -> None: """A reversal is still a box; `strides()` is magnitudes, so it matches the forward twin.""" - reversed_view = source.lazy[::-2] - forward = source.lazy[::2] + reversed_view = source[::-2] + forward = source[::2] assert reversed_view.is_box assert reversed_view.strides() == forward.strides() == (2, 1, 1) assert reversed_view.bounding_box() == ((0, 7), (0, 5), (0, 4)) -def test_a_reversed_view_is_re_based_to_origin_zero() -> None: - """The literal domain of a reversal is negative; the positional dialect hides it.""" - view = make_source("numpy-whole").lazy[::-1] - # The algebra's own answer keeps the source frame. +def test_a_reversed_view_keeps_its_negative_literal_domain() -> None: + """The literal domain of a reversal is negative; positional keys hide it.""" + view = make_source("numpy-whole")[::-1] + # The algebra's answer keeps the source frame, and so does the wrapper. assert IndexTransform.from_shape(SHAPE)[::-1].domain.inclusive_min[0] == -6 - # The wrapper re-bases, so positions start at 0 as NumPy expects. - assert view.transform.domain.inclusive_min == (0, 0, 0) + assert view.transform.domain.inclusive_min == (-6, 0, 0) assert view.shape == SHAPE np.testing.assert_array_equal(np.asarray(view.result()), reference()[::-1]) + # Position 0 is still the first element, whatever the domain says. + np.testing.assert_array_equal(view[0].result(), reference()[-1]) def test_zero_step_is_rejected() -> None: with pytest.raises(ValueError, match="step cannot be zero"): - make_source("numpy-whole").lazy[::0] + make_source("numpy-whole")[::0] def test_reversed_positional_interval_is_empty_not_an_error() -> None: """NumPy's rule at the boundary; the literal layer keeps TensorStore's.""" - view = make_source("numpy-uniform-parts").lazy[2:5:-1] + view = make_source("numpy-uniform-parts")[2:5:-1] assert view.shape == (0, 5, 4) np.testing.assert_array_equal(np.asarray(view.result()), reference()[2:5:-1]) # Literal coordinates, on the other hand, call it a direction error. @@ -2178,7 +2167,7 @@ def test_reversed_positional_interval_is_empty_not_an_error() -> None: def test_negative_step_over_a_fancy_axis_reverses_the_coordinates(source: LazyArray) -> None: """Reversing a gathered axis materializes, rather than attaching a stride.""" - view = source.lazy.oindex[[3, 1, 2], :, :].lazy[::-1] + view = source.oindex[[3, 1, 2], :, :][::-1] expected = outer(reference(), ([3, 1, 2], slice(None), slice(None)))[::-1] np.testing.assert_array_equal(np.asarray(view.result()), expected) m = view.transform.output[0] @@ -2224,8 +2213,8 @@ def __getitem__(self, key: Any) -> Any: @pytest.mark.parametrize( "build", [ - lambda a: a.lazy[:, 2:2, :], - lambda a: a.lazy.vindex[np.array([[6], [3], [0]]), -4].lazy[0, 0:0], + lambda a: a[:, 2:2, :], + lambda a: a.vindex[np.array([[6], [3], [0]]), -4][0, 0:0], ], ids=["ordinary", "unreferenced-axis"], ) @@ -2272,10 +2261,10 @@ def __getitem__(self, selection: Any) -> Any: @pytest.mark.parametrize( ("build", "description"), [ - (lambda a: a.lazy[1:3, :, :], "a basic slice"), - (lambda a: a.lazy[:, :, :], "the whole array"), - (lambda a: a.lazy[::-1, :, :], "a reversal"), - (lambda a: a.lazy.oindex[[2, 0], :, :], "a gather"), + (lambda a: a[1:3, :, :], "a basic slice"), + (lambda a: a[:, :, :], "the whole array"), + (lambda a: a[::-1, :, :], "a reversal"), + (lambda a: a.oindex[[2, 0], :, :], "a gather"), ], ) def test_materializing_never_hands_back_the_wrapped_array( @@ -2311,9 +2300,9 @@ def test_materializing_never_hands_back_the_wrapped_array( np.testing.assert_array_equal(data, before, err_msg=description) -def test_an_eager_getitem_never_hands_back_the_wrapped_array() -> None: +def test_eager_adapter_getitem_never_hands_back_the_wrapped_array() -> None: data = reference() - block = LazyArray(data)[1:3] + block = EagerArrayAdapter(LazyArray(data))[1:3] block[...] = -1 np.testing.assert_array_equal(data, reference()) @@ -2327,7 +2316,7 @@ def test_result_refuses_to_return_a_partly_written_buffer( the parts tile the view. This is the guard that turns any future break of that contract into a failure instead of into plausible-looking numbers. """ - view = LazyArray(reference()).with_parts(PART_SHAPE).lazy[:, 1:, :] + view = LazyArray(reference()).with_parts(PART_SHAPE)[:, 1:, :] complete = LazyArray.parts def drop_one(self: LazyArray) -> Any: @@ -2340,7 +2329,7 @@ def drop_one(self: LazyArray) -> Any: def test_result_refuses_a_partition_of_the_wrong_rank(monkeypatch: pytest.MonkeyPatch) -> None: """A part addressing fewer axes than the view has is caught by name.""" - view = LazyArray(reference()).with_parts(PART_SHAPE).lazy[:, 1:, :] + view = LazyArray(reference()).with_parts(PART_SHAPE)[:, 1:, :] complete = LazyArray.parts def truncate(self: LazyArray) -> Any: @@ -2385,13 +2374,13 @@ def __getitem__(self, key: Any) -> DuckBlock: @pytest.mark.parametrize( ("build", "oracle"), [ - (lambda a: a.lazy[1:5, ::2, :], lambda r: r[1:5, ::2, :]), + (lambda a: a[1:5, ::2, :], lambda r: r[1:5, ::2, :]), ( - lambda a: a.lazy.oindex[[4, 0, 0], :, :], + lambda a: a.oindex[[4, 0, 0], :, :], lambda r: r[np.ix_([4, 0, 0], range(5), range(4))], ), - (lambda a: a.lazy.vindex[[4, 0], [1, 1]], lambda r: r[[4, 0], [1, 1]]), - (lambda a: a.lazy[::-1, :, :], lambda r: r[::-1, :, :]), + (lambda a: a.vindex[[4, 0], [1, 1]], lambda r: r[[4, 0], [1, 1]]), + (lambda a: a[::-1, :, :], lambda r: r[::-1, :, :]), ], ids=["basic", "oindex", "vindex", "reversal"], ) @@ -2427,7 +2416,7 @@ def test_numpy_matrix_is_refused() -> None: @pytest.mark.parametrize("parts", [None, (2, 2), (1, 4), (3, 4)]) def test_a_masked_source_keeps_its_mask_under_every_partitioning(parts: Any) -> None: data = np.ma.masked_greater(np.arange(12).reshape(3, 4), 7) - view = repartition(LazyArray(data), parts).lazy[:, 1:] + view = repartition(LazyArray(data), parts)[:, 1:] got = view.result() expected = data[:, 1:] assert isinstance(got, np.ma.MaskedArray), parts @@ -2444,7 +2433,7 @@ def test_a_masked_source_keeps_its_mask_under_every_partitioning(parts: Any) -> def test_a_masked_source_keeps_its_mask_when_the_view_is_empty(parts: Any) -> None: """Empty views of masked sources return masked arrays for every partitioning.""" data = np.ma.masked_greater(np.arange(12).reshape(3, 4), 7) - got = repartition(LazyArray(data), parts).lazy[:, 2:2].result() + got = repartition(LazyArray(data), parts)[:, 2:2].result() assert isinstance(got, np.ma.MaskedArray), parts assert np.asarray(got).shape == (3, 0), parts @@ -2457,8 +2446,8 @@ def test_a_masked_source_keeps_its_mask_when_the_view_is_empty(parts: Any) -> No def test_a_reversing_view_covers_its_parts() -> None: """A reversal reads every cell of every box, back to front.""" data = np.arange(48).reshape(8, 6) - forward = LazyArray(data).with_parts((2, 2)).lazy[:, :] - reversed_view = LazyArray(data).with_parts((2, 2)).lazy[::-1, ::-1] + forward = LazyArray(data).with_parts((2, 2))[:, :] + reversed_view = LazyArray(data).with_parts((2, 2))[::-1, ::-1] assert [part.is_complete for part in reversed_view.parts()] == [ part.is_complete for part in forward.parts() ] @@ -2467,7 +2456,7 @@ def test_a_reversing_view_covers_its_parts() -> None: def test_a_strided_reversal_is_still_incomplete() -> None: data = np.arange(48).reshape(8, 6) - view = LazyArray(data).with_parts((2, 2)).lazy[::-2, :] + view = LazyArray(data).with_parts((2, 2))[::-2, :] assert not any(part.is_complete for part in view.parts()) @@ -2499,7 +2488,7 @@ def test_the_coverage_count_agrees_with_numpy_for_reversed_selections( ) -> None: """Coverage counts match NumPy selection sizes for reversed slices.""" data = reference() - view = LazyArray(data).with_parts((2, 2, 2)).lazy[selection] + view = LazyArray(data).with_parts((2, 2, 2))[selection] out_shape = view.shape for part in view.parts(): counted = _out_selection_cell_count(part.out_selection, out_shape) @@ -2538,9 +2527,9 @@ def test_a_zero_dimensional_index_array_drops_its_axis_like_a_scalar() -> None: ("vindex", data[np.array(2), np.array(3)]), ): view = ( - LazyArray(data).lazy.oindex[np.array(2), slice(None)] + LazyArray(data).oindex[np.array(2), slice(None)] if mode == "oindex" - else LazyArray(data).lazy.vindex[np.array(2), np.array(3)] + else LazyArray(data).vindex[np.array(2), np.array(3)] ) assert view.shape == expected.shape, mode np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=mode) @@ -2549,7 +2538,7 @@ def test_a_zero_dimensional_index_array_drops_its_axis_like_a_scalar() -> None: def test_a_multidimensional_array_in_an_orthogonal_selection_is_refused() -> None: """Reject multidimensional orthogonal index arrays with a selection-level error.""" with pytest.raises(IndexError, match="must be 1-dimensional"): - LazyArray(np.arange(20).reshape(4, 5)).lazy.oindex[[[0, 1], [2, 3]], slice(None)] + LazyArray(np.arange(20).reshape(4, 5)).oindex[[[0, 1], [2, 3]], slice(None)] def test_with_parts_rejects_a_bare_integer() -> None: @@ -2564,11 +2553,11 @@ def test_with_parts_rejects_a_bare_integer() -> None: def test_fancy_composition_over_an_empty_axis() -> None: """Fancy composition over an empty axis preserves shape through later selections.""" base = np.empty((3, 0, 6), dtype=np.int64) - view = LazyArray(base).lazy.oindex[[2, 1], :, [5, 0, 3]] + view = LazyArray(base).oindex[[2, 1], :, [5, 0, 3]] assert view.shape == (2, 0, 3) - composed = view.lazy.oindex[[1, 0], :, [2, 2]] + composed = view.oindex[[1, 0], :, [2, 2]] assert composed.shape == (2, 0, 2) - scalar = composed.lazy.vindex[..., np.array(1)] + scalar = composed.vindex[..., np.array(1)] assert scalar.shape == (2, 0) assert np.asarray(scalar.result()).shape == (2, 0) @@ -2627,7 +2616,7 @@ def test_dask_is_only_required_for_tokenization(monkeypatch: pytest.MonkeyPatch) monkeypatch.setitem(sys.modules, "dask.base", None) data = np.arange(4) - view = LazyArray(data).lazy[1:] + view = LazyArray(data)[1:] np.testing.assert_array_equal(view.result(), data[1:]) with pytest.raises(ModuleNotFoundError): view.__dask_tokenize__() diff --git a/packages/zarr-indexing/tests/test_lazy_default.py b/packages/zarr-indexing/tests/test_lazy_default.py new file mode 100644 index 0000000000..875b7cd5c2 --- /dev/null +++ b/packages/zarr-indexing/tests/test_lazy_default.py @@ -0,0 +1,213 @@ +"""The wrapper defers selection; explicit execution touches the source.""" + +from collections.abc import Callable +from typing import Any + +import numpy as np +import pytest + +from zarr_indexing import EagerArrayAdapter, IndexDomain, IndexTransform, LazyArray +from zarr_indexing.errors import BoundsCheckError + + +class RecordingArray: + def __init__(self) -> None: + self.data = np.arange(30).reshape(5, 6) + self.shape = self.data.shape + self.dtype = self.data.dtype + self.reads: list[Any] = [] + + def __getitem__(self, key: Any) -> Any: + self.reads.append(key) + return self.data[key] + + +@pytest.mark.parametrize("mode", ["basic", "oindex", "vindex", "iteration"]) +def test_selection_defers_reads(mode: str) -> None: + source = RecordingArray() + array = LazyArray(source) + if mode == "basic": + view = array[1:][::-1, ::2] + expected = source.data[1:][::-1, ::2] + elif mode == "oindex": + view = array.oindex[[4, 1], :][:, ::2] + expected = source.data[[4, 1]][:, ::2] + elif mode == "vindex": + view = array.vindex[[4, 1], [2, 0]][::-1] + expected = source.data[[4, 1], [2, 0]][::-1] + else: + view = list(array)[2] + expected = source.data[2] + assert isinstance(view, LazyArray) + assert source.reads == [] + np.testing.assert_array_equal(view.result(), expected) + assert source.reads + + +def test_no_redundant_lazy_accessor() -> None: + assert not hasattr(LazyArray(np.arange(3)), "lazy") + + +@pytest.mark.parametrize("selection", [slice(None), slice(0, 0), 0]) +def test_eager_adapter_executes_selection(selection: Any) -> None: + source = RecordingArray() + view = LazyArray(source)[1:, ::2] + adapter = EagerArrayAdapter(view) + assert (adapter.shape, adapter.ndim, adapter.dtype) == (view.shape, view.ndim, view.dtype) + assert source.reads == [] + result = adapter[selection] + assert isinstance(result, np.ndarray) + np.testing.assert_array_equal(result, source.data[1:, ::2][selection]) + assert not np.shares_memory(result, source.data) + + +def test_eager_adapter_distinguishes_token_semantics() -> None: + pytest.importorskip("dask") + view = LazyArray(np.arange(8))[::2] + assert EagerArrayAdapter(view).__dask_tokenize__() != view.__dask_tokenize__() + + +def test_eager_adapter_refuses_no_copy_conversion() -> None: + with pytest.raises(ValueError, match="copy"): + EagerArrayAdapter(LazyArray(np.arange(3))).__array__(copy=False) + + +def test_views_keep_literal_domains_while_keys_stay_positional() -> None: + source = np.arange(30) + view = LazyArray(source)[10:20] + assert view.transform.domain == IndexDomain((10,), (20,)) + nested = view[2:5] + assert nested.transform.domain == IndexDomain((12,), (15,)) + np.testing.assert_array_equal(nested.result(), source[12:15]) + # NumPy keys are positions in the current view, not literal coordinates. + assert view[0].result() == source[10] + assert view[-1].result() == source[19] + assert nested[-1].result() == source[14] + np.testing.assert_array_equal(view[::-1][:3].result(), source[10:20][::-1][:3]) + # Equivalent selections built by different routes share one domain. + assert LazyArray(source)[0:20][5:10].transform == LazyArray(source)[5:10].transform + + +def test_domain_key_selects_literal_coordinates() -> None: + source = np.arange(30) + view = LazyArray(source)[10:20] + inner = view[IndexDomain((12,), (15,))] + assert inner.transform.domain == IndexDomain((12,), (15,)) + np.testing.assert_array_equal(inner.result(), source[12:15]) + np.testing.assert_array_equal(LazyArray(source)[IndexDomain((2,), (5,))].result(), source[2:5]) + with pytest.raises(BoundsCheckError): + view[IndexDomain((0,), (3,))] + # An empty domain outside the view is refused too: the key names the view's coordinates. + with pytest.raises(BoundsCheckError): + view[IndexDomain((50,), (50,))] + assert view[IndexDomain((15,), (15,))].shape == (0,) + with pytest.raises(ValueError, match="rank"): + view[IndexDomain((0, 0), (1, 1))] + view[IndexDomain((12,), (15,))] = [-1, -2, -3] + np.testing.assert_array_equal(source[12:15], [-1, -2, -3]) + + +def test_transform_key_composes_onto_the_view() -> None: + source = np.arange(30) + view = LazyArray(source)[10:20] + reversed_key = IndexTransform.identity(view.transform.domain)[::-1] + np.testing.assert_array_equal(view[reversed_key].result(), source[10:20][::-1]) + rebased = IndexTransform.identity(IndexDomain((12,), (15,))).translate_domain_to((0,)) + composed = view[rebased] + assert composed.transform.domain == IndexDomain((0,), (3,)) + np.testing.assert_array_equal(composed.result(), source[12:15]) + with pytest.raises((ValueError, BoundsCheckError)): + view[IndexTransform.identity(IndexDomain((0,), (3,)))] + + +def test_box_part_views_are_sub_domains_of_their_parent() -> None: + source = np.arange(40).reshape(4, 10) + view = LazyArray(source).with_parts((2, 4))[1:, 3:9] + origin = view.transform.domain.inclusive_min + for part in view.parts(): + domain = part.view.transform.domain + assert all( + lo >= parent_lo and hi <= parent_hi + for lo, hi, parent_lo, parent_hi in zip( + domain.inclusive_min, + domain.exclusive_max, + view.transform.domain.inclusive_min, + view.transform.domain.exclusive_max, + strict=True, + ) + ) + # Placement is readable from the domain: it is `out_selection`. + assert part.out_selection == tuple( + slice(lo - o, hi - o, 1) + for lo, hi, o in zip(domain.inclusive_min, domain.exclusive_max, origin, strict=True) + ) + np.testing.assert_array_equal(part.view.result(), source[1:, 3:9][part.out_selection]) + for child in part.view.parts(): + assert child.view.transform.domain == domain + # A part placed by index arrays cannot carry the request's coordinates: its + # domain is fresh and zero-origin, and `out_selection` is the placement. + fancy = LazyArray(source).with_parts((2, 4))[1:, 3:9].oindex[[2, 0], :] + assembled = np.empty(fancy.shape, dtype=fancy.dtype) + for part in fancy.parts(): + assert part.view.transform.domain.inclusive_min == (0,) * part.view.ndim + assembled[part.out_selection] = part.view.result() + np.testing.assert_array_equal(assembled, source[1:, 3:9][[2, 0], :]) + + +def test_reader_contexts_are_re_based_whatever_the_view_domain() -> None: + from zarr_indexing.reader import ReadContext + + view = LazyArray(np.arange(30))[10:20][::-1] + context = ReadContext(view.transform) + assert context.transform.domain == IndexDomain((0,), (10,)) + assert context.transform.apply((0,)) == view.transform.apply( + view.transform.domain.inclusive_min + ) + + +DOMAIN_CASES: list[ + tuple[str, Callable[[LazyArray], LazyArray], tuple[int, ...], tuple[int, ...]] +] = [ + ("whole", lambda v: v[:], (0, 0), (6, 8)), + ("ellipsis", lambda v: v[...], (0, 0), (6, 8)), + ("slice", lambda v: v[2:5], (2, 0), (5, 8)), + ("nested slice", lambda v: v[2:5][1:], (3, 0), (5, 8)), + ("negative stop", lambda v: v[2:5][:-1], (2, 0), (4, 8)), + ("empty", lambda v: v[2:5][1:1], (3, 0), (3, 8)), + ("int drops axis", lambda v: v[2:5][-1], (0,), (8,)), + ("strided", lambda v: v[:, 1:7:2], (0, 0), (6, 3)), + ("reversed", lambda v: v[::-1], (-5, 0), (1, 8)), + ("reversed then slice", lambda v: v[::-1][1:3], (-4, 0), (-2, 8)), + ("reversed twice", lambda v: v[1:4, 2:6][::-1, ::-1], (-3, -5), (0, -1)), + ("newaxis", lambda v: v[None, 2:4], (0, 2, 0), (1, 4, 8)), + ("oindex", lambda v: v.oindex[[3, 1], 2:6], (0, 2), (2, 6)), + ("oindex on slice", lambda v: v[2:5].oindex[[1, 0], :], (0, 0), (2, 8)), + ("vindex", lambda v: v.vindex[[1, 2], [3, 4]], (0,), (2,)), + ("domain key", lambda v: v[2:5][IndexDomain((3, 0), (4, 8))], (3, 0), (4, 8)), +] + + +@pytest.mark.parametrize( + ("case", "select", "lo", "hi"), DOMAIN_CASES, ids=[c[0] for c in DOMAIN_CASES] +) +def test_indexing_preserves_the_literal_domain( + case: str, select: Callable[[LazyArray], LazyArray], lo: tuple[int, ...], hi: tuple[int, ...] +) -> None: + """A view's domain is the literal domain TensorStore-style composition gives it. + + Slices keep their coordinates, nested slices compose them, integers drop + axes, `None` and index arrays open fresh zero-origin axes, and reversals + run negative. Values are unaffected: the same positional keys select the + same NumPy elements. + """ + source = np.arange(48).reshape(6, 8) + view = select(LazyArray(source)) + expected = IndexDomain(lo, hi) + assert view.transform.domain == expected, case + assert view.shape == expected.shape + # The literal frame is live: the domain's own first coordinate addresses the + # view's first element, and `ReadContext` still hands readers a zero origin. + if view.size: + np.testing.assert_array_equal( + view.result().flat[0], source[view.transform.apply(expected.inclusive_min)] + ) diff --git a/packages/zarr-indexing/tests/test_reader.py b/packages/zarr-indexing/tests/test_reader.py index 2bf426753b..d3da049f0e 100644 --- a/packages/zarr-indexing/tests/test_reader.py +++ b/packages/zarr-indexing/tests/test_reader.py @@ -69,7 +69,7 @@ def __getitem__(self, key: tuple[Any, ...]) -> np.ndarray[Any, Any]: np.zeros((1, 0), dtype=np.intp), np.array([[3]], dtype=np.intp), np.array(3), - lambda array: array.lazy[3], + lambda array: array[3], id="constant", ), pytest.param( @@ -79,7 +79,7 @@ def __getitem__(self, key: tuple[Any, ...]) -> np.ndarray[Any, Any]: np.array([[0], [1], [2], [3]], dtype=np.intp), np.array([[1], [3], [5], [7]], dtype=np.intp), np.array([1, 3, 5, 7]), - lambda array: array.lazy[1:8:2], + lambda array: array[1:8:2], id="positive-affine", ), pytest.param( @@ -89,7 +89,7 @@ def __getitem__(self, key: tuple[Any, ...]) -> np.ndarray[Any, Any]: np.array([[-3], [-2], [-1], [0]], dtype=np.intp), np.array([[7], [5], [3], [1]], dtype=np.intp), np.array([7, 5, 3, 1]), - lambda array: array.lazy[::-2], + lambda array: array[::-2], id="negative-affine", ), pytest.param( @@ -102,7 +102,7 @@ def __getitem__(self, key: tuple[Any, ...]) -> np.ndarray[Any, Any]: np.array([[0], [1], [2]], dtype=np.intp), np.array([[2], [2], [2]], dtype=np.intp), np.array([2, 2, 2]), - lambda array: array.lazy.oindex[[2, 2, 2]], + lambda array: array.oindex[[2, 2, 2]], id="zero-affine", ), pytest.param( @@ -112,7 +112,7 @@ def __getitem__(self, key: tuple[Any, ...]) -> np.ndarray[Any, Any]: np.array([[0, 0], [0, 1], [1, 0], [1, 1], [2, 0], [2, 1]], dtype=np.intp), np.array([[3, 4], [3, 0], [1, 4], [1, 0], [1, 4], [1, 0]], dtype=np.intp), np.array([[19, 15], [9, 5], [9, 5]]), - lambda array: array.lazy.oindex[[3, 1, 1], [4, 0]], + lambda array: array.oindex[[3, 1, 1], [4, 0]], id="orthogonal-array", ), pytest.param( @@ -122,7 +122,7 @@ def __getitem__(self, key: tuple[Any, ...]) -> np.ndarray[Any, Any]: np.array([[0], [1], [2]], dtype=np.intp), np.array([[3, 4], [1, 0], [1, 4]], dtype=np.intp), np.array([19, 5, 9]), - lambda array: array.lazy.vindex[[3, 1, 1], [4, 0, 4]], + lambda array: array.vindex[[3, 1, 1], [4, 0, 4]], id="correlated-array", ), pytest.param( @@ -145,7 +145,7 @@ def __getitem__(self, key: tuple[Any, ...]) -> np.ndarray[Any, Any]: np.empty((0, 1), dtype=np.intp), np.empty((0, 1), dtype=np.intp), np.array([], dtype=np.intp), - lambda array: array.lazy[2:2], + lambda array: array[2:2], id="empty", ), ) @@ -239,9 +239,7 @@ def test_successful_transform_contract_across_planning_readers_and_lazy_array( for child in part.view.parts(): np.testing.assert_array_equal(child.view.result(), part.view.result()) reverse = (slice(None, None, -1),) * part.view.ndim - np.testing.assert_array_equal( - part.view.lazy[reverse].result(), part.view.result()[reverse] - ) + np.testing.assert_array_equal(part.view[reverse].result(), part.view.result()[reverse]) for candidate in ( part.view.unpartitioned(), part.view.with_parts((1,) * source_data.ndim), @@ -412,8 +410,8 @@ def test_empty_domain_composed_fancy_transform_reads_as_empty() -> None: Empty index arrays are valid when the domain selects no elements.""" source_data = np.arange(6).reshape(2, 3) - view = LazyArray.from_numpy(source_data).lazy.oindex[slice(0, 0), np.array([2, 1, 2, 0])] - transform = view.lazy.oindex[slice(None), np.array([1, 3, 1])].transform + view = LazyArray.from_numpy(source_data).oindex[slice(0, 0), np.array([2, 1, 2, 0])] + transform = view.oindex[slice(None), np.array([1, 3, 1])].transform assert transform.domain.shape == (0, 3) for reader, source in ( @@ -433,12 +431,12 @@ def test_unit_step_reader_reads_through_lazy_array() -> None: reached the source — partitioned and unpartitioned alike. """ selections: tuple[Callable[[LazyArray], LazyArray], ...] = ( - lambda v: v.lazy[1:5, ::2, ::-1], - lambda v: v.lazy[5:1:-2, None, 3, ::3], - lambda v: v.lazy.oindex[[3, 0, 3], ::-2, [7, 7]], - lambda v: v.lazy.vindex[np.array([[0, 5]]), np.array([[6], [0]]), 2], - lambda v: v.lazy[2:2, :, ::-1], - lambda v: v.lazy[::5, 6, 1:8:4], + lambda v: v[1:5, ::2, ::-1], + lambda v: v[5:1:-2, None, 3, ::3], + lambda v: v.oindex[[3, 0, 3], ::-2, [7, 7]], + lambda v: v.vindex[np.array([[0, 5]]), np.array([[6], [0]]), 2], + lambda v: v[2:2, :, ::-1], + lambda v: v[::5, 6, 1:8:4], ) for select in selections: for parts in (None, (2, 3, 8), (6, 7, 1)): diff --git a/packages/zarr-indexing/tests/test_transform.py b/packages/zarr-indexing/tests/test_transform.py index ac59eecdfc..6fffca110b 100644 --- a/packages/zarr-indexing/tests/test_transform.py +++ b/packages/zarr-indexing/tests/test_transform.py @@ -1149,11 +1149,7 @@ class TestDerivedMapDependency: def test_a_vindex_over_a_fancy_view_is_marked_correlated(self) -> None: base = np.arange(6) - view = ( - LazyArray(base) - .lazy.oindex[np.array([0, 1])] - .lazy.vindex[np.array([[0, 1, 0], [1, 0, 1]])] - ) + view = LazyArray(base).oindex[np.array([0, 1])].vindex[np.array([[0, 1, 0], [1, 0, 1]])] np.testing.assert_array_equal( np.asarray(view.result()), base[[0, 1]][[[0, 1, 0], [1, 0, 1]]] ) @@ -1162,7 +1158,7 @@ def test_the_same_view_resolves_alike_however_it_is_partitioned(self) -> None: base = np.arange(36).reshape(6, 6) def build(array: LazyArray) -> LazyArray: - return array.lazy.oindex[np.array([-3, -6, -4]), -4].lazy.vindex[np.array([[-2, -3]])] + return array.oindex[np.array([-3, -6, -4]), -4].vindex[np.array([[-2, -3]])] unpartitioned = np.asarray(build(LazyArray(base)).result()) partitioned = np.asarray(build(LazyArray(base).with_parts((3, 3))).result()) @@ -1188,10 +1184,10 @@ def test_dependency_axes_are_read_from_the_shape(self) -> None: def test_an_orthogonal_step_over_a_correlated_view_is_an_outer_product() -> None: """Orthogonal indexing after vectorized indexing selects an outer product.""" base = np.arange(14).reshape(7, 2) - view = LazyArray(base).lazy.vindex[ + view = LazyArray(base).vindex[ np.array([[5, 5], [1, 2], [0, 4]]), np.array([[1, 1], [1, 0], [1, 0]]) ] - result = np.asarray(view.lazy.oindex[np.array([1, 1, 0]), np.array([1, 1, 0, 1])].result()) + result = np.asarray(view.oindex[np.array([1, 1, 0]), np.array([1, 1, 0, 1])].result()) assert result.shape == (3, 4) np.testing.assert_array_equal(result, np.array([[4, 4, 3, 4], [4, 4, 3, 4], [11, 11, 11, 11]])) diff --git a/packages/zarr-indexing/tests/test_writer.py b/packages/zarr-indexing/tests/test_writer.py new file mode 100644 index 0000000000..af9623b26e --- /dev/null +++ b/packages/zarr-indexing/tests/test_writer.py @@ -0,0 +1,462 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np +import pytest + +from zarr_indexing.domain import IndexDomain +from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap +from zarr_indexing.transform import IndexTransform +from zarr_indexing.writer import write_into + + +class BasicSource: + """Reject source reads and advanced writes, recording basic writes.""" + + def __init__(self, data: np.ndarray[Any, Any]) -> None: + self.data = data + self.shape = data.shape + self.dtype = data.dtype + self.calls: list[tuple[int | slice, ...]] = [] + + def __setitem__(self, key: tuple[int | slice, ...], value: Any) -> None: + assert all(isinstance(item, (int, slice)) for item in key) + assert all( + not isinstance(item, slice) or item.step is None or item.step > 0 for item in key + ) + self.calls.append(key) + self.data[key] = value + + +@pytest.mark.parametrize( + "case", + ["reverse", "scalar", "newaxis", "empty", "outer", "composed", "correlated", "transpose"], +) +def test_write_coordinates(case: str) -> None: + data = np.arange(24).reshape(4, 6) + expected = data.copy() + # Select flat source identifiers independently of the transform evaluator. + identifiers = np.arange(data.size).reshape(data.shape) + transform = IndexTransform.from_shape(data.shape) + if case == "reverse": + transform = transform[::-1, ::2] + identifiers = identifiers[::-1, ::2] + elif case == "scalar": + transform = transform[2, 3] + identifiers = identifiers[2, 3] + elif case == "newaxis": + transform = IndexTransform( + IndexDomain.from_shape((1, 4, 6)), (DimensionMap(1), DimensionMap(2)) + ) + identifiers = identifiers[None] + elif case == "empty": + transform = transform[1:1, :] + identifiers = identifiers[1:1, :] + elif case == "outer": + transform = transform.oindex[[3, 1, 3], [5, 0, 5]] + identifiers = identifiers[np.ix_([3, 1, 3], [5, 0, 5])] + elif case == "composed": + transform = transform.oindex[[3, 1, 3], :].oindex[[2, 0], [5, 0]] + identifiers = identifiers[[3, 1, 3]][np.ix_([2, 0], [5, 0])] + elif case == "correlated": + transform = IndexTransform( + IndexDomain.from_shape((3, 2)), + ( + ArrayMap(np.array([[3], [0], [3]], dtype=np.intp)), + ArrayMap(np.array([[5, 1], [0, 2], [5, 1]], dtype=np.intp)), + ), + ) + identifiers = identifiers[np.array([[3], [0], [3]]), np.array([[5, 1], [0, 2], [5, 1]])] + else: + transform = IndexTransform( + IndexDomain.from_shape((6, 4)), (DimensionMap(1), DimensionMap(0)) + ) + identifiers = identifiers.T + values = np.arange(identifiers.size).reshape(identifiers.shape) + 100 + for index, value in zip(np.asarray(identifiers).flat, values.flat, strict=True): + expected.flat[index] = value + source = BasicSource(data) + assert write_into(source, transform, values) is None + np.testing.assert_array_equal(data, expected) + if case in ("reverse", "scalar", "newaxis", "transpose"): + assert len(source.calls) == 1 + elif case == "empty": + assert source.calls == [] + + +@pytest.mark.parametrize("fancy", [False, True]) +def test_write_alias_and_broadcast(fancy: bool) -> None: + data = np.arange(6) + expected = data[::-1].copy() + transform = IndexTransform.from_shape(data.shape) + transform = transform.oindex[[5, 4, 3, 2, 1, 0]] if fancy else transform[::-1] + write_into(BasicSource(data), transform, data) + np.testing.assert_array_equal(data, expected) + write_into(BasicSource(data), transform, np.array([[17]])) + np.testing.assert_array_equal(data, np.full(6, 17)) + + +def test_write_broadcast_error_before_mutation() -> None: + source = BasicSource(np.zeros((4, 6))) + with pytest.raises(ValueError): + write_into(source, IndexTransform.from_shape(source.shape), [1, 2]) + assert source.calls == [] + + +def test_write_cast_error_before_mutation() -> None: + source = BasicSource(np.zeros(2, dtype=np.int64)) + with pytest.raises(ValueError): + write_into(source, IndexTransform.from_shape(source.shape), ["1", "invalid"]) + assert source.calls == [] + + +def test_write_bounds_error_before_mutation() -> None: + source = BasicSource(np.zeros(2)) + transform = IndexTransform( + IndexDomain.from_shape((2,)), (ArrayMap(np.array([0, 2], dtype=np.intp)),) + ) + with pytest.raises(IndexError, match="outside"): + write_into(source, transform, [1, 2]) + assert source.calls == [] + + +def test_write_readonly_error() -> None: + data = np.zeros(2) + data.flags.writeable = False + with pytest.raises(ValueError, match="read-only"): + write_into(data, IndexTransform.from_shape(data.shape), 1) + + +def test_write_constant_repeated_destination() -> None: + source = BasicSource(np.zeros(3)) + transform = IndexTransform(IndexDomain.from_shape((2, 3)), (ConstantMap(1),)) + write_into(source, transform, [[1], [9]]) + np.testing.assert_array_equal(source.data, [0, 9, 0]) + assert len(source.calls) == 6 + + +@pytest.mark.parametrize("scalar", [False, True]) +def test_write_scalar_source_and_shared_dimensions(scalar: bool) -> None: + if scalar: + source = BasicSource(np.array(0)) + transform = IndexTransform.from_shape(()) + expected = np.array(7) + else: + source = BasicSource(np.zeros((3, 3), dtype=np.int64)) + transform = IndexTransform(IndexDomain.from_shape((3,)), (DimensionMap(0), DimensionMap(0))) + expected = np.diag([7, 7, 7]) + write_into(source, transform, 7) + np.testing.assert_array_equal(source.data, expected) + + +def test_write_rank_error_before_mutation() -> None: + source = BasicSource(np.zeros((2, 2))) + with pytest.raises(ValueError, match="output rank"): + write_into(source, IndexTransform.from_shape((2,)), 1) + assert source.calls == [] + + +@pytest.mark.parametrize("masked_source", [False, True]) +@pytest.mark.parametrize("fancy", [False, True]) +def test_write_masked_values(masked_source: bool, fancy: bool) -> None: + data: Any = np.arange(12).reshape(3, 4) + if masked_source: + data = np.ma.array(data, mask=np.ones(data.shape, dtype=bool)) + expected = data.copy() + values = np.ma.array([[31, 32, 33, 34]], mask=[[False, True, False, True]]) + transform = IndexTransform.from_shape(data.shape) + if fancy: + transform = transform.oindex[[2, 0, 2], :] + expected[[2, 0, 2], :] = values + else: + transform = transform[::-1, :] + expected[::-1, :] = values + write_into(data, transform, values) + np.testing.assert_array_equal(np.ma.getdata(data), np.ma.getdata(expected)) + np.testing.assert_array_equal(np.ma.getmaskarray(data), np.ma.getmaskarray(expected)) + + +def test_write_masked_lazy_alias() -> None: + from zarr_indexing.lazy_array import LazyArray + + source = np.ma.array(np.arange(6), mask=[True, False, False, True, False, True]) + expected = source[::-1].copy() + array = LazyArray(source) + array[::-1] = array + np.testing.assert_array_equal(source.data, expected.data) + np.testing.assert_array_equal(source.mask, expected.mask) + + +@pytest.mark.parametrize("backend", ["numpy", "zarr"]) +@pytest.mark.parametrize("case", ["basic", "outer", "vector", "newaxis", "empty", "alias"]) +def test_lazy_write_and_assignment(backend: str, case: str) -> None: + from zarr_indexing.lazy_array import LazyArray + + initial = np.arange(20).reshape(4, 5) + source: Any = initial.copy() + if backend == "zarr": + zarr = pytest.importorskip("zarr") + source = zarr.create_array({}, shape=initial.shape, chunks=(2, 3), dtype="int64") + source[:] = initial + array = LazyArray(source) + identifiers = initial.copy() + expected = initial.copy() + if case == "basic": + view = array[::-1, ::2][1:, 1:] + identifiers = identifiers[::-1, ::2][1:, 1:] + elif case == "outer": + view = array.oindex[[3, 0, 3], [4, 1]].oindex[[2, 0, 1], [1, 0]] + identifiers = identifiers[np.ix_([3, 0, 3], [4, 1])][np.ix_([2, 0, 1], [1, 0])] + elif case == "vector": + view = array.oindex[[3, 0, 3], :].vindex[[2, 0, 1], [4, 4, 1]] + identifiers = identifiers[[3, 0, 3], :][[2, 0, 1], [4, 4, 1]] + elif case == "newaxis": + view = array[None, 1:3, :] + identifiers = identifiers[None, 1:3, :] + elif case == "empty": + view = array[2:2, :] + identifiers = identifiers[2:2, :] + else: + array[::-1, :] = array + np.testing.assert_array_equal(source[:], initial[::-1]) + return + values = np.arange(identifiers.size).reshape(identifiers.shape) + 100 + for identifier, value in zip(identifiers.flat, values.flat, strict=True): + expected.flat[identifier] = value + assert view.write(values) is None + np.testing.assert_array_equal(source[:], expected) + # All selector assignment entry points write through to the original source. + array.oindex[[0, 3], [1, 4]] = 77 + expected[np.ix_([0, 3], [1, 4])] = 77 + array.vindex[[0, 0, 2], [2, 2, 4]] = [10, 11, 12] + expected[[0, 0, 2], [2, 2, 4]] = [10, 11, 12] + np.testing.assert_array_equal(source[:], expected) + + +@pytest.mark.parametrize("backend", ["numpy", "zarr"]) +def test_random_selection_chain_scatter(backend: str) -> None: + """Map selected integer labels back to storage without consulting transforms.""" + from zarr_indexing.lazy_array import LazyArray + + rng = np.random.default_rng(58123) + for _ in range(24): + initial = np.arange(30).reshape(5, 6) + source: Any = initial.copy() + if backend == "zarr": + zarr = pytest.importorskip("zarr") + source = zarr.create_array({}, shape=initial.shape, chunks=(2, 3), dtype="int64") + source[:] = initial + view = LazyArray(source) + identifiers = initial + for _ in range(3): + if rng.integers(2): + rows = rng.integers(0, view.shape[0], size=4) + cols = rng.integers(0, view.shape[1], size=3) + view = view.oindex[rows, cols] + identifiers = identifiers[np.ix_(rows, cols)] + else: + step = int(rng.choice([-2, -1, 1, 2])) + view = view[::step, ::-1] + identifiers = identifiers[::step, ::-1] + values = rng.integers(100, 200, size=identifiers.shape) + expected = initial.copy() + for identifier, value in zip(identifiers.flat, values.flat, strict=True): + expected.flat[identifier] = value + view.write(values) + np.testing.assert_array_equal(source[:], expected) + + +class _ChunkTouches: + """Count the storage round trips a zarr array makes, one per chunk key.""" + + def __init__(self) -> None: + self.gets = 0 + self.sets = 0 + + @classmethod + def wrap(cls, zarr: Any) -> tuple[Any, _ChunkTouches]: + counter = cls() + + class CountingStore(zarr.storage.MemoryStore): + async def get(self, key: str, prototype: Any, byte_range: Any = None) -> Any: + counter.gets += 1 + return await super().get(key, prototype, byte_range) + + async def set(self, key: str, value: Any) -> None: + counter.sets += 1 + await super().set(key, value) + + return CountingStore(), counter + + +@pytest.mark.parametrize("sharded", [False, True]) +@pytest.mark.parametrize("case", ["outer_rows", "vector_points", "duplicates"]) +def test_fancy_writes_touch_each_write_chunk_once(sharded: bool, case: str) -> None: + """Storage round trips scale with touched write chunks, not selected elements.""" + zarr = pytest.importorskip("zarr") + from zarr_indexing.lazy_array import LazyArray + + store, touches = _ChunkTouches.wrap(zarr) + kwargs: dict[str, Any] = {"chunks": (2, 2), "shards": (4, 4)} if sharded else {"chunks": (4, 4)} + source = zarr.create_array(store, shape=(12, 12), dtype="int64", **kwargs) + source[:] = 0 + expected = np.zeros((12, 12), dtype=np.int64) + touches.gets = touches.sets = 0 + view = LazyArray(source) + if case == "outer_rows": + rows = [1, 5, 11] + values = np.arange(3 * 12).reshape(3, 12) + view.oindex[rows, :] = values + expected[rows, :] = values + touched = {(r // 4, c) for r in rows for c in range(3)} + elif case == "vector_points": + rows, cols = [0, 7, 7, 11, 3], [0, 2, 9, 11, 5] + values = np.arange(5) + 10 + view.vindex[rows, cols] = values + expected[rows, cols] = values + touched = {(r // 4, c // 4) for r, c in zip(rows, cols, strict=True)} + else: + view.oindex[[5, 5, 6], 2:5] = np.array([[1] * 3, [2] * 3, [3] * 3]) + expected[5, 2:5] = 2 + expected[6, 2:5] = 3 + touched = {(1, 0), (1, 1)} + gets, sets = touches.gets, touches.sets + np.testing.assert_array_equal(source[:], expected) + assert sets == len(touched), (sets, len(touched)) + # At most the hull read plus zarr's own partial-chunk read per cell. + assert gets <= 2 * len(touched), (gets, len(touched)) + + +def test_write_grid_prefers_write_chunk_sizes_over_chunks() -> None: + """A source whose write grid is coarser than its read grid is written by the write grid.""" + from zarr_indexing.lazy_array import LazyArray + + class ShardedSource: + chunks = ((2, 2, 2), (3, 3)) + write_chunk_sizes = ((6,), (3, 3)) + + def __init__(self) -> None: + self.data = np.zeros((6, 6), dtype=np.int64) + self.shape = self.data.shape + self.dtype = self.data.dtype + self.boxes: list[tuple[slice, ...]] = [] + + def __getitem__(self, key: Any) -> Any: + return self.data[key] + + def __setitem__(self, key: Any, value: Any) -> None: + self.boxes.append(key) + self.data[key] = value + + source = ShardedSource() + LazyArray(source).oindex[[0, 5], :] = 7 + expected = np.zeros((6, 6), dtype=np.int64) + expected[[0, 5], :] = 7 + np.testing.assert_array_equal(source.data, expected) + # One read-modify-write per touched write cell: two column shards, each + # spanning all six rows, rather than the four inner chunks a read grid names. + assert len(source.boxes) == 2 + assert all(box[0] == slice(0, 6) for box in source.boxes) + + +def test_write_only_source_falls_back_to_element_assignment() -> None: + source = BasicSource(np.zeros((3, 4), dtype=np.int64)) + transform = IndexTransform.from_shape(source.shape).oindex[[2, 0], [1, 3]] + write_into(source, transform, np.array([[1, 2], [3, 4]])) + assert len(source.calls) == 4 + expected = np.zeros((3, 4), dtype=np.int64) + expected[np.ix_([2, 0], [1, 3])] = [[1, 2], [3, 4]] + np.testing.assert_array_equal(source.data, expected) + + +def test_masked_zero_rank_affine_write_keeps_payload() -> None: + from zarr_indexing.lazy_array import LazyArray + + source = np.ma.array([1, 2, 3]) + LazyArray(source)[1].write(np.ma.array(7, mask=True)) + np.testing.assert_array_equal(source.data, [1, 7, 3]) + np.testing.assert_array_equal(np.ma.getmaskarray(source), [False, True, False]) + + +def test_write_rank_error_precedes_value_broadcasting() -> None: + source = np.zeros((3, 4), dtype=np.int64) + transform = IndexTransform.from_shape((3,)) + with pytest.raises(ValueError, match="rank"): + write_into(source, transform, np.arange(6).reshape(2, 3)) + np.testing.assert_array_equal(source, 0) + + +def test_empty_selection_still_validates_values() -> None: + source = np.zeros(5, dtype=np.int64) + with pytest.raises(ValueError, match="broadcast"): + write_into(source, IndexTransform.from_shape((5,))[2:2], np.arange(3)) + with pytest.raises(ValueError): + write_into(source, IndexTransform.from_shape((5,))[2:2], ["not a number"]) + np.testing.assert_array_equal(source, 0) + + +def test_gridless_readable_source_never_reads() -> None: + """Without a write grid the hull could be the whole array, so elements are assigned.""" + from zarr_indexing.lazy_array import LazyArray + + class ReadableSource(BasicSource): + def __getitem__(self, key: Any) -> Any: + raise AssertionError("gridless writes must not read the source") + + source = ReadableSource(np.zeros((4, 4), dtype=np.int64)) + LazyArray(source).vindex[[0, 3], [3, 0]] = [1, 2] + expected = np.zeros((4, 4), dtype=np.int64) + expected[[0, 3], [3, 0]] = [1, 2] + np.testing.assert_array_equal(source.data, expected) + assert len(source.calls) == 2 + + +def test_unfactorable_transform_falls_back_to_elements_on_a_gridded_source() -> None: + """A diagonal reads one input axis twice, which the planner rejects; it is still written.""" + + class GriddedSource(BasicSource): + chunks = ((2, 2), (2, 2)) + + def __getitem__(self, key: Any) -> Any: + return self.data[key] + + source = GriddedSource(np.zeros((4, 4), dtype=np.int64)) + diagonal = IndexTransform(IndexDomain.from_shape((4,)), (DimensionMap(0), DimensionMap(0))) + write_into(source, diagonal, np.arange(1, 5), write_grid=None) + from zarr_indexing.grid import dimension_grids_from_chunks + + source.data[:] = 0 + source.calls.clear() + write_into( + source, + diagonal, + np.arange(1, 5), + write_grid=dimension_grids_from_chunks(source.chunks, (4, 4)), + ) + np.testing.assert_array_equal(source.data, np.diag([1, 2, 3, 4])) + assert len(source.calls) == 4 + + +def test_writes_bypass_the_reader_so_a_caching_reader_serves_stale_values() -> None: + """Pins the documented hazard: the reader is not invalidated by `write`.""" + from zarr_indexing.lazy_array import LazyArray + from zarr_indexing.reader import ReadContext, basic_reader + + class CachingReader: + def __init__(self) -> None: + self.cache: dict[tuple[Any, ...], Any] = {} + + def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: + key = (context.transform.domain.shape, str(context.transform.to_json())) + if key not in self.cache: + basic_reader.read_into(source, context, out) + self.cache[key] = out.copy() + out[...] = self.cache[key] + + source = np.arange(6) + view = LazyArray(source).with_reader(CachingReader())[1:4] + np.testing.assert_array_equal(view.result(), [1, 2, 3]) + view.write([7, 8, 9]) + np.testing.assert_array_equal(source, [0, 7, 8, 9, 4, 5]) + np.testing.assert_array_equal(view.result(), [1, 2, 3]) From 9f0ae4e27fa1a7bdcff668f9fe08c884066c9dfc Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 16 Sep 2026 13:10:03 +0200 Subject: [PATCH 17/29] fix: replace runtime assert statements with explicit checks (#4363) * fix: replace runtime assert statements with explicit checks Asserts are stripped under `python -O`, so checks that matter for correctness or type narrowing must be explicit. Two were load-bearing: `GroupMetadata.from_dict` asserted on node_type and the array-to-group fallback in `zarr.api.asynchronous.open` caught the AssertionError, and `make_store` asserted on mode ahead of the real validation. Redundant asserts are deleted, narrowing asserts are restructured so mypy narrows on its own, and the rest become explicit raises. Ruff S101 is enabled with `tests/` and `src/zarr/testing/` excluded. Assisted-by: ClaudeCode:claude-fable-5-1 * docs: add changelog fragment for #4363 Assisted-by: ClaudeCode:claude-fable-5-1 * ci: scope the no-assert lint to runtime code across the monorepo The S101 per-file ignores only covered the root tests/ and src/zarr/testing/. The packages under packages/ inherit the root ruff config, so the rule fired on 1524 asserts in their tests, examples, and test-support modules and broke the ruff, Lint, pre-commit.ci, and zarr-http-server jobs. Widen the ignores to **/tests/**, **/examples/**, and **/testing/**, and exclude zarr-indexing's runtime source for now; its own asserts are tracked as a separate change. Assisted-by: ClaudeCode:claude-fable-5-1 --- changes/4363.bugfix.md | 8 +++++ pyproject.toml | 9 ++++++ src/zarr/__init__.py | 5 +++- src/zarr/api/asynchronous.py | 2 +- src/zarr/codecs/bytes.py | 3 +- src/zarr/codecs/cast_value.py | 8 +++-- src/zarr/codecs/sharding.py | 20 ++++++++----- src/zarr/codecs/vlen_utf8.py | 10 +------ src/zarr/core/array.py | 52 ++++++++++++++++++--------------- src/zarr/core/buffer/cpu.py | 1 - src/zarr/core/buffer/gpu.py | 5 ++-- src/zarr/core/codec_pipeline.py | 49 +++++++++++++++++++------------ src/zarr/core/group.py | 24 +++++++++------ src/zarr/core/sync.py | 22 +++++++------- src/zarr/storage/_common.py | 3 +- src/zarr/storage/_local.py | 5 ---- src/zarr/storage/_memory.py | 6 ---- src/zarr/storage/_zip.py | 2 -- tests/test_array.py | 10 +++++++ tests/test_group.py | 9 ++++++ tests/test_store/test_core.py | 8 +++++ 21 files changed, 158 insertions(+), 103 deletions(-) create mode 100644 changes/4363.bugfix.md diff --git a/changes/4363.bugfix.md b/changes/4363.bugfix.md new file mode 100644 index 0000000000..fbccb109f4 --- /dev/null +++ b/changes/4363.bugfix.md @@ -0,0 +1,8 @@ +Removed every `assert` statement from runtime code and enabled ruff's `S101` +rule to keep them out. Asserts are stripped under `python -O`, and two of them +were load-bearing: `GroupMetadata.from_dict` asserted on `node_type` and the +array-to-group fallback in `zarr.open` relied on catching the resulting +`AssertionError`, and `make_store` asserted on `mode` before the real +validation. Both now raise proper errors (`NodeTypeValidationError` and +`ValueError`). Redundant asserts were deleted, type-narrowing asserts were +restructured so mypy narrows without them, and the rest became explicit raises. diff --git a/pyproject.toml b/pyproject.toml index 57443aed80..4f256c0cb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -361,6 +361,7 @@ extend-select = [ "RET", # flake8-return "RSE", # flake8-raise "RUF", + "S101", # flake8-bandit: no assert in runtime code "SIM", # flake8-simplify "SLOT", # flake8-slots "TC", # flake8-type-checking @@ -395,6 +396,14 @@ ignore = [ [tool.ruff.lint.extend-per-file-ignores] "tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"] +# S101 (no assert) is for runtime code only. Tests, examples, and test-support +# packages (store conformance suite, hypothesis strategies) assert on purpose. +# The globs cover the root package and every package under packages/. +"**/tests/**" = ["S101"] +"**/examples/**" = ["S101"] +"**/testing/**" = ["S101"] +# zarr-indexing still has asserts in runtime code; tracked as a separate change. +"packages/zarr-indexing/src/**" = ["S101"] [tool.mypy] files = ["src", "tests"] diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index cdf3840c3b..f305245f4e 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -39,7 +39,10 @@ from zarr.core.group import AsyncGroup, Group # in case setuptools scm screw up and find version to be 0.0.0 -assert not __version__.startswith("0.0.0") +if __version__.startswith("0.0.0"): + raise RuntimeError( + f"zarr resolved its own version to {__version__!r}; the build metadata is broken." + ) _logger = logging.getLogger(__name__) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 3bdc254ea5..1fc10cdd1e 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -403,7 +403,7 @@ async def open( return AsyncArray( store_path=store_path, metadata=_metadata_dict, config=kwargs.get("config") ) - except (AssertionError, FileNotFoundError, NodeTypeValidationError): + except (FileNotFoundError, NodeTypeValidationError): pass return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs) diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index fae762fd08..622161c458 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -7,7 +7,6 @@ from zarr.abc.codec import ArrayBytesCodec from zarr.codecs._deprecated_enum import _coerce_enum_input, _DeprecatedStrEnumMeta -from zarr.core.buffer import Buffer, NDBuffer from zarr.core.common import JSON, parse_named_configuration from zarr.core.dtype.common import HasEndianness from zarr.core.dtype.npy.structured import Struct @@ -16,6 +15,7 @@ from typing import Self from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer, NDBuffer EndianLiteral = Literal["little", "big"] @@ -142,7 +142,6 @@ def _encode_sync( chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: - assert isinstance(chunk_array, NDBuffer) if chunk_array.dtype.itemsize > 1 and self.endian is not None: # Compare full dtypes rather than the top-level byteorder: numpy reports # byteorder '|' for structured dtypes even when their fields are diff --git a/src/zarr/codecs/cast_value.py b/src/zarr/codecs/cast_value.py index b19a10c873..629b4e7a27 100644 --- a/src/zarr/codecs/cast_value.py +++ b/src/zarr/codecs/cast_value.py @@ -311,7 +311,9 @@ def _validate_scalar_map( target_zdtype: ZDType[TBaseDType, TBaseScalar], ) -> None: """Validate that scalar map entries are compatible with source/target dtypes.""" - assert self.scalar_map is not None + scalar_map = self.scalar_map + if scalar_map is None: + return # For encode: keys are source values, values are target values. # For decode: keys are target values, values are source values. direction_dtypes: dict[ @@ -321,9 +323,9 @@ def _validate_scalar_map( "decode": (target_zdtype, source_zdtype), } for direction, (key_zdtype, val_zdtype) in direction_dtypes.items(): - if direction not in self.scalar_map: + if direction not in scalar_map: continue - sub_map = self.scalar_map[direction] # type: ignore[literal-required] + sub_map = scalar_map[direction] # type: ignore[literal-required] for k, v in sub_map.items(): _check_representable(k, key_zdtype, f"scalar_map {direction} key") _check_representable(v, val_zdtype, f"scalar_map {direction} value") diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index bd0760f7e3..3d5057cecd 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -160,9 +160,11 @@ class _ShardingByteGetter(ByteGetter): def get_sync( self, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None ) -> Buffer | None: - assert prototype is None or prototype == default_buffer_prototype(), ( - f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}" - ) + if prototype is not None and prototype != default_buffer_prototype(): + raise ValueError( + "Non-default buffer prototypes are not supported within shards. " + f"Got {prototype}, expected {default_buffer_prototype()}." + ) value = self.shard_dict.get(self.chunk_coords) if value is None: return None @@ -193,7 +195,8 @@ def delete_sync(self) -> None: del self.shard_dict[self.chunk_coords] async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: - assert byte_range is None, "byte_range is not supported within shards" + if byte_range is not None: + raise NotImplementedError("byte_range is not supported within shards.") self.set_sync(value) async def delete(self) -> None: @@ -656,7 +659,8 @@ def _encode_shard_index_sync(self, index: _ShardIndex) -> Buffer: index_spec = self._get_index_chunk_spec(index.chunks_per_shard) index_nd = get_ndbuffer_class().from_numpy_array(index.offsets_and_lengths) result: Buffer | None = index_transform.encode_chunk(index_nd, index_spec) - assert result is not None + if result is None: + raise RuntimeError("Encoding the shard index produced no bytes.") return result def _shard_reader_from_bytes_sync( @@ -1531,7 +1535,8 @@ async def _decode_shard_index( .decode([(index_bytes, self._get_index_chunk_spec(chunks_per_shard))]) ) ) - assert index_array is not None # the bytes are already in hand + if index_array is None: + raise RuntimeError("Decoding the shard index produced no array.") return _ShardIndex(chunks_per_shard, index_array.as_numpy_array()) async def _encode_shard_index(self, index: _ShardIndex) -> Buffer: @@ -1553,7 +1558,8 @@ async def _encode_shard_index(self, index: _ShardIndex) -> Buffer: ) ) ) - assert index_bytes is not None + if index_bytes is None: + raise RuntimeError("Encoding the shard index produced no bytes.") return index_bytes def _shard_index_size(self, chunks_per_shard: tuple[int, ...]) -> int: diff --git a/src/zarr/codecs/vlen_utf8.py b/src/zarr/codecs/vlen_utf8.py index 078e6032fc..6433c80be4 100644 --- a/src/zarr/codecs/vlen_utf8.py +++ b/src/zarr/codecs/vlen_utf8.py @@ -8,13 +8,13 @@ from zarr._compat import _reshape_view from zarr.abc.codec import ArrayBytesCodec -from zarr.core.buffer import Buffer, NDBuffer from zarr.core.common import JSON, parse_named_configuration if TYPE_CHECKING: from typing import Self from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer, NDBuffer # can use a global because there are no parameters @@ -45,11 +45,8 @@ def _decode_sync( chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: - assert isinstance(chunk_bytes, Buffer) - raw_bytes = chunk_bytes.as_array_like() decoded = _vlen_utf8_codec.decode(raw_bytes) - assert decoded.dtype == np.object_ decoded = _reshape_view(decoded, chunk_spec.shape) as_string_dtype = decoded.astype(chunk_spec.dtype.to_native_dtype(), copy=False) return chunk_spec.prototype.nd_buffer.from_numpy_array(as_string_dtype) @@ -66,7 +63,6 @@ def _encode_sync( chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: - assert isinstance(chunk_array, NDBuffer) # numcodecs vlen codecs flatten with order="A", so an F-contiguous chunk # would be encoded in transposed element order (gh-3558) return chunk_spec.prototype.buffer.from_bytes( @@ -106,11 +102,8 @@ def _decode_sync( chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: - assert isinstance(chunk_bytes, Buffer) - raw_bytes = chunk_bytes.as_array_like() decoded = _vlen_bytes_codec.decode(raw_bytes) - assert decoded.dtype == np.object_ decoded = _reshape_view(decoded, chunk_spec.shape) return chunk_spec.prototype.nd_buffer.from_numpy_array(decoded) @@ -126,7 +119,6 @@ def _encode_sync( chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: - assert isinstance(chunk_array, NDBuffer) # numcodecs vlen codecs flatten with order="A", so an F-contiguous chunk # would be encoded in transposed element order (gh-3558) return chunk_spec.prototype.buffer.from_bytes( diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 047e7bb3b3..5a8d6bf57e 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -158,6 +158,7 @@ from zarr.abc.codec import CodecPipeline from zarr.abc.store import Store from zarr.codecs.sharding import IndexLocation, ShardingCodec + from zarr.core.buffer import Buffer from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar from zarr.storage import StoreLike from zarr.types import AnyArray, AnyAsyncArray, ArrayV2, ArrayV3, AsyncArrayV2, AsyncArrayV3 @@ -282,6 +283,7 @@ async def get_array_metadata( f"{store_path.store!r} at path {store_path.path!r}." ) raise ArrayNotFoundError(msg) + return _array_metadata_dict_v2(zarray_bytes, zattrs_bytes) elif zarr_format == 3: zarr_json_bytes = await (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype) if zarr_json_bytes is None: @@ -290,6 +292,7 @@ async def get_array_metadata( f"{store_path.store!r} at path {store_path.path!r}." ) raise ArrayNotFoundError(msg) + return _array_metadata_dict_v3(zarr_json_bytes) elif zarr_format is None: zarr_json_bytes, zarray_bytes, zattrs_bytes = await gather( (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype), @@ -300,35 +303,34 @@ async def get_array_metadata( # warn and favor v3 msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store_path}. Zarr v3 will be used." warnings.warn(msg, category=ZarrUserWarning, stacklevel=1) - if zarr_json_bytes is None and zarray_bytes is None: - msg = ( - f"Neither Zarr V3 nor Zarr V2 array metadata documents " - f"were found in store {store_path.store!r} at path {store_path.path!r}." - ) - raise ArrayNotFoundError(msg) - # set zarr_format based on which keys were found + # favor v3 when both are present if zarr_json_bytes is not None: - zarr_format = 3 - else: - zarr_format = 2 + return _array_metadata_dict_v3(zarr_json_bytes) + if zarray_bytes is not None: + return _array_metadata_dict_v2(zarray_bytes, zattrs_bytes) + msg = ( + f"Neither Zarr V3 nor Zarr V2 array metadata documents " + f"were found in store {store_path.store!r} at path {store_path.path!r}." + ) + raise ArrayNotFoundError(msg) else: msg = f"Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '{zarr_format}'." # type: ignore[unreachable] raise MetadataValidationError(msg) - metadata_dict: dict[str, JSON] - if zarr_format == 2: - # V2 arrays are comprised of a .zarray and .zattrs objects - assert zarray_bytes is not None - metadata_dict = buffer_to_json_object(zarray_bytes) - zattrs_dict = buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {} - metadata_dict["attributes"] = zattrs_dict - else: - # V3 arrays are comprised of a zarr.json object - assert zarr_json_bytes is not None - metadata_dict = buffer_to_json_object(zarr_json_bytes) - parse_node_type_array(metadata_dict.get("node_type")) +def _array_metadata_dict_v2(zarray_bytes: Buffer, zattrs_bytes: Buffer | None) -> dict[str, JSON]: + """Combine a `.zarray` document and an optional `.zattrs` document into one metadata dict.""" + metadata_dict: dict[str, JSON] = buffer_to_json_object(zarray_bytes) + metadata_dict["attributes"] = ( + buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {} + ) + return metadata_dict + +def _array_metadata_dict_v3(zarr_json_bytes: Buffer) -> dict[str, JSON]: + """Parse a `zarr.json` document, checking that it describes an array.""" + metadata_dict: dict[str, JSON] = buffer_to_json_object(zarr_json_bytes) + parse_node_type_array(metadata_dict.get("node_type")) return metadata_dict @@ -5896,7 +5898,11 @@ async def _resize( If False, the data in those chunks will be preserved. """ new_shape = parse_shapelike(new_shape) - assert len(new_shape) == len(array.metadata.shape) + if len(new_shape) != len(array.metadata.shape): + raise ValueError( + f"The new shape must have the same number of dimensions as the array. " + f"Got {len(new_shape)} dimension(s), expected {len(array.metadata.shape)}." + ) new_metadata = array.metadata.update_shape(new_shape) new_chunk_grid = ChunkGrid.from_metadata(new_metadata) diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py index 8994281b58..f987ef9dfe 100644 --- a/src/zarr/core/buffer/cpu.py +++ b/src/zarr/core/buffer/cpu.py @@ -111,7 +111,6 @@ def combine(self, others: Iterable[core.Buffer]) -> Self: data = [np.asanyarray(self._data)] for buf in others: other_array = buf.as_array_like() - assert other_array.dtype == np.dtype("B") data.append(np.asanyarray(other_array)) return self.__class__(np.concatenate(data)) diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py index 8672942364..6f0e4aec12 100644 --- a/src/zarr/core/buffer/gpu.py +++ b/src/zarr/core/buffer/gpu.py @@ -111,7 +111,6 @@ def combine(self, others: Iterable[core.Buffer]) -> Self: data = [cp.asanyarray(self._data)] for other in others: other_array = other.as_array_like() - assert other_array.dtype == np.dtype("B") gpu_other = Buffer(other_array) gpu_other_array = gpu_other.as_array_like() data.append(cp.asanyarray(gpu_other_array)) @@ -149,8 +148,8 @@ def __init__(self, array: NDArrayLike) -> None: "Cannot use zarr.buffer.gpu.NDBuffer without cupy. Please install cupy." ) - # assert array.ndim > 0 - assert array.dtype != object + if array.dtype == object: + raise TypeError("zarr.buffer.gpu.NDBuffer does not support arrays with object dtype.") self._data = array if not hasattr(array, "__cuda_array_interface__"): diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py index 597f338c42..9ed16bc381 100644 --- a/src/zarr/core/codec_pipeline.py +++ b/src/zarr/core/codec_pipeline.py @@ -5,7 +5,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from itertools import batched, chain, pairwise -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from warnings import warn from zarr.abc.codec import ( @@ -397,9 +397,8 @@ async def _async_write_fallback( sync IO / sync transform is unavailable). """ - if use_sync := ( - isinstance(pipeline, FusedCodecPipeline) and pipeline.sync_transform is not None - ): + sync_transform = pipeline.sync_transform if isinstance(pipeline, FusedCodecPipeline) else None + if sync_transform is not None: # Read each chunk's existing bytes (skipping complete chunks) and decode # as fetches complete, overlapping the sync decode with in-flight reads. chunk_array_decoded: Iterable[NDBuffer | None] = await _fetch_and_decode_as_completed( @@ -407,7 +406,7 @@ async def _async_write_fallback( (None if is_complete_chunk else byte_setter, chunk_spec) for byte_setter, chunk_spec, _, _, is_complete_chunk in batch ], - pipeline.sync_transform, + sync_transform, ) else: @@ -460,9 +459,7 @@ async def _read_key( for chunk_array, (_, chunk_spec, *_) in zip(chunk_array_merged, batch, strict=False) ] - if use_sync: - sync_transform = cast(FusedCodecPipeline, pipeline).sync_transform - assert sync_transform is not None + if sync_transform is not None: await _encode_and_write_as_completed( [ (byte_setter, chunk_array, chunk_spec) @@ -709,9 +706,12 @@ async def decode_partial_batch( self, batch_info: Iterable[tuple[ByteGetter, SelectorTuple, ArraySpec]], ) -> Iterable[NDBuffer | None]: - assert self.supports_partial_decode - assert isinstance(self.array_bytes_codec, ArrayBytesCodecPartialDecodeMixin) - return await self.array_bytes_codec.decode_partial(batch_info) + codec = self.array_bytes_codec + if not self.supports_partial_decode or not isinstance( + codec, ArrayBytesCodecPartialDecodeMixin + ): + raise ValueError("This codec pipeline does not support partial decoding.") + return await codec.decode_partial(batch_info) async def encode_batch( self, @@ -744,9 +744,12 @@ async def encode_partial_batch( self, batch_info: Iterable[tuple[ByteSetter, NDBuffer, SelectorTuple, ArraySpec]], ) -> None: - assert self.supports_partial_encode - assert isinstance(self.array_bytes_codec, ArrayBytesCodecPartialEncodeMixin) - await self.array_bytes_codec.encode_partial(batch_info) + codec = self.array_bytes_codec + if not self.supports_partial_encode or not isinstance( + codec, ArrayBytesCodecPartialEncodeMixin + ): + raise ValueError("This codec pipeline does not support partial encoding.") + await codec.encode_partial(batch_info) async def read_batch( self, @@ -1117,8 +1120,12 @@ def read_sync( the read selection. Otherwise the pipeline fetches the full blob and decodes the whole chunk. """ - assert self.sync_transform is not None transform = self.sync_transform + if transform is None: + raise RuntimeError( + "This codec pipeline contains codecs without synchronous support; " + "use the async read/write path instead." + ) batch = list(batch_info) if not batch: @@ -1193,8 +1200,12 @@ def write_sync( the full write cycle — reading existing data, merging, encoding, and writing — matching the async `BatchedCodecPipeline` path. """ - assert self.sync_transform is not None transform = self.sync_transform + if transform is None: + raise RuntimeError( + "This codec pipeline contains codecs without synchronous support; " + "use the async read/write path instead." + ) batch = list(batch_info) if not batch: @@ -1294,9 +1305,9 @@ async def read( # inner-chunk byte ranges (coalesced via get_ranges), matching # BatchedCodecPipeline. Without this, the whole-shard _async_read_fallback # below would over-read and diverge from the batched pipeline's IO. - if self.supports_partial_decode: - assert isinstance(self.array_bytes_codec, ArrayBytesCodecPartialDecodeMixin) - chunk_array_batch = await self.array_bytes_codec.decode_partial( + codec = self.array_bytes_codec + if self.supports_partial_decode and isinstance(codec, ArrayBytesCodecPartialDecodeMixin): + chunk_array_batch = await codec.decode_partial( [ (byte_getter, chunk_selection, chunk_spec) for byte_getter, chunk_spec, chunk_selection, *_ in batch diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index d061e1a5c6..d734e6b7cd 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -56,6 +56,7 @@ ContainsGroupError, GroupNotFoundError, MetadataValidationError, + NodeTypeValidationError, ZarrUserWarning, ) from zarr.storage import StoreLike, StorePath @@ -372,7 +373,11 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: ZATTRS_JSON: self.attributes, } consolidated_metadata = self.consolidated_metadata.to_dict()["metadata"] - assert isinstance(consolidated_metadata, dict) + if not isinstance(consolidated_metadata, dict): + raise TypeError( + "Expected consolidated metadata to serialize to a dict, " + f"got {type(consolidated_metadata).__name__}." + ) for k, v in consolidated_metadata.items(): attrs = v.pop("attributes", {}) d[f"{k}/{ZATTRS_JSON}"] = attrs @@ -412,7 +417,11 @@ def __init__( @classmethod def from_dict(cls, data: dict[str, Any]) -> GroupMetadata: data = dict(data) - assert data.pop("node_type", None) in ("group", None) + node_type = data.pop("node_type", None) + if node_type not in ("group", None): + raise NodeTypeValidationError( + f"Invalid value for 'node_type'. Expected 'group' or None. Got {node_type!r}." + ) consolidated_metadata = data.pop("consolidated_metadata", None) if consolidated_metadata: data["consolidated_metadata"] = ConsolidatedMetadata.from_dict(consolidated_metadata) @@ -583,8 +592,8 @@ async def open( raise MetadataValidationError(msg) if zarr_format == 2: - # this is checked above, asserting here for mypy - assert zgroup_bytes is not None + if zgroup_bytes is None: + raise FileNotFoundError(store_path) if use_consolidated and maybe_consolidated_metadata_bytes is None: # the user requested consolidated metadata, but it was missing @@ -600,7 +609,8 @@ async def open( ) else: # V3 groups are comprised of a zarr.json object - assert zarr_json_bytes is not None + if zarr_json_bytes is None: + raise FileNotFoundError(store_path) if not isinstance(use_consolidated, bool | None): raise TypeError("use_consolidated must be a bool or None for Zarr format 3.") @@ -742,9 +752,6 @@ def _getitem_consolidated( # Note that this is a regular def (non async) function. # This shouldn't do any additional I/O. - # the caller needs to verify this! - assert self.metadata.consolidated_metadata is not None - # we support nested getitems like group/subgroup/array indexers = normalize_path(key).split("/") indexers.reverse() @@ -1049,7 +1056,6 @@ async def require_group(self, name: str, overwrite: bool = False) -> AsyncGroup: raise TypeError( f"Incompatible object ({item.__class__.__name__}) already exists" ) - assert isinstance(item, AsyncGroup) # make mypy happy grp = item except KeyError: grp = await self.create_group(name) diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index 724b31a464..16b1c577ff 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -149,7 +149,6 @@ def sync[T]( finished, unfinished = wait([future], return_when=asyncio.ALL_COMPLETED, timeout=timeout) if len(unfinished) > 0: raise TimeoutError(f"Coroutine {coro} failed to finish within {timeout} s") - assert len(finished) == 1 return_result = next(iter(finished)).result() if isinstance(return_result, BaseException): @@ -163,20 +162,21 @@ def _get_loop() -> asyncio.AbstractEventLoop: The loop will be running on a separate thread. """ - if loop[0] is None: + current = loop[0] + if current is None: with _get_lock(): # repeat the check just in case the loop got filled between the # previous two calls from another thread - if loop[0] is None: + current = loop[0] + if current is None: logger.debug("Creating Zarr event loop") - new_loop = asyncio.new_event_loop() - loop[0] = new_loop - iothread[0] = threading.Thread(target=new_loop.run_forever, name="zarr_io") - assert iothread[0] is not None - iothread[0].daemon = True - iothread[0].start() - assert loop[0] is not None - return loop[0] + current = asyncio.new_event_loop() + loop[0] = current + thread = threading.Thread(target=current.run_forever, name="zarr_io") + thread.daemon = True + iothread[0] = thread + thread.start() + return current async def _collect_aiterator[T](data: AsyncIterator[T]) -> tuple[T, ...]: diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 72b5fc8a40..df08968e5c 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -360,7 +360,8 @@ async def make_store( "'storage_options' is only used when the store is passed as an FSSpec URI string.", ) - assert mode in (None, "r", "r+", "a", "w", "w-") + if mode is not None and mode not in ANY_ACCESS_MODE: + raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") _read_only = mode == "r" if isinstance(store_like, StorePath): diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index d6a3edcd9a..c34ca44cc1 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -255,7 +255,6 @@ def get_sync( if prototype is None: prototype = default_buffer_prototype() self._ensure_open_sync() - assert isinstance(key, str) path = self.root / key try: return _get(path, prototype, byte_range) @@ -265,7 +264,6 @@ def get_sync( def set_sync(self, key: str, value: Buffer) -> None: self._ensure_open_sync() self._check_writable() - assert isinstance(key, str) if not isinstance(value, Buffer): raise TypeError( f"LocalStore.set(): `value` must be a Buffer instance. " @@ -294,7 +292,6 @@ async def get( prototype = default_buffer_prototype() if not self._is_open: await self._open() - assert isinstance(key, str) path = self.root / key try: @@ -310,7 +307,6 @@ async def get_partial_values( # docstring inherited args = [] for key, byte_range in key_ranges: - assert isinstance(key, str) path = self.root / key args.append((_get, path, prototype, byte_range)) return await concurrent_map(args, asyncio.to_thread, limit=None) # TODO: fix limit @@ -330,7 +326,6 @@ async def _set(self, key: str, value: Buffer, exclusive: bool = False) -> None: if not self._is_open: await self._open() self._check_writable() - assert isinstance(key, str) if not isinstance(value, Buffer): raise TypeError( f"LocalStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index f42c38df69..3bece0e841 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -118,7 +118,6 @@ def get_sync( prototype = default_buffer_prototype() if not self._is_open: self._is_open = True - assert isinstance(key, str) try: value = self._store_dict[key] start, stop = _normalize_byte_range_index(value, byte_range) @@ -130,7 +129,6 @@ def set_sync(self, key: str, value: Buffer) -> None: self._check_writable() if not self._is_open: self._is_open = True - assert isinstance(key, str) if not isinstance(value, Buffer): raise TypeError( f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." @@ -157,7 +155,6 @@ async def get( prototype = default_buffer_prototype() if not self._is_open: await self._open() - assert isinstance(key, str) try: value = self._store_dict[key] start, stop = _normalize_byte_range_index(value, byte_range) @@ -186,7 +183,6 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None # docstring inherited self._check_writable() await self._ensure_open() - assert isinstance(key, str) if not isinstance(value, Buffer): raise TypeError( f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." @@ -305,7 +301,6 @@ def from_dict(cls, store_dict: MutableMapping[str, Buffer]) -> Self: async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: # docstring inherited self._check_writable() - assert isinstance(key, str) if not isinstance(value, Buffer): raise TypeError( f"GpuMemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." @@ -317,7 +312,6 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None def set_sync(self, key: str, value: Buffer) -> None: # docstring inherited self._check_writable() - assert isinstance(key, str) if not isinstance(value, Buffer): raise TypeError( f"GpuMemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index 69ae18bc2c..6b9ebca561 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -271,7 +271,6 @@ async def get( byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited - assert isinstance(key, str) with self._lock: return self._get(key, prototype=prototype, byte_range=byte_range) @@ -306,7 +305,6 @@ async def set(self, key: str, value: Buffer) -> None: self._check_writable() if not self._is_open: self._sync_open() - assert isinstance(key, str) if not isinstance(value, Buffer): raise TypeError( f"ZipStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." diff --git a/tests/test_array.py b/tests/test_array.py index 560941b101..3ae502a4e8 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -676,6 +676,16 @@ async def test_info_complete_async( ) +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_resize_wrong_ndim_raises(store: MemoryStore, zarr_format: ZarrFormat) -> None: + """ + Resizing to a shape with a different number of dimensions is a ValueError. + """ + z = zarr.create(shape=(10, 10), chunks=(5, 5), dtype="i4", store=store, zarr_format=zarr_format) + with pytest.raises(ValueError, match="same number of dimensions"): + z.resize((20,)) + + @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_resize_1d(store: MemoryStore, zarr_format: ZarrFormat) -> None: z = zarr.create( diff --git a/tests/test_group.py b/tests/test_group.py index 4ce326621f..31fbd138cd 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -47,6 +47,7 @@ ContainsGroupError, GroupNotFoundError, MetadataValidationError, + NodeTypeValidationError, ZarrUserWarning, ) from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore @@ -1112,6 +1113,14 @@ async def test_asyncgroup_open_wrong_format( await AsyncGroup.open(store=store, zarr_format=zarr_format_wrong) +def test_group_metadata_from_dict_wrong_node_type_raises() -> None: + """ + A metadata document whose node_type is not 'group' cannot become GroupMetadata. + """ + with pytest.raises(NodeTypeValidationError, match="node_type"): + GroupMetadata.from_dict({"zarr_format": 3, "node_type": "array"}) + + # todo: replace the dict[str, Any] type with something a bit more specific # should this be async? @pytest.mark.parametrize( diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index 7ba4344810..f39dd0cfd3 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -230,6 +230,14 @@ async def test_store_path_invalid_mode_raises( await StorePath.open(LocalStore(str(tmp_path), read_only=modes[0]), path="", mode=modes[1]) # type: ignore[arg-type] +async def test_make_store_invalid_mode_raises() -> None: + """ + Test that make_store raises ValueError for a mode outside the access-mode literals. + """ + with pytest.raises(ValueError, match="Invalid mode"): + await make_store({}, mode="x") # type: ignore[arg-type] + + async def test_make_store_path_invalid() -> None: """ Test that invalid types raise TypeError From dfa18e827d0738a70685b42f18c6650114310044 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 16 Sep 2026 14:05:34 +0200 Subject: [PATCH 18/29] fix(zarr-indexing): replace runtime assert statements with explicit checks (#4364) --- packages/zarr-indexing/changes/4364.misc.md | 7 ++ .../src/zarr_indexing/chunk_resolution.py | 9 +- .../src/zarr_indexing/messages.py | 29 +++++-- .../zarr-indexing/src/zarr_indexing/reader.py | 24 +++--- .../src/zarr_indexing/transform.py | 83 +++++++++---------- .../zarr-indexing/src/zarr_indexing/writer.py | 25 +++--- pyproject.toml | 2 - 7 files changed, 97 insertions(+), 82 deletions(-) create mode 100644 packages/zarr-indexing/changes/4364.misc.md diff --git a/packages/zarr-indexing/changes/4364.misc.md b/packages/zarr-indexing/changes/4364.misc.md new file mode 100644 index 0000000000..0ae29424bc --- /dev/null +++ b/packages/zarr-indexing/changes/4364.misc.md @@ -0,0 +1,7 @@ +Removed every `assert` statement from the package's runtime code and dropped +the `zarr-indexing` exemption from the repo-wide ruff `S101` rule, so none can +return. Asserts are stripped under `python -O`; the ones here narrowed types +or guarded internal invariants rather than validating input, so most were +restructured away (branching on the map type, carrying the narrowed value in a +local) and the remainder became explicit `RuntimeError`s for states the +public API cannot reach. No user-visible behavior changes. diff --git a/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py b/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py index da985513fa..01383b8e34 100644 --- a/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py +++ b/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py @@ -637,7 +637,14 @@ def _strided_set( def _indexed_set(out_dim: int, m: ArrayMap, dg: DimensionGridLike) -> IndexedSet: dependent = m.dependent_axis - assert dependent is not None + if dependent is None: + # `_partition_transform` sends every map that varies over no axis down + # the joint (general) path; an orthogonal map always has its one axis. + raise RuntimeError( + f"output dimension {out_dim} has an index array of shape " + f"{m.index_array.shape} that varies over no input axis; it cannot be " + "partitioned as an orthogonal map" + ) flat = m.index_array.reshape(-1) n = int(flat.size) storage = checked_affine(m.offset, m.stride, flat) diff --git a/packages/zarr-indexing/src/zarr_indexing/messages.py b/packages/zarr-indexing/src/zarr_indexing/messages.py index ef3b051e22..305d765947 100644 --- a/packages/zarr-indexing/src/zarr_indexing/messages.py +++ b/packages/zarr-indexing/src/zarr_indexing/messages.py @@ -203,6 +203,18 @@ def _bound_is_implicit(bound: int | str | list[int | str]) -> bool: return isinstance(bound, list) +def _finite(value: int | str, where: str) -> int: + """The integer behind a validated `index-value` that is not a sentinel. + + Every value reaching the desugaring helpers has passed `_check_index_value`, + so a string here is one of the two sentinels, which each helper handles + before calling this. Anything else is an internal error, not bad input. + """ + if isinstance(value, int): + return value + raise RuntimeError(f"{where} is {value!r}; expected an integer or an infinity sentinel") + + def _ext_key(value: int | str) -> tuple[int, int]: """A sort key giving the extended-integer order `-inf < n < +inf` exactly. @@ -213,8 +225,7 @@ def _ext_key(value: int | str) -> tuple[int, int]: return (0, 0) if value == "+inf": return (2, 0) - assert isinstance(value, int) - return (1, value) + return (1, _finite(value, "bound")) def _rewrap(value: int | str, *, implicit: bool) -> int | str | list[int | str]: @@ -276,10 +287,10 @@ def _resolve_upper_bound( The implicit/explicit bracket travels with the extent-bearing field (the upper bound, or `shape`), matching the spec's `[n]`-bracket convention. """ - if upper_field is None: + if upper_raw is None: + # No upper-bound field was supplied (`upper_field` is None as well). return [["+inf"] for _ in range(rank)] - assert upper_raw is not None if kind_of == "exclusive": return list(upper_raw) @@ -316,8 +327,7 @@ def _checked_i64(value: int, where: str) -> int: def _inclusive_to_exclusive(value: int | str, where: str) -> int | str: if value == "+inf" or value == "-inf": return value - assert isinstance(value, int) - return _checked_i64(value + 1, f"{where} converted to an exclusive bound") + return _checked_i64(_finite(value, where) + 1, f"{where} converted to an exclusive bound") def _shape_to_exclusive(min_value: int | str, shape_value: int | str, where: str) -> int | str: @@ -330,9 +340,10 @@ def _shape_to_exclusive(min_value: int | str, shape_value: int | str, where: str return "+inf" if min_value == "-inf": return "-inf" - assert isinstance(min_value, int) - assert isinstance(shape_value, int) - return _checked_i64(min_value + shape_value, f"{where} added to its inclusive_min") + return _checked_i64( + _finite(min_value, f"inclusive_min for {where}") + _finite(shape_value, where), + f"{where} added to its inclusive_min", + ) def _validate_domain(inclusive_min: list[Any], exclusive_max: list[Any], *, prefix: str) -> None: diff --git a/packages/zarr-indexing/src/zarr_indexing/reader.py b/packages/zarr-indexing/src/zarr_indexing/reader.py index 3da7267335..9e14d59c10 100644 --- a/packages/zarr-indexing/src/zarr_indexing/reader.py +++ b/packages/zarr-indexing/src/zarr_indexing/reader.py @@ -379,14 +379,14 @@ def _lower_orthogonal(array: Any, transform: IndexTransform) -> Any: if isinstance(m, ConstantMap): selection.append(m.offset) continue - if out_dim in gathered: - selection.append(slice(None)) - else: - assert isinstance(m, DimensionMap) + if isinstance(m, DimensionMap) and out_dim not in gathered: d = m.input_dimension lo = transform.domain.inclusive_min[d] hi = transform.domain.exclusive_max[d] selection.append(slice(m.offset + m.stride * lo, m.offset + m.stride * hi, m.stride)) + else: + # Every ArrayMap, and every non-positive-stride DimensionMap, was gathered above. + selection.append(slice(None)) if isinstance(m, ArrayMap): axis = m.dependent_axis if axis is None: @@ -414,15 +414,14 @@ def _lower_general(array: Any, transform: IndexTransform) -> Any: flat axis with row-major strides, and a single `take` collects them. """ outputs = transform.output - correlated_dims = [d for d, m in enumerate(outputs) if isinstance(m, ArrayMap)] + correlated = [(d, m) for d, m in enumerate(outputs) if isinstance(m, ArrayMap)] + correlated_dims = [d for d, _ in correlated] slice_input_dims = {m.input_dimension for m in outputs if isinstance(m, DimensionMap)} broadcast_axes = [d for d in range(transform.input_rank) if d not in slice_input_dims] broadcast_shape = tuple(transform.domain.shape[d] for d in broadcast_axes) - for d in correlated_dims: - arr_map = outputs[d] - assert isinstance(arr_map, ArrayMap) + for _, arr_map in correlated: # The axes the array varies over (its non-singleton axes; see # transform._array_map_dependency_axes) must all live in the block. dependency = (axis for axis, size in enumerate(arr_map.index_array.shape) if size > 1) @@ -454,16 +453,14 @@ def _lower_general(array: Any, transform: IndexTransform) -> Any: if isinstance(m, ConstantMap): selection.append(m.offset) continue - if out_dim in correlated_dims: + if isinstance(m, ArrayMap): selection.append(slice(None)) correlated_positions.append(axis) elif out_dim in gathered: selection.append(slice(None)) residual_positions.append(axis) - assert isinstance(m, DimensionMap) residual_axis_dims.append(m.input_dimension) else: - assert isinstance(m, DimensionMap) d = m.input_dimension lo = transform.domain.inclusive_min[d] hi = transform.domain.exclusive_max[d] @@ -487,10 +484,9 @@ def _lower_general(array: Any, transform: IndexTransform) -> Any: flat_index = np.zeros(math.prod(broadcast_shape), dtype=np.intp) stride = 1 for position in range(n_corr - 1, -1, -1): - m = outputs[correlated_dims[position]] - assert isinstance(m, ArrayMap) + _, corr_map = correlated[position] flat_index = flat_index + ( - _correlated_map_coords(m, broadcast_axes, broadcast_shape, transform.input_rank) + _correlated_map_coords(corr_map, broadcast_axes, broadcast_shape, transform.input_rank) * stride ) stride *= corr_sizes[position] diff --git a/packages/zarr-indexing/src/zarr_indexing/transform.py b/packages/zarr-indexing/src/zarr_indexing/transform.py index 3486060b20..adf5e5ff09 100644 --- a/packages/zarr-indexing/src/zarr_indexing/transform.py +++ b/packages/zarr-indexing/src/zarr_indexing/transform.py @@ -29,6 +29,7 @@ from __future__ import annotations +import math from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal, cast @@ -398,43 +399,33 @@ def inverted(self) -> IndexTransform: f"{self.input_rank} and {self.output_rank}" ) + # Nothing below touches `self`, so validating and building in one pass + # leaves no partial state behind when a map is rejected. referenced: set[int] = set() + inverse_min: list[int] = [] + inverse_max: list[int] = [] + inverse_output: dict[int, OutputIndexMap] = {} for output_dimension, output_map in enumerate(self.output): if isinstance(output_map, ArrayMap): raise ValueError( # noqa: TRY004 - valid map, invalid inverse f"cannot invert transform: output[{output_dimension}] is an ArrayMap" ) - if isinstance(output_map, DimensionMap): - if output_map.stride not in (-1, 1): - raise ValueError( - "cannot invert transform: DimensionMap stride must be +1 or -1, " - f"got {output_map.stride} for output[{output_dimension}]" - ) - if output_map.input_dimension in referenced: - raise ValueError( - "cannot invert transform: input dimension " - f"{output_map.input_dimension} is referenced more than once" - ) - referenced.add(output_map.input_dimension) - - for input_dimension, extent in enumerate(self.domain.shape): - if input_dimension not in referenced and extent != 1: - raise ValueError( - "cannot invert transform: unreferenced input dimension " - f"{input_dimension} has extent {extent}, not 1" - ) - - inverse_min: list[int] = [] - inverse_max: list[int] = [] - inverse_output: dict[int, OutputIndexMap] = {} - for output_dimension, output_map in enumerate(self.output): if isinstance(output_map, ConstantMap): inverse_min.append(output_map.offset) inverse_max.append(output_map.offset + 1) continue - - assert isinstance(output_map, DimensionMap) + if output_map.stride not in (-1, 1): + raise ValueError( + "cannot invert transform: DimensionMap stride must be +1 or -1, " + f"got {output_map.stride} for output[{output_dimension}]" + ) input_dimension = output_map.input_dimension + if input_dimension in referenced: + raise ValueError( + "cannot invert transform: input dimension " + f"{input_dimension} is referenced more than once" + ) + referenced.add(input_dimension) lower = self.domain.inclusive_min[input_dimension] upper = self.domain.exclusive_max[input_dimension] if output_map.stride == 1: @@ -453,9 +444,17 @@ def inverted(self) -> IndexTransform: stride=-1, ) - for input_dimension, lower in enumerate(self.domain.inclusive_min): - if input_dimension not in referenced: - inverse_output[input_dimension] = ConstantMap(lower) + for input_dimension, (lower, extent) in enumerate( + zip(self.domain.inclusive_min, self.domain.shape, strict=True) + ): + if input_dimension in referenced: + continue + if extent != 1: + raise ValueError( + "cannot invert transform: unreferenced input dimension " + f"{input_dimension} has extent {extent}, not 1" + ) + inverse_output[input_dimension] = ConstantMap(lower) return IndexTransform( domain=IndexDomain(tuple(inverse_min), tuple(inverse_max)), @@ -1164,14 +1163,12 @@ def _intersect_general( broadcast_shape = block.broadcast_shape # Joint bounds mask over the broadcast block. - combined: np.ndarray[Any, np.dtype[np.bool_]] | None = None + combined = np.ones(math.prod(broadcast_shape), dtype=np.bool_) for out_dim in correlated_dims: storage = block.flat_storage[out_dim] lo = output_domain.inclusive_min[out_dim] hi = output_domain.exclusive_max[out_dim] - mask = (storage >= lo) & (storage < hi) - combined = mask if combined is None else (combined & mask) - assert combined is not None + combined &= (storage >= lo) & (storage < hi) surviving = np.flatnonzero(combined).astype(np.intp) if surviving.size == 0: return None @@ -1222,19 +1219,18 @@ def _intersect_general( corr_shape = points_shape + (1,) * n_slice new_output: list[OutputIndexMap] = [] for out_dim, m in enumerate(transform.output): - if out_dim in correlated_dims: - corr = cast("ArrayMap", m) + if isinstance(m, ArrayMap): + # `correlated_dims` is exactly the ArrayMap output dimensions. new_output.append( ArrayMap( index_array=corr_values[out_dim].reshape(corr_shape), - offset=corr.offset, - stride=corr.stride, + offset=m.offset, + stride=m.stride, ) ) elif isinstance(m, ConstantMap): new_output.append(m) else: - assert isinstance(m, DimensionMap) new_output.append( DimensionMap( input_dimension=new_input_dim_of[m.input_dimension], @@ -1780,7 +1776,7 @@ def _apply_vindex(transform: IndexTransform, selection: Any) -> IndexTransform: # Separate array dims and slice dims array_dims: list[int] = [] - slice_dims: list[int] = [] + slices: dict[int, slice] = {} arrays: list[np.ndarray[Any, np.dtype[np.intp]]] = [] for i, sel in enumerate(processed): @@ -1791,7 +1787,8 @@ def _apply_vindex(transform: IndexTransform, selection: Any) -> IndexTransform: array_dims.append(i) arrays.append(sel) else: - slice_dims.append(i) + slices[i] = sel + slice_dims = list(slices) # Determine the shared domain without expanding the index arrays: their # singleton axes describe independent dependencies during grid partitioning. @@ -1800,12 +1797,10 @@ def _apply_vindex(transform: IndexTransform, selection: Any) -> IndexTransform: # Slice dimensions (preserved-domain literal semantics, like basic indexing) slice_dim_params: dict[int, tuple[int, int, int]] = {} slice_bounds: list[tuple[int, int]] = [] - for old_dim in slice_dims: - sel = processed[old_dim] - assert isinstance(sel, slice) + for old_dim, sl in slices.items(): lo = transform.domain.inclusive_min[old_dim] hi = transform.domain.exclusive_max[old_dim] - start, step, origin, size = _resolve_slice_ts(sel, old_dim, lo, hi) + start, step, origin, size = _resolve_slice_ts(sl, old_dim, lo, hi) slice_bounds.append((origin, origin + size)) slice_dim_params[old_dim] = (start, step, origin) diff --git a/packages/zarr-indexing/src/zarr_indexing/writer.py b/packages/zarr-indexing/src/zarr_indexing/writer.py index 3301e22574..f18146b71b 100644 --- a/packages/zarr-indexing/src/zarr_indexing/writer.py +++ b/packages/zarr-indexing/src/zarr_indexing/writer.py @@ -123,8 +123,7 @@ def _write_affine(source: Any, transform: IndexTransform, broadcast: Any) -> boo output.input_dimension for output in transform.output if isinstance(output, DimensionMap) ] if ( - any(isinstance(output, ArrayMap) for output in transform.output) - or len(set(axes)) != len(axes) + len(set(axes)) != len(axes) or any(shape[axis] != 1 for axis in range(len(shape)) if axis not in axes) or any( isinstance(output, DimensionMap) and output.stride == 0 for output in transform.output @@ -134,19 +133,21 @@ def _write_affine(source: Any, transform: IndexTransform, broadcast: Any) -> boo selection: list[int | slice] = [] reverse_axes: list[int] = [] for output in transform.output: + if isinstance(output, ArrayMap): + # An index array is a gather, not one basic assignment. + return False if isinstance(output, ConstantMap): selection.append(output.offset) + continue + axis = output.input_dimension + start = output.offset + output.stride * transform.domain.inclusive_min[axis] + if output.stride < 0: + reverse_axes.append(axes.index(axis)) + selection.append( + slice(start + output.stride * (shape[axis] - 1), start + 1, -output.stride) + ) else: - assert isinstance(output, DimensionMap) - axis = output.input_dimension - start = output.offset + output.stride * transform.domain.inclusive_min[axis] - if output.stride < 0: - reverse_axes.append(axes.index(axis)) - selection.append( - slice(start + output.stride * (shape[axis] - 1), start + 1, -output.stride) - ) - else: - selection.append(slice(start, start + output.stride * shape[axis], output.stride)) + selection.append(slice(start, start + output.stride * shape[axis], output.stride)) unused = tuple(axis for axis in range(len(shape)) if axis not in axes) squeezed = np.squeeze(broadcast, axis=unused) remaining = sorted(axes) diff --git a/pyproject.toml b/pyproject.toml index 4f256c0cb7..29f2a2285b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -402,8 +402,6 @@ ignore = [ "**/tests/**" = ["S101"] "**/examples/**" = ["S101"] "**/testing/**" = ["S101"] -# zarr-indexing still has asserts in runtime code; tracked as a separate change. -"packages/zarr-indexing/src/**" = ["S101"] [tool.mypy] files = ["src", "tests"] From b3c5972697bec6c2286119f8abfffe0237b85ff7 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 16 Sep 2026 19:45:28 +0200 Subject: [PATCH 19/29] fix(zarr-metadata): v2 array document is open and filters may be empty (#4365) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(zarr-metadata): v2 array document is open and filters may be empty Two v2 structural rules were stricter than the spec, and the shared conformance corpus (zarr-metadata.js, conformance/v2_array.json cases 5 and 8) has been updated first; this brings the reference implementation back into agreement. - Members outside the .zarray definition were rejected. The spec: "Other keys SHOULD NOT be present within the metadata object and SHOULD be ignored by implementations" — a recommendation, unlike .zgroup's "Other keys MUST NOT be present". Extras are now tolerated by the validator, dropped by the model, and permitted by the Pydantic schema (the TypedDict is open, like the v3 one). The on-disk `.zarray` rule that `attributes` belongs in `.zattrs` is unchanged. - filters: [] was rejected ("expected at least one filter"). The spec: "A list of JSON objects providing codec configurations, or null" — an empty list is a list. The Pydantic schema's min_length is dropped. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): changelog fragment for #4365 Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): link the spec text the v2 open-array change cites Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): link the spec text behind every spec statement Every docstring or comment that cites the Zarr spec or a zarr-extensions README now carries a commit-pinned permalink with a line range (zarr-specs fc7dd9c; zarr-extensions 4da7b37, the registry commit the TypeScript port vendors). Unpinned zarr-extensions `tree/main` page links are pinned the same way. Three statements were wrong or stale and are corrected: - zstd: `checksum` was typed required "per the proposed specification" (zarr-specs PR #256, never merged). The published zarr-extensions entry makes it optional ("Should be omitted if false"; schema requires only `level`), so it is now `NotRequired[bool]`. - v3 consolidated metadata was described as "not a spec artifact"; since zarr-specs #373 the core spec names the field and fixes its envelope (core/index.rst L802-L816); the entry format remains a convention. - The v2 array `*Partial` docstring spoke of a "closed shape"; the array document is open (other keys SHOULD be ignored), unlike `.zgroup`. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- packages/zarr-metadata/changes/4365.bugfix.md | 1 + .../src/zarr_metadata/_pydantic_schema.py | 12 +++- .../src/zarr_metadata/model/_array.py | 11 ++-- .../src/zarr_metadata/model/_group.py | 5 +- .../src/zarr_metadata/model/_validation.py | 14 ++--- .../src/zarr_metadata/v2/array.py | 10 ++-- .../src/zarr_metadata/v2/attributes.py | 1 + .../src/zarr_metadata/v2/group.py | 14 +++-- .../v3/chunk_grid/rectilinear.py | 4 +- .../zarr_metadata/v3/chunk_grid/regular.py | 2 + .../v3/chunk_key_encoding/__init__.py | 4 +- .../v3/chunk_key_encoding/default.py | 1 + .../zarr_metadata/v3/chunk_key_encoding/v2.py | 1 + .../src/zarr_metadata/v3/codec/blosc.py | 2 + .../src/zarr_metadata/v3/codec/bytes.py | 2 + .../src/zarr_metadata/v3/codec/cast_value.py | 4 +- .../src/zarr_metadata/v3/codec/crc32c.py | 2 + .../src/zarr_metadata/v3/codec/gzip.py | 1 + .../zarr_metadata/v3/codec/scale_offset.py | 3 +- .../v3/codec/sharding_indexed.py | 4 ++ .../src/zarr_metadata/v3/codec/transpose.py | 2 + .../src/zarr_metadata/v3/codec/zstd.py | 20 ++++--- .../src/zarr_metadata/v3/consolidated.py | 15 +++-- .../src/zarr_metadata/v3/data_type/bytes.py | 2 +- .../src/zarr_metadata/v3/data_type/float16.py | 7 ++- .../src/zarr_metadata/v3/data_type/float32.py | 7 ++- .../src/zarr_metadata/v3/data_type/float64.py | 7 ++- .../v3/data_type/numpy_datetime64.py | 2 +- .../v3/data_type/numpy_timedelta64.py | 2 +- .../src/zarr_metadata/v3/data_type/raw.py | 9 ++- .../src/zarr_metadata/v3/data_type/string.py | 2 +- .../src/zarr_metadata/v3/data_type/struct.py | 2 +- .../zarr-metadata/tests/model/test_array.py | 57 ++++++++++++------- .../zarr-metadata/tests/model/test_group.py | 5 +- .../tests/model/test_pydantic_module.py | 19 +++++-- 35 files changed, 176 insertions(+), 80 deletions(-) create mode 100644 packages/zarr-metadata/changes/4365.bugfix.md diff --git a/packages/zarr-metadata/changes/4365.bugfix.md b/packages/zarr-metadata/changes/4365.bugfix.md new file mode 100644 index 0000000000..6cda30f2d2 --- /dev/null +++ b/packages/zarr-metadata/changes/4365.bugfix.md @@ -0,0 +1 @@ +The v2 array validator follows the spec on two points it was stricter than: members outside the `.zarray` definition are tolerated and dropped (the spec says other keys ["SHOULD NOT be present ... and SHOULD be ignored"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), unlike `.zgroup`'s ["MUST NOT"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313)), and `filters: []` is accepted (["a list of JSON objects providing codec configurations, or null"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79) sets no minimum). The Pydantic schema follows: `ZarrV2ArrayMetadataJSON` is open and the filter pipeline has no minimum length. diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index e9792d6931..51ec633b82 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -38,7 +38,7 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] -ZarrV2FilterPipelineJSON = Annotated[tuple[ZarrV2CodecMetadata, ...], Field(min_length=1)] +ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...] class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): @@ -74,8 +74,14 @@ class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None] -class ZarrV2ArrayMetadataJSON(TypedDict, closed=True): - """Schema input for the closed, merged v2 array representation.""" +class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for the merged v2 array representation. + + Open, like the runtime validator: the v2 spec says other keys "SHOULD NOT + be present ... and SHOULD be ignored by implementations" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92); the group document's "MUST NOT" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313) keeps + `ZarrV2GroupMetadataJSON` closed. + """ zarr_format: Literal[2] shape: tuple[NonNegativeInt, ...] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 0b562bc188..db708c0152 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -51,7 +51,7 @@ class ZarrV3NamedConfig: Bare names and missing configurations normalize to an empty configuration. Bare names and missing `must_understand` members normalize to the spec's - implicit `True` value. + implicit `True` value (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1573). """ name: str @@ -107,8 +107,8 @@ def must_understand_subset( ) -> dict[str, ZarrV3ExtensionField]: """The subset of `extra_fields` the reader is obligated to understand. - Per the v3 spec, an extension field is implicitly `must_understand: True` - unless it explicitly says otherwise, and an implementation MUST fail to + Per the v3 spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578), an extension field is implicitly `must_understand: + True` unless it explicitly says otherwise, and an implementation MUST fail to open a group or array carrying fields it does not recognize that are not explicitly `must_understand: false`. A non-mapping field value cannot carry the explicit waiver, so it always requires understanding (the @@ -310,7 +310,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with - `must_understand: false` (the spec's implicit-true rule). A compliant + `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). A compliant reader MUST fail to open the array if this contains any field it does not recognize; the model layer only partitions by obligation, since recognition is reader-specific. @@ -428,7 +428,8 @@ def to_json(self) -> ZarrV2ArrayMetadataJSON: `attributes` is included when set (even empty). This is not the on-disk `.zarray` content: a conforming `.zarray` must exclude `attributes` (they live in the sibling `.zattrs` file). Use - `to_key_value` to produce the spec-conforming split for storage. + `to_key_value` to produce the spec-conforming split for storage + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). """ # to_json output shares no mutable state with the model: every value # that can hold a mutable container is deep-copied. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 63dfe5611f..bb2e14da1b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -168,7 +168,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with - `must_understand: false` (the spec's implicit-true rule). A compliant + `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). A compliant reader MUST fail to open the group if this contains any field it does not recognize; the model layer only partitions by obligation, since recognition is reader-specific. @@ -299,7 +299,8 @@ def to_json(self) -> ZarrV2GroupMetadataJSON: `attributes` is included when set (even empty). This is not the on-disk `.zgroup` content: a conforming `.zgroup` must exclude `attributes` (they live in the sibling `.zattrs` file). Use - `to_key_value` to produce the spec-conforming split for storage. + `to_key_value` to produce the spec-conforming split for storage + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313; https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). """ # to_json output shares no mutable state with the model. out: ZarrV2GroupMetadataJSON = {"zarr_format": self.zarr_format} diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index a12e1911b1..d1cd74b719 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -584,10 +584,12 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: if not isinstance(value, Mapping): return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) + # Unlike the group document ("Other keys MUST NOT be present", + # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), the v2 array document is open: other keys "SHOULD NOT be + # present within the metadata object and SHOULD be ignored by + # implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so members outside + # ARRAY_METADATA_STANDARD_KEYS_V2 are not problems. problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) - problems.extend( - _unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) - ) problems.extend(_check_literal(doc, "zarr_format", 2)) shape_problems = _validate_dim_sequence(doc, "shape") chunks_problems = _validate_dim_sequence(doc, "chunks") @@ -642,10 +644,8 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: ) ) elif filters is not None: - if len(cast("Sequence[object]", filters)) == 0: - problems.append( - ValidationProblem(("filters",), "expected at least one filter", "invalid_value") - ) + # "A list of JSON objects providing codec configurations, or + # null" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79): an empty list is a list. for index, item in enumerate(cast("Sequence[object]", filters)): problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index e026e5c655..7e4f87be7c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -81,8 +81,8 @@ class ZarrV2ArrayMetadataJSON(TypedDict): """ Zarr v2 array metadata document, in-memory merged form. - Models the union of `.zarray` (the spec-defined fields) and `.zattrs` - (user attributes). On disk, attributes live in a sibling `.zattrs` file + Models the union of `.zarray` (the spec-defined fields, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L51-L92) + and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk, attributes live in a sibling `.zattrs` file and are not part of `.zarray`; this type folds them in as the `attributes` field so a single TypedDict represents the complete in-memory state of a v2 array node. Consumers that read or write a @@ -126,8 +126,10 @@ class ZarrV2ArrayMetadataJSONPartial(TypedDict, total=False): `tests/test_partial_equivalence.py` passes without special-casing those fields (PEP 655 explicitly permits `NotRequired` inside `total=False`). - Note: v2 array metadata has no `extra_items` setting (the v2 spec has no - extension-field concept), so this partial inherits the same closed shape. + Note: v2 array metadata has no `extra_items` setting: the v2 spec has no + extension-field concept, and other `.zarray` keys "SHOULD be ignored by + implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so nothing beyond the spec-defined + fields is modeled. Drift between this type and `ZarrV2ArrayMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py index 68785d1660..74e9fe938f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py @@ -15,6 +15,7 @@ Spec-defined keys for arrays / groups live in sibling `.zarray` / `.zgroup` files (modeled by `ZarrV2ZArrayJSON` / `ZarrV2ZGroupJSON`). This type does not constrain the keys or values of the attributes mapping. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/group.py b/packages/zarr-metadata/src/zarr_metadata/v2/group.py index 34d72742c2..3a645dc29b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/group.py @@ -16,9 +16,10 @@ class ZarrV2ZGroupJSON(TypedDict): On-disk `.zgroup` file content. Strict shape of the JSON document persisted at `/.zgroup` for - a v2 group. The spec defines exactly one field. User attributes live - in a sibling `.zattrs` file and are NOT part of this type; see - `ZarrV2ZAttrsJSON`. + a v2 group. The spec defines exactly one field and forbids others. User + attributes live in a sibling `.zattrs` file and are NOT part of this + type; see `ZarrV2ZAttrsJSON`. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313 See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -30,8 +31,8 @@ class ZarrV2GroupMetadataJSON(TypedDict): """ Zarr v2 group metadata document, in-memory merged form. - Models the union of `.zgroup` (the spec-defined `zarr_format` field) - and `.zattrs` (user attributes). On disk these are persisted as two + Models the union of `.zgroup` (the spec-defined `zarr_format` field, + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313) and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk these are persisted as two separate files; this type folds them so a single TypedDict represents the complete in-memory state of a v2 group node. Consumers that read or write the real on-disk files should use `ZarrV2ZGroupJSON` (strict @@ -64,7 +65,8 @@ class ZarrV2GroupMetadataJSONPartial(TypedDict, total=False): `total=False`). Note: v2 group metadata has no `extra_items` setting (the v2 spec has no - extension-field concept), so this partial inherits the same closed shape. + extension-field concept, and `.zgroup` forbids other keys outright: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), so this partial inherits the same closed shape. Drift between this type and `ZarrV2GroupMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index e3551e3c72..480f5538fa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -1,7 +1,7 @@ """ Rectilinear chunk grid (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-grids/rectilinear +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md """ from typing import Final, Literal @@ -42,6 +42,8 @@ class RectilinearChunkGridObject(TypedDict): `kind` and `chunk_shapes` are required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md#L59-L62 + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 2f7a089934..a0b33688c8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -33,6 +33,8 @@ class RegularChunkGridObject(TypedDict): `chunk_shape` is required and has no default, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L528-L537 ("must be an object with the names name and configuration") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py index b6774efbe3..fa4a2aaf84 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py @@ -6,7 +6,9 @@ - `default` -- v3 default encoding (`/`-separated) - `v2` -- v2-compatibility encoding (`.`-separated by default) -Both are defined by the v3 core spec. +Both are defined by the v3 core spec: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst The `ChunkKeyEncodingMetadata` aliases re-exported here are the canonical type for each encoding's permitted JSON shapes. For the underlying diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index c783861b34..39c5385ce4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -31,6 +31,7 @@ class DefaultChunkKeyEncodingConfiguration(TypedDict): """Configuration for the default chunk key encoding. `separator` is optional and defaults to `"/"` per spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst#L27-L29 """ separator: NotRequired[DefaultChunkKeyEncodingSeparator] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index e2783d296d..3face04f91 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -37,6 +37,7 @@ class V2ChunkKeyEncodingConfiguration(TypedDict): """Configuration for the v2 chunk key encoding. `separator` is optional and defaults to `"."` per spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst#L27-L29 """ separator: NotRequired[V2ChunkKeyEncodingSeparator] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 5a986c8260..4c3631a032 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -50,6 +50,8 @@ class BloscCodecObject(TypedDict): The configuration has multiple required keys (`cname`, `clevel`, `shuffle`, `blocksize`), so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/blosc/index.rst#L57-L98 (configuration parameters) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 04e746f898..43b165f9e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -51,6 +51,8 @@ class BytesCodecObject(TypedDict): at runtime based on data type), so the spec's short-hand-name form is permitted in addition to the object form, and the object form may itself omit `configuration` entirely. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69 ("endian: Required for data types for which endianness is applicable") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 96c39e5916..d1b878c95c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -1,7 +1,7 @@ """ Cast-value codec types. -See https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/cast_value +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ from typing import Final, Literal, NotRequired @@ -89,6 +89,8 @@ class CastValueCodecObject(TypedDict): `configuration.data_type` is required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md#L33-L36 and #L46-L48 (required fields) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 6b9b46c43d..aa72fcae5a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -27,6 +27,7 @@ class Crc32cCodecObject(TypedDict): Per spec the codec has no configuration fields. `configuration` is optional and, if present, should be an empty mapping. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/crc32c/index.rst#L63-L66 """ name: Crc32cCodecName @@ -39,6 +40,7 @@ class Crc32cCodecObject(TypedDict): The spec's Extension definition allows extensions with no required configuration to be encoded as a bare short-hand name. CRC32C has no configuration, so both forms are valid. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 3b9936f8cd..9a9647263c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -26,6 +26,7 @@ class GzipCodecConfiguration(TypedDict): is required for the metadata to fulfill its reproducibility role, even though the spec text does not mark it required with RFC 2119 keywords. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/gzip/index.rst#L57-L66 """ level: int diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 9701db8497..abf11b4211 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -1,7 +1,7 @@ """ Scale-offset codec types. -See https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/scale_offset +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md """ from typing import Final, Literal, NotRequired @@ -38,6 +38,7 @@ class ScaleOffsetCodecObject(TypedDict): `configuration` is itself optional per spec — when both `offset` and `scale` are at their identity defaults, the codec is a no-op and the entire `configuration` field may be omitted. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md#L18 and #L35 """ name: ScaleOffsetCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a8c9247ec4..ac40e78b7b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -35,8 +35,10 @@ class ShardingIndexedCodecConfiguration(TypedDict): `index_codecs` is the codec pipeline applied to the shard index; it must be deterministic (no variable-size compression). + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L147-L155 `index_location` defaults to `"end"` per the spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L157-L161 """ chunk_shape: tuple[int, ...] @@ -58,6 +60,8 @@ class ShardingIndexedCodecObject(TypedDict): The configuration has multiple required keys (`chunk_shape`, `codecs`, `index_codecs`), so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L141-L155 (required members) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index ac469b356a..41d9bcdc6d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -38,6 +38,8 @@ class TransposeCodecObject(TypedDict): `order` is required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/transpose/index.rst#L60-L66 ("order: Required") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index c0faa64bed..9fdf177b4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -1,12 +1,12 @@ """ Zstandard codec types. -See https://github.com/zarr-developers/zarr-specs/pull/256 (unmerged at -time of writing; the configuration shape below reflects the proposed -specification). +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md +(the zarr-extensions registry entry; zarr-specs PR #256, which first +proposed the codec, was never merged). """ -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import TypedDict @@ -21,11 +21,13 @@ class ZstdCodecConfiguration(TypedDict): """ Configuration for the Zarr v3 `zstd` codec. - Both fields are required per the proposed specification. + `level` is required; `checksum` is optional ("Should be omitted if + false"). + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 """ level: int - checksum: bool + checksum: NotRequired[bool] class ZstdCodecObject(TypedDict): @@ -38,8 +40,10 @@ class ZstdCodecObject(TypedDict): ZstdCodecMetadata = ZstdCodecObject """Permitted JSON shape for `zstd` codec metadata. -Both `level` and `checksum` are required, so only the object form is -valid; the short-hand-name form is not permitted by the spec for this codec. +`level` is required, so only the object form is valid; the short-hand-name +form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index a9fe0c1f8f..ba7c9aec0d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -25,8 +25,11 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict): Inline consolidated metadata embedded in a v3 group. The `metadata` map contains only v3 array and group entries. V2 entries - are excluded from this interoperability convention by design; the v3 core - specification does not define consolidated metadata. + are excluded from this interoperability convention by design. The v3 core + specification acknowledges `consolidated_metadata` as a historical + additional field and fixes this envelope, but leaves the entries to the + reference implementation: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 """ kind: Literal["inline"] @@ -38,8 +41,12 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict): """The key under which consolidated metadata is embedded in a v3 group document. Unlike the v2 `.zmetadata` file, this is not a store key: consolidated metadata -is carried as an extension field inside the group's own `zarr.json`. Like its v2 -counterpart it is a reference-implementation convention, not a spec artifact. +is carried as an additional field inside the group's own `zarr.json`. The core +spec names the field and its envelope ("For historical reasons, group metadata +documents may contain an additional field named ``consolidated_metadata``"); +the entry format, like the v2 counterpart, is a reference-implementation +convention. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index c7eed64f0f..5892a1bdaa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -1,7 +1,7 @@ """ Zarr `bytes` data type (variable-length raw bytes, zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/bytes +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md """ import re diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 41eec441df..264b2c262c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -14,7 +14,10 @@ """Literal type of the `data_type` field for `float16`.""" Float16SpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""Named non-finite fill values permitted by the spec for IEEE 754 floats.""" +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" HexFloat16 = NewType("HexFloat16", str) """A 6-character hex string (`0x` + 4 hex digits) encoding the @@ -45,7 +48,7 @@ def hex_float16(value: str) -> HexFloat16: CANONICAL_NAN_HEX_FLOAT16: Final = "0x7e00" """Canonical hex form of the float16 NaN sentinel `"NaN"`. -Per spec the named `"NaN"` sentinel denotes the float with sign=0, the +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 37b7d4f6e8..3b2e786f07 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -14,7 +14,10 @@ """Literal type of the `data_type` field for `float32`.""" Float32SpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""Named non-finite fill values permitted by the spec for IEEE 754 floats.""" +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" HexFloat32 = NewType("HexFloat32", str) """A 10-character hex string (`0x` + 8 hex digits) encoding the @@ -45,7 +48,7 @@ def hex_float32(value: str) -> HexFloat32: CANONICAL_NAN_HEX_FLOAT32: Final = "0x7fc00000" """Canonical hex form of the float32 NaN sentinel `"NaN"`. -Per spec the named `"NaN"` sentinel denotes the float with sign=0, the +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 9a5cf98288..21373d63f6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -14,7 +14,10 @@ """Literal type of the `data_type` field for `float64`.""" Float64SpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""Named non-finite fill values permitted by the spec for IEEE 754 floats.""" +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" HexFloat64 = NewType("HexFloat64", str) """An 18-character hex string (`0x` + 16 hex digits) encoding the @@ -46,7 +49,7 @@ def hex_float64(value: str) -> HexFloat64: CANONICAL_NAN_HEX_FLOAT64: Final = "0x7ff8000000000000" """Canonical hex form of the float64 NaN sentinel `"NaN"`. -Per spec the named `"NaN"` sentinel denotes the float with sign=0, the +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 8784160f71..bed264e96e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -1,7 +1,7 @@ """ Zarr `numpy.datetime64` data type (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.datetime64 +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md """ from typing import Final, Literal diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index f5c8c77bf8..ceb31d8c40 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -1,7 +1,7 @@ """ Zarr `numpy.timedelta64` data type (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.timedelta64 +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.timedelta64/README.md """ from typing import Final, Literal diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index c9c688c9fa..66e69c9b53 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -4,14 +4,19 @@ The `data_type` value is a string of the form `r` where `N` is a positive multiple of 8 (e.g. `r8`, `r16`, `r24`). -See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html +See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html +(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47; fill value: https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L97-L99) """ import re from typing import Final, NewType RawBytesDataTypeName = NewType("RawBytesDataTypeName", str) -"""A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`).""" +"""A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`). + +"raw bits, variable size given by *, limited to be a multiple of 8": + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47 +""" _RAW_BYTES_RE: Final = re.compile(r"^r(\d+)$") diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 0a778ccecc..1e93a95d50 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -1,7 +1,7 @@ """ Zarr `string` data type (variable-length utf-8, zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/string +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md """ from typing import Final, Literal diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index b1b6b50308..5795c927f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -1,7 +1,7 @@ """ Zarr `struct` data type (heterogeneous record, zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/blob/main/data-types/struct/README.md +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md """ from collections.abc import Mapping diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 95dc7aea3a..c30f457618 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -180,6 +180,7 @@ def test_json_value_type_accepts_json_shapes() -> None: def test_string_nan_fill_value_roundtrips() -> None: # Non-finite floats are represented as the spec strings ("NaN", "Infinity", # "-Infinity") by the caller — the metadata layer does not interpret dtypes. + # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 # The string form round-trips cleanly under default dataclass equality, # unlike a raw float('nan') (which is an invalid fill_value the caller must # not pass). @@ -850,21 +851,31 @@ def test_v2_from_key_value_remerges_zattrs() -> None: assert model.shape == (10,) -@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) -def test_v2_from_key_value_rejects_zarray_extra_members(extra_key: str) -> None: - """Raw `.zarray` documents reject every non-spec member.""" +def test_v2_from_key_value_rejects_zarray_attributes() -> None: + """A raw `.zarray` document must not carry `attributes`: they live in `.zattrs`.""" doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc.pop("attributes", None) - doc[extra_key] = {} + doc["attributes"] = {} with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ - ((extra_key,), "invalid_value") + (("attributes",), "invalid_value") ] +def test_v2_from_key_value_ignores_zarray_extra_members() -> None: + """Other raw `.zarray` members "SHOULD be ignored by implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92).""" + doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc.pop("attributes", None) + doc["vendor_extension"] = {} + + model = ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + assert "vendor_extension" not in model.to_json() + + def test_v2_zattrs_presence_round_trips() -> None: """The .zattrs file's presence is part of the store: an absent file reads as UNSET and emits no .zattrs; an explicit empty file reads as {} and @@ -1326,16 +1337,16 @@ def test_v2_shape_and_chunks_must_have_equal_rank() -> None: ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) -def test_v2_filters_must_be_nonempty_when_present() -> None: - """A non-null v2 filter sequence contains one or more codec configurations.""" +def test_v2_filters_may_be_empty() -> None: + """An empty filter list is a list: the spec says "a list ... or null", with no minimum. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79 + """ doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc["filters"] = () - assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ - (("filters",), "invalid_value") - ] - with pytest.raises(MetadataValidationError, match="at least one filter"): - ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + assert validate_array_metadata_v2(doc) == [] + assert ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}).filters == () def test_v2_dimension_separator_literal_enforced() -> None: @@ -1381,13 +1392,15 @@ def test_array_zarr_format_rejects_float( assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] -def test_array_v2_rejects_unknown_document_member() -> None: - """The closed v2 merged-document shape rejects undeclared members.""" +def test_array_v2_ignores_unknown_document_member() -> None: + """Other .zarray keys "SHOULD NOT be present ... and SHOULD be ignored": tolerated, dropped. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92 + """ doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} - assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ - (("unexpected",), "invalid_value") - ] + assert validate_array_metadata_v2(doc) == [] + assert "unexpected" not in ZarrV2ArrayMetadata.from_json(doc).to_json() def test_array_v3_from_json_materializes_abstract_containers() -> None: @@ -1636,7 +1649,10 @@ def test_must_understand_fields_partition() -> None: """must_understand_fields contains every extra field not explicitly waived with must_understand: false, including implicitly-true and non-mapping fields, so a reader can discharge the spec's fail-to-open duty by - subtracting the extensions it recognizes.""" + subtracting the extensions it recognizes. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1575-L1578 + """ model = ZarrV3ArrayMetadata.create_default( extra_fields={ "ext_a": {"name": "a", "must_understand": False}, @@ -1662,7 +1678,10 @@ def test_dimension_names_null_field_rejected() -> None: """A dimension_names field whose VALUE is null is invalid: the spec permits null as an element (an unnamed dimension), never as the field value — "not specified" is spelled by omitting the key. Consumers bridging from an - in-memory None sentinel must drop the key, not write null.""" + in-memory None sentinel must drop the key, not write null. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L635-L638 + """ doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"dimension_names": None} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 4b8c22b84d..d280091ba6 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -503,7 +503,10 @@ def test_v2_consolidated_rejects_unknown_document_member() -> None: def test_group_must_understand_fields_partition() -> None: """The group model partitions extra fields by the spec's implicit-true rule, - like the array model.""" + like the array model. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1573 + """ model = ZarrV3GroupMetadata.create_default( extra_fields={ "waived": {"name": "w", "must_understand": False}, diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index d15b3f118c..80067e81ae 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -182,12 +182,14 @@ def test_array_schemas_reject_negative_dimensions() -> None: _assert_runtime_and_schema_reject(field_type, doc) -def test_v2_array_schema_rejects_empty_filters() -> None: - """The v2 schema mirrors the runtime one-or-more filter rule.""" +def test_v2_array_schema_allows_empty_filters() -> None: + """The v2 schema, like the runtime, takes "a list ... or null" at its word (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79).""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) doc["filters"] = [] + adapter = TypeAdapter(zmp.ZarrV2ArrayMetadata) - _assert_runtime_and_schema_reject(zmp.ZarrV2ArrayMetadata, doc) + assert adapter.validate_python(doc).filters == () + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) @pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) @@ -210,7 +212,6 @@ def test_metadata_field_schema_rejects_unknown_members() -> None: @pytest.mark.parametrize( ("field_type", "source"), [ - (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), (zmp.ZarrV2GroupMetadata, V2_GROUP_DOC), (zmp.ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC), ], @@ -225,6 +226,16 @@ def test_v2_schema_rejects_unknown_document_members( _assert_runtime_and_schema_reject(field_type, doc) +def test_v2_array_schema_allows_unknown_document_members() -> None: + """The v2 array document is open ("SHOULD be ignored", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), in runtime and schema.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["unexpected"] = 1 + adapter = TypeAdapter(zmp.ZarrV2ArrayMetadata) + + assert "unexpected" not in adapter.validate_python(doc).to_json() + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) + + def test_v3_array_schema_allows_unknown_extension_fields() -> None: """Schema constraints do not close the v3 top-level extension namespace.""" doc = json.loads(json.dumps(V3_ARRAY_DOC)) From b8cc68a0c8d2e103158e34e506c1b5429a0962b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 08:26:21 +0200 Subject: [PATCH 20/29] chore(deps): bump the python-dependencies group with 5 updates (#4371) Bumps the python-dependencies group with 5 updates: | Package | From | To | | --- | --- | --- | | [numpy](https://github.com/numpy/numpy) | `2.5.2` | `2.5.3` | | [hypothesis](https://github.com/HypothesisWorks/hypothesis) | `6.167.1` | `6.168.0` | | [uv](https://github.com/astral-sh/uv) | `0.12.9` | `0.12.12` | | [towncrier](https://github.com/twisted/towncrier) | `25.8.0` | `26.9.0` | | [ruff](https://github.com/astral-sh/ruff) | `0.16.5` | `0.16.6` | Updates `numpy` from 2.5.2 to 2.5.3 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.5.2...v2.5.3) Updates `hypothesis` from 6.167.1 to 6.168.0 - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/v6.167.1...v6.168.0) Updates `uv` from 0.12.9 to 0.12.12 - [Release notes](https://github.com/astral-sh/uv/releases) - [Changelog](https://github.com/astral-sh/uv/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/uv/compare/0.12.9...0.12.12) Updates `towncrier` from 25.8.0 to 26.9.0 - [Release notes](https://github.com/twisted/towncrier/releases) - [Changelog](https://github.com/twisted/towncrier/blob/trunk/NEWS.rst) - [Commits](https://github.com/twisted/towncrier/compare/25.8.0...26.9.0) Updates `ruff` from 0.16.5 to 0.16.6 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.16.5...0.16.6) --- updated-dependencies: - dependency-name: numpy dependency-version: 2.5.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: hypothesis dependency-version: 6.168.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-dependencies - dependency-name: uv dependency-version: 0.12.12 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies - dependency-name: towncrier dependency-version: 26.9.0 dependency-type: direct:development update-type: version-update:semver-major dependency-group: python-dependencies - dependency-name: ruff dependency-version: 0.16.6 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 8 +- uv.lock | 380 ++++++++++++++++++++++++------------------------- 2 files changed, 194 insertions(+), 194 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 29f2a2285b..4be9d3342e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,13 +113,13 @@ test = [ "pytest-cov==7.1.0", "pytest-accept==0.3.0", "numpydoc==1.10.0", - "hypothesis==6.167.1", + "hypothesis==6.168.0", "pytest-reportlog==1.0.0", "pytest-xdist==3.8.0", "pytest-benchmark==5.3.0", "pytest-codspeed==5.0.3", "tomlkit==0.15.1", - "uv==0.12.9", + "uv==0.12.12", ] remote-tests = [ {include-group = "test"}, @@ -131,7 +131,7 @@ remote-tests = [ "requests==2.34.2", ] release = [ - "towncrier==25.8.0", + "towncrier==26.9.0", ] docs = [ # Doc building @@ -143,7 +143,7 @@ docs = [ "mkdocs-redirects==1.2.3", "markdown-exec[ansi]==1.12.3", "griffe-inherited-docstrings==1.1.3", - "ruff==0.16.5", + "ruff==0.16.6", # Changelog generation {include-group = "release"}, # Optional dependencies to run examples diff --git a/uv.lock b/uv.lock index c5539e88e2..989f842645 100644 --- a/uv.lock +++ b/uv.lock @@ -1054,74 +1054,74 @@ wheels = [ [[package]] name = "hypothesis" -version = "6.167.1" +version = "6.168.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sortedcontainers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8cee74c1390b2932406faaab76980f18946f258fa5a8afca17189b3bc655/hypothesis-6.167.1.tar.gz", hash = "sha256:62eefcb4d2791423626e9901c3027a6e0c5ffda2ac0b44b3c7e797ab9d2d5a4c", size = 505849, upload-time = "2026-08-30T19:53:09.05Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/53/4d/3592ca336deafbd3e9b0f47dc4c727aa32d30e765ef6370da8ecd590d388/hypothesis-6.167.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d28118fd70e4e15ff9c308a98b312b544b6145ae45aaa3b566328c1fdee8058f", size = 785476, upload-time = "2026-08-30T19:51:02.154Z" }, - { url = "https://files.pythonhosted.org/packages/18/bf/e33c431148994cbcb3332c6df94b833ecfb4aa6a8e51ea4b83da55ddd581/hypothesis-6.167.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e517be7f82a0a917758cc489a88b826b5371f56381fd94b4a8a09ce82d8de406", size = 781033, upload-time = "2026-08-30T19:51:27.314Z" }, - { url = "https://files.pythonhosted.org/packages/94/a3/e0de9a82c7e790a1def0801076e0ef43110f98e95ed54a3554877d0cb66d/hypothesis-6.167.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f8cec74c4fad7aeb0852cb34c2134b16db05d878ad3946a53337dace7016f4", size = 1117814, upload-time = "2026-08-30T19:53:04.109Z" }, - { url = "https://files.pythonhosted.org/packages/71/a4/8dd6bdc909324d1c39da1c86d65f75512ae049c159952af4cfe8feb5f8d4/hypothesis-6.167.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd202d02d129197a5e771f8a11c7d30559927284c23ec3a8bd4f37a7955964d1", size = 1141639, upload-time = "2026-08-30T19:51:50.399Z" }, - { url = "https://files.pythonhosted.org/packages/fa/bd/c13ed6145c360d0770415efd7d5a7e63c29905aeef52ab88004fe7e7f924/hypothesis-6.167.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b1e393ab01b71f683ba2a783785871cc6b81a6e41017780c64a5bc0b99759ae", size = 1143334, upload-time = "2026-08-30T19:50:38.045Z" }, - { url = "https://files.pythonhosted.org/packages/c0/a8/060d79ed8504b54ced9ad16f33d674b1b98a9debe9733c02709d7dd5c71c/hypothesis-6.167.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c385c7741893404306f9e5559ab3835432e85e7c153e25f854c502c410bbcbb", size = 1163345, upload-time = "2026-08-30T19:53:06.583Z" }, - { url = "https://files.pythonhosted.org/packages/cb/f7/6e68e2b705f729a6f7b4f41030022b1a5264c5434d3bcd917233d6801c6a/hypothesis-6.167.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:94920ca1fae70c26b0bd3fabbeef9437ffc17a39fe85696fb9a86187d92f6dba", size = 1123029, upload-time = "2026-08-30T19:52:03.134Z" }, - { url = "https://files.pythonhosted.org/packages/a5/c0/d274fe37ed5ecd5ad8ed555edc1f5e2abc8e1c3be3d5404b7edd5cc353a8/hypothesis-6.167.1-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b8d90ded2ffdc7e56b5e571993f384b52fade0a7b424e614f999cc2491789970", size = 1154003, upload-time = "2026-08-30T19:50:55.053Z" }, - { url = "https://files.pythonhosted.org/packages/b4/2f/2b5bb386f43fc965eb86fd69fcb2bd62c08cb6d7c6708a40dc39b3b97440/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:40cd5de7dd252942a08480639f5850594b1aca4a463e8a7f15e1fb6c2c3760c1", size = 1293729, upload-time = "2026-08-30T19:52:59.481Z" }, - { url = "https://files.pythonhosted.org/packages/ac/32/22436b072d79011fe81abb933edcd2476057c7b971588c5f3caf07519a88/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:b9c33f921ddc7fea93660eca408b25fe755516e22ec7ab21cb9951031f1cd608", size = 1419248, upload-time = "2026-08-30T19:50:52.903Z" }, - { url = "https://files.pythonhosted.org/packages/9e/3e/abf39faaff0f78112112a82316a5c9fe472574480c1ecee526734775b812/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:495989cf0a5ee03f7f9598ee9efeaabf15fd861ec52b5a9d6435849453e17e5d", size = 1274903, upload-time = "2026-08-30T19:52:27.25Z" }, - { url = "https://files.pythonhosted.org/packages/29/83/69b89ed5692ba3aad117facfb9ce099633a22c35acc3d64829b72253ec8c/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:bc73c46ce8ff93b0eb220f2b75adcbd9fcc9112078a74d3522d2532ad8069bad", size = 1294185, upload-time = "2026-08-30T19:50:35.038Z" }, - { url = "https://files.pythonhosted.org/packages/2b/1e/55dfcbe45c72df0a5c5b86a6b7c9365121acab69c2cc060bd55336a48c8f/hypothesis-6.167.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:36e83e1d7e97aaacbf6cd778e14a841344f848a674b20dfe4fe997546a6a2151", size = 1330013, upload-time = "2026-08-30T19:52:54.841Z" }, - { url = "https://files.pythonhosted.org/packages/3b/a6/0a36cead4ccff58bedb5d1aa2f894f7a580c317424b4fa788c6b22232b2d/hypothesis-6.167.1-cp310-abi3-win32.whl", hash = "sha256:fb4d87454d2459c2ccb541a4c61c92ce13058b91305ed3304695a409a1d886e4", size = 671942, upload-time = "2026-08-30T19:51:32.732Z" }, - { url = "https://files.pythonhosted.org/packages/ec/5b/360285ed42109f5ef48d98ca9ffcf71d1477130c0ff1f1a8d535c8507259/hypothesis-6.167.1-cp310-abi3-win_amd64.whl", hash = "sha256:5e35f98b427bf438a946203426b485dd5b62485f3d5a69a0e0862870a545e518", size = 678637, upload-time = "2026-08-30T19:50:33.573Z" }, - { url = "https://files.pythonhosted.org/packages/79/2d/cc084c1a8bfa296048ec0461f0fa11731abe0097f5c2310c8d39d94c8dc4/hypothesis-6.167.1-cp310-abi3-win_arm64.whl", hash = "sha256:dd6a0808a2eb8b5b1ac06bca4244eee18ed2c0e7b105599e1662203d164317b5", size = 676657, upload-time = "2026-08-30T19:51:34.494Z" }, - { url = "https://files.pythonhosted.org/packages/72/82/07987292cfb59c73ce6574e2912c015f678d5aa4d8c0712b78ae4415a535/hypothesis-6.167.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1937ae4e23f7dde6d4202d3d08c2633bcd535a091bdf866b8799abaabcb1e6f0", size = 787050, upload-time = "2026-08-30T19:51:10.782Z" }, - { url = "https://files.pythonhosted.org/packages/c7/e9/0d37051bec44ec433d87da03c9a7fe389b1b58210790c1f166af249bf941/hypothesis-6.167.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1434bbd25d05aaf75c4e6829b4cad8e9931b690f840d92b747b2e6e5af575922", size = 778617, upload-time = "2026-08-30T19:50:31.939Z" }, - { url = "https://files.pythonhosted.org/packages/13/2a/60c18a493215c22c9cfcb4574b381497bd1971eed9fb5f9831b26e73cffb/hypothesis-6.167.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af3c09428e553b1dd2f9abbc4738377c58bf6d74cb0b8b528cc1dde3a9cdfbe8", size = 1116743, upload-time = "2026-08-30T19:50:49.293Z" }, - { url = "https://files.pythonhosted.org/packages/12/1f/b6796f11d6502e0b1764aec99f2792ca60382b6a45e26bbe163dc5757980/hypothesis-6.167.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04f807b85d425a7005e8a24498ca832bf5590f0d306737471d94c842569cecef", size = 1162718, upload-time = "2026-08-30T19:52:47.544Z" }, - { url = "https://files.pythonhosted.org/packages/a8/6c/e3cf35474b799e299fa08980b6756f500d730781686916ab65f87cbc0613/hypothesis-6.167.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:803c6a98ff66cee4caf03245bfd00e442a907264031b994a3a650dc6e4786f51", size = 1292417, upload-time = "2026-08-30T19:50:56.822Z" }, - { url = "https://files.pythonhosted.org/packages/9a/4c/875803c80c373a1628f8eb62f215ad23ce7b50ed61f884d6be0838ebea4a/hypothesis-6.167.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:21a7122ddf072906083e3704fe3961ecbc49d7d30a9b51bcd525d977b3afe65e", size = 1329035, upload-time = "2026-08-30T19:51:46.659Z" }, - { url = "https://files.pythonhosted.org/packages/90/a8/a8daed3796623884471dc0ee8ed63917b1e2b979b4074bcea19a964fcd71/hypothesis-6.167.1-cp312-cp312-win_amd64.whl", hash = "sha256:a2837c60d782eb0b8a910c541264675b9d11486e186af8c82a5e2920b5fe4fe8", size = 675966, upload-time = "2026-08-30T19:51:56.58Z" }, - { url = "https://files.pythonhosted.org/packages/4b/44/dca7b211804f60c789aced2792b1e7803ccd8b70b79041cbb92788df5d19/hypothesis-6.167.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6478d19a7887731cc2afaa1ec15f62811c9ceb6fd18e5b7563e0a18399a9528f", size = 786947, upload-time = "2026-08-30T19:51:29.165Z" }, - { url = "https://files.pythonhosted.org/packages/b9/6a/3cffa138492c9e3d5f98f4ff8b467273dc87af6ca3c18084272d106bde10/hypothesis-6.167.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8f13167a4b81c93e7e051d1f02790814a6495fb79cacf3fb89560a796a2f7d00", size = 778584, upload-time = "2026-08-30T19:52:25.091Z" }, - { url = "https://files.pythonhosted.org/packages/7a/7d/e8039791aaca3b21557bc520a71cdb88751892f66fd1a0a459b59872e463/hypothesis-6.167.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ef7dd225f7df7d74d1c5a905592cd8b4cd348e6be639b189a43def8b0b5dd79", size = 1116749, upload-time = "2026-08-30T19:52:38.178Z" }, - { url = "https://files.pythonhosted.org/packages/ee/b8/f9b8d93bd6178870f0daa868ca99915f6d9df1f99dc7291e9ce2743a6dc5/hypothesis-6.167.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d7585429f2263d3ceeb3474bae3871024630a7a598e71eeb4b0dcf03e291623", size = 1162599, upload-time = "2026-08-30T19:52:11.72Z" }, - { url = "https://files.pythonhosted.org/packages/a1/0d/53d419094e6f8a7e7377c09de15ac23f842ab698ff07241f7b73e19bd559/hypothesis-6.167.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fb9194f450417cf35f66b6c72737cc6b8f21f20567ba4d39822c64f0d1075784", size = 1292230, upload-time = "2026-08-30T19:52:49.732Z" }, - { url = "https://files.pythonhosted.org/packages/5c/df/cf4c482323ae4f06b5326b5bdd89cf17d8232fdb3186c9913e0b19a5fa58/hypothesis-6.167.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c63a0d292a5dde3c0fe999892e76d8375a003ca40c0c00d763f3360f91be5b96", size = 1328899, upload-time = "2026-08-30T19:51:05.366Z" }, - { url = "https://files.pythonhosted.org/packages/49/05/780c4b0396491d294fda69a541cb1dedb37fb9eb2e3a696e85fe19064c40/hypothesis-6.167.1-cp313-cp313-win_amd64.whl", hash = "sha256:ff07f98a0b230632bb2836b5dad3e94d85c114ae155a316afd251c58760958ae", size = 675927, upload-time = "2026-08-30T19:50:58.472Z" }, - { url = "https://files.pythonhosted.org/packages/6a/f1/1e602f090dcb7e38655f1f7909482742891332275fc01f241e255cdfa514/hypothesis-6.167.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:fcfc2a78fc1025644f889a74684b3201f4652ce8e6694c2a01af0f100d0348cf", size = 787054, upload-time = "2026-08-30T19:50:40.88Z" }, - { url = "https://files.pythonhosted.org/packages/52/57/cfa930719c7af5a33627abba826a3fa2efa61a5f23e38d4111eace5dfe53/hypothesis-6.167.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:769bdd9aa0af08c063327730ab6dc18b7a23837a2912f2aeaab3912f11a7e3ad", size = 778721, upload-time = "2026-08-30T19:50:36.503Z" }, - { url = "https://files.pythonhosted.org/packages/01/37/4c4bc3d319eac85bfe17515c9786bf49e57181ca8757886110d2cfb13d10/hypothesis-6.167.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d75d44bdead6679b6ee9a7c90d10207db865ca0c77c5212103b5ff421379f99e", size = 1116972, upload-time = "2026-08-30T19:51:52.472Z" }, - { url = "https://files.pythonhosted.org/packages/33/45/0d61ceef2739e7b96ea1faa0f3d5aa5917c8156797993bf3acbadfcd7f0a/hypothesis-6.167.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:742be00d7bb53d10634e6435e5b98f51fcdbe7ed377d473ab7387d9499c87169", size = 1162776, upload-time = "2026-08-30T19:51:09.084Z" }, - { url = "https://files.pythonhosted.org/packages/4a/cf/756666ce2262e90fd61fec41a95548cceab94b0669381d8f0387cd89af93/hypothesis-6.167.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b6fcdc8d03b37a902262be13112d113bb4ac87edf3b08afb47f3d1210deb038a", size = 1292748, upload-time = "2026-08-30T19:51:17.22Z" }, - { url = "https://files.pythonhosted.org/packages/e8/b4/87eb3c695d6c37fb44f4d49f9faa2033af496e24965658942a1706e22620/hypothesis-6.167.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56e7841514276c308c2bb4d033cf01860d0fc8c76e2b79ce748a9f123eaf83b", size = 1329101, upload-time = "2026-08-30T19:51:36.313Z" }, - { url = "https://files.pythonhosted.org/packages/a7/3b/87faa4a86533eaaa19037741fb9cdde8647f7ffdf8fd4279828ac9d81f8b/hypothesis-6.167.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:bbf4f0cad201d0b8e821e82ad828b2aec99ce6d9967779eecb2cad4d4a93debd", size = 618079, upload-time = "2026-08-30T19:52:43.226Z" }, - { url = "https://files.pythonhosted.org/packages/e0/46/96b7ac9605887447d267b4b3a9ecf61c6caaabf39eef667173b0cc9222b3/hypothesis-6.167.1-cp314-cp314-win_amd64.whl", hash = "sha256:3e04f6001299708b6fd4512267b189c0b029ef1e34500deb4e4c9639023598d7", size = 675812, upload-time = "2026-08-30T19:52:52.298Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f6/0337a91c50ce4be323c3d6aa852fcf08199ffbb1072da09fbe6d602f4dfe/hypothesis-6.167.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:47c99256df28555ecc2aed0e22ca17cd61c63c8c44207a07b4e402cc49661fae", size = 785525, upload-time = "2026-08-30T19:51:03.637Z" }, - { url = "https://files.pythonhosted.org/packages/5a/08/9bb52de855169d31888c7033ee2f94b94138fde021c1af9dbc7ba5e83cd5/hypothesis-6.167.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5d6614e88fd267bbd870e3ec02f8a5387897d2d573626a02f5ac06d81533afa6", size = 777142, upload-time = "2026-08-30T19:52:18.472Z" }, - { url = "https://files.pythonhosted.org/packages/85/79/f1a7e088e13a641357abb9b43d75c116c2a0902711b1a25a203864b96c9b/hypothesis-6.167.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27829aa89fe2e47c5c8d13b3ce31e0f53a01a98f76a4f99cfa3369ab35362f33", size = 1115311, upload-time = "2026-08-30T19:52:40.975Z" }, - { url = "https://files.pythonhosted.org/packages/e0/38/e28b1fc20bd3d67d43cf1aab7a15daa2a24ec01d17a153e82fcf38c882f3/hypothesis-6.167.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e936777d92ae27393b4a941839bbb43c1f339b5a0e394c7f3730454cdf091b3a", size = 1161238, upload-time = "2026-08-30T19:52:20.501Z" }, - { url = "https://files.pythonhosted.org/packages/5a/de/9b4fc7992166299e0fc5c13c8766919ae57d0fb9eed5319b7a3bad4f2f17/hypothesis-6.167.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:971ce0d8a367a37c4690b83a2e7f6ef832fa3543357eb6da0da27ba078e5088a", size = 1290974, upload-time = "2026-08-30T19:51:15.673Z" }, - { url = "https://files.pythonhosted.org/packages/cc/79/ca086eea02588212ab796ee4bd7fe6ed514e10d1a99967e478691608e8d9/hypothesis-6.167.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:af84ce2416be2a65bc0ea18e64d2dbb9796b7692593b5b2064d60ea1d52ec1e2", size = 1327969, upload-time = "2026-08-30T19:52:35.846Z" }, - { url = "https://files.pythonhosted.org/packages/26/79/1875380fa30e8411553e76b3e9695aca0845f9f265523f20f879bdea2b82/hypothesis-6.167.1-cp314-cp314t-win_amd64.whl", hash = "sha256:3b596efec5bd714588e3bb269544d993c5258c979f3a26f51fadf62c215d0e68", size = 675735, upload-time = "2026-08-30T19:52:22.821Z" }, - { url = "https://files.pythonhosted.org/packages/b0/45/59abecd75e52b9dfb5b3eb991276f54954c44917a1c83d148cfb3580bd39/hypothesis-6.167.1-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:c25c556d51d55d94988dc0a2c716d471ff19cf9632cd33f5e6db2914d802428a", size = 785097, upload-time = "2026-08-30T19:51:12.528Z" }, - { url = "https://files.pythonhosted.org/packages/01/7c/e6d978dc9564ba70352da60c00f55f6ad7d66d99ecbf336a228978206cb4/hypothesis-6.167.1-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:b57e950f9d5c93ca335bc612e8fa8fb49abb187c3fc9d5e7d9966d52eb27d747", size = 776798, upload-time = "2026-08-30T19:50:47.247Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a4/f8ecedcf96790aab69d750afe3fcbf503229d0bb4e0c32be655385a4fc8c/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0807ae8d399162827fc1c396ab4c697a41921c2a2baacfada439771e9dc2b867", size = 1115116, upload-time = "2026-08-30T19:52:16.029Z" }, - { url = "https://files.pythonhosted.org/packages/7e/97/8bca7c262ac4fcb1ee684c04e4ba75f26541d3a30417e3743d19912d257e/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:546fef39c7aadba74bf3e592585694a71340d1775e9b3274bb3f94106dbde4b7", size = 1137812, upload-time = "2026-08-30T19:51:25.507Z" }, - { url = "https://files.pythonhosted.org/packages/d6/07/9cb4a7fc2aa2b2ad063b446c378f0d7acfa5303e84afd1b1374ba23fd6f3/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a17a5618b6a5b84f17c8acb3bce37122647cf7a3e48b660a39d68a773bd627dd", size = 1140384, upload-time = "2026-08-30T19:52:05.264Z" }, - { url = "https://files.pythonhosted.org/packages/1c/18/813bf7efa18f11ce938a0da22a7518a54db46d4a181ebf4cb0a8061c263f/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ff5ad833480c1e34ae902cb52fc00802b08ac6c87bd22d7e5f04fd925869608", size = 1160569, upload-time = "2026-08-30T19:51:40.099Z" }, - { url = "https://files.pythonhosted.org/packages/52/1d/6658d9294ed33bb17da4acf06fe63b010b205ea1861f6921c38060793255/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:630eb37df80b5bc4ec6f391b13caaecc06942ff5da3aadebacf85f53bbc55757", size = 1120605, upload-time = "2026-08-30T19:53:01.848Z" }, - { url = "https://files.pythonhosted.org/packages/a6/81/a75821c0e879223a2635b8eded84ae874cb6c711b23e9930668008d0b13f/hypothesis-6.167.1-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bc4e65f7c43b187f7a40964706b5ded1073e0c1839e9fb5e041d7ed973bb65fe", size = 1149479, upload-time = "2026-08-30T19:51:30.972Z" }, - { url = "https://files.pythonhosted.org/packages/4e/f2/c8c3faf4ec796d6dbf36b84662806696434aef38616e5a65b46188c04262/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:e849f518cbc4e76ab15f2f1473c60dd3103da8d32399187325ceb84309105976", size = 1290423, upload-time = "2026-08-30T19:50:51.114Z" }, - { url = "https://files.pythonhosted.org/packages/a4/e3/e0903abe7e8634cedb5414931452e82daacdd3b8b46d6348bbefcaa45f2a/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:5c5a26d4d3dca0c84e01bde41df4cabaa5a373c7393f9eef372d19fe93b07ccd", size = 1415749, upload-time = "2026-08-30T19:51:48.456Z" }, - { url = "https://files.pythonhosted.org/packages/a6/9b/03f09c1ecac1dfb0f4cd7fcc6dc50d9c6ea8067b295a728e242650bafe32/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:4819adbc5911648f6bfaeb574f276add184b4b49f54731dbde46fd71256bb157", size = 1272086, upload-time = "2026-08-30T19:52:29.368Z" }, - { url = "https://files.pythonhosted.org/packages/7c/7c/7a63bfc0bfbaf000f71352c4faac72ff611376330a2ce2e9a1bf4668848a/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:3ad7206de9c398c8da5745b69b5ba2ef45100082eeb174656490bc4f262b112c", size = 1291553, upload-time = "2026-08-30T19:52:07.545Z" }, - { url = "https://files.pythonhosted.org/packages/e8/5c/8065bdab53bc81743ca68fc76ca53fc7531a5b3f01c0de4ba40467955d6a/hypothesis-6.167.1-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:96d5e8017a9508f06c8a61a6130cb0d0b4810847ed5c76923cb5cfb9952b31af", size = 1327734, upload-time = "2026-08-30T19:51:42.306Z" }, - { url = "https://files.pythonhosted.org/packages/0d/a0/5c15d480aea3a8e6e5c17c7cb1170171707ac643ffd319473bc194743ad8/hypothesis-6.167.1-cp315-abi3.abi3t-win32.whl", hash = "sha256:a4e4de36a397cba49d949d89cbc26135977c15f9d797caa95317962ceb5b5674", size = 669115, upload-time = "2026-08-30T19:51:14.192Z" }, - { url = "https://files.pythonhosted.org/packages/1b/36/4cf494bc96384189fedb7d3f272580315f2284a9f8a7f6a59796612eb76d/hypothesis-6.167.1-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:f6fe9c40ab14def363d9e7ab22863fa31652bd5e08f8495b34ff7bd0062b3f8d", size = 675438, upload-time = "2026-08-30T19:51:06.881Z" }, - { url = "https://files.pythonhosted.org/packages/b0/7f/db1a37e5f45be32c0e64f9ed1268eba56aeedcb2ef20d195fa60c6610347/hypothesis-6.167.1-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:627ce3bd166799a6c0ddcf1351049be5b9a772d5bce436216d42b41a935f42c0", size = 673123, upload-time = "2026-08-30T19:52:13.991Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/5a/ce/c0946bebffb99b62426a6a7643d4272cc6c5cf777a488b3b4d0ee724e960/hypothesis-6.168.0.tar.gz", hash = "sha256:72af51087b7b5ab21c49f0d502f803c20897678652835596bd2a8b169a39135e", size = 510805, upload-time = "2026-09-08T18:48:36.072Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/f8/8b2cc9ae7b439538f6f2d32a92892340b6343a6b04d171c516666901dedc/hypothesis-6.168.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:47b89491ff02e3ae9b302c440457938e87b47a45b9a1d98ff5575b6910d779e2", size = 791358, upload-time = "2026-09-08T18:47:37.076Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f4/4d7d897310cde5085779fb96feadb8529d98cb8e51ed7b24f7da9b6c6bdc/hypothesis-6.168.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1f4cd0ff11bd470a1a846296ed5fe55e84214194850370994fd1370fe73d3099", size = 787081, upload-time = "2026-09-08T18:47:16.227Z" }, + { url = "https://files.pythonhosted.org/packages/26/7b/9d52066d363faba7f3ac20ee60a3c696a475feb2c477d235d0d649d41cc1/hypothesis-6.168.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:732ae5d47482f99d8028cca096729625f05690a83f5e7ce31466e266155792f4", size = 1123850, upload-time = "2026-09-08T18:48:11.504Z" }, + { url = "https://files.pythonhosted.org/packages/50/cf/aa46d76fa7df43caf2c372e394fda84ce1dc08421814f8674a6b9295e2ea/hypothesis-6.168.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2085ee74ac3ab6b70e2f7ffae9b4cb74c246da2f574b2de81a0818a8a30f659f", size = 1147685, upload-time = "2026-09-08T18:46:58.219Z" }, + { url = "https://files.pythonhosted.org/packages/84/c2/78ed8c8d5aa37e4baae2a4b3e29687ee3d9b7f1a5e56ea5ea5ec7ec71ecb/hypothesis-6.168.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1894782fae5d9a7bb44e6dcf848ccb09ccb5babab48d8b5c31a0a7fc025b82a1", size = 1149294, upload-time = "2026-09-08T18:47:04.757Z" }, + { url = "https://files.pythonhosted.org/packages/78/7f/d57440f19e9de70e85359cf179ce786f309ee17609bd3c5a0113272875c0/hypothesis-6.168.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecf0ab13cef899efb816ffdd7963e0679f372520884ce06756c7642f3df94213", size = 1169729, upload-time = "2026-09-08T18:47:44.056Z" }, + { url = "https://files.pythonhosted.org/packages/a1/86/dc74410a186990bb22c2a3eea0e77804f2eb0f300860b46d7d8948073674/hypothesis-6.168.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3f6dcf66270278d078bed01b401f47db4e26456cd909d8e23c6b9366a6c0b131", size = 1129182, upload-time = "2026-09-08T18:46:33.382Z" }, + { url = "https://files.pythonhosted.org/packages/b2/5e/be048fc4f6dac831625e155bdf11e4233caf46bb54030391d6fba8e19449/hypothesis-6.168.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bfef4d46dbf1704a7b8fa3a78778651a2cb18870ca0a70da19c381646822b149", size = 1160180, upload-time = "2026-09-08T18:47:26.459Z" }, + { url = "https://files.pythonhosted.org/packages/0f/4a/15a34498a5f08720fbbdbbe4668a8050fe4e17c16c9eeb6f56a017f2fa6b/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1d1aa5b3484e329295d88488a5ba06243909e65c2ab616513c2d36721de4ed1d", size = 1299711, upload-time = "2026-09-08T18:47:28.34Z" }, + { url = "https://files.pythonhosted.org/packages/77/09/5354e0dae302ab98c4f0046b7e8699c2186ae4349397bda5d5c852bda68b/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3bc00fd8cda04b58e37a1163e8a65389b247b4f5ee547ae37d244a4960995517", size = 1425341, upload-time = "2026-09-08T18:46:56.785Z" }, + { url = "https://files.pythonhosted.org/packages/59/7c/3c0e1f59043ff128c70a51d298d3d6b5973525c357a1e1d0542dbc05ac90/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:990026952d5b2eca290c88f639ac639233f47e13dae338c6dfb6e4774bcab349", size = 1281063, upload-time = "2026-09-08T18:46:55.401Z" }, + { url = "https://files.pythonhosted.org/packages/40/dd/db884db9a7d42ae6b72a00c13c725940b638dfb263e618b22af59d5dfa2f/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:a74b0945acbbd552c7c2d0a99a3b5232962b8848c8eed1829451800a9bfcf00b", size = 1300247, upload-time = "2026-09-08T18:47:02.949Z" }, + { url = "https://files.pythonhosted.org/packages/99/8a/4ee9769e1d48676efb6a78a130f82e0d52d3f87b2055294102272a08615c/hypothesis-6.168.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a380b521b5a76a9e8917d64adcf7f861a45a4360a34b1579af14c5df8eb0377", size = 1336084, upload-time = "2026-09-08T18:48:05.675Z" }, + { url = "https://files.pythonhosted.org/packages/61/17/d4ed11bc99d205d6d2651a0f1a4874f377150f836b5a0849bd511d68a2eb/hypothesis-6.168.0-cp310-abi3-win32.whl", hash = "sha256:2264f15a1c80329e3ad48e39c44bd5c9429b7b04c9ee62cdd72f4b10aaac9f29", size = 677989, upload-time = "2026-09-08T18:47:24.722Z" }, + { url = "https://files.pythonhosted.org/packages/77/51/abf1fde7b8afab87db30afb73b3847e62440551d146472111cabeba2fe00/hypothesis-6.168.0-cp310-abi3-win_amd64.whl", hash = "sha256:5b54769033b84477931d2072e7133a7555e0de5c53fd5ca3bbde960762d7d31b", size = 684692, upload-time = "2026-09-08T18:46:24.449Z" }, + { url = "https://files.pythonhosted.org/packages/12/e2/64d79aed47a95186ce9edcef60eb4684870c72542da3b7027c2483d8ee8c/hypothesis-6.168.0-cp310-abi3-win_arm64.whl", hash = "sha256:112b0900059bf9d7d6528ed729770629ab146e0d133c4143b9bd4a01dc002bcc", size = 682709, upload-time = "2026-09-08T18:48:03.756Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5e/0035896c101f0484c364353f8ee30175eef8936b49171618677287fdd85d/hypothesis-6.168.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6b750390dac4429da0cb70ab3fe758457f0cea3d9c843d48c59d0690d1189fda", size = 793115, upload-time = "2026-09-08T18:48:21.352Z" }, + { url = "https://files.pythonhosted.org/packages/11/5c/938173e27df771cc6e92bc47f117a1b1be4a88fc6dc214f7fc65f9c7ad93/hypothesis-6.168.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8e4b2d434e0dd134f3d31ac1efc1825bf99730dfe70fec005ff66d7211836d79", size = 784634, upload-time = "2026-09-08T18:47:21.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/00/0b2c6ac07d519131f97712f2533750ffe9b2490eec8f518ef3a5ed2dd514/hypothesis-6.168.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d4d36ed2fd62de11382f1d608169c1ffa9a49d3b9351146d8ff87cb81a66f7", size = 1122855, upload-time = "2026-09-08T18:46:21.967Z" }, + { url = "https://files.pythonhosted.org/packages/8d/21/dde930fe43171cab37572bd993d70c2a271f240f428f8ccd64ec4c2d661b/hypothesis-6.168.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5920d267f7d8cfd376672f2bde5905cdf284d47519582e41ce7c142d48ee46c4", size = 1168932, upload-time = "2026-09-08T18:46:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/4c/5d/92b83c3d06194ec626e92723d0b0f70221ebf42d7cb355ed36929df6d735/hypothesis-6.168.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fb8cdf45361e259df86e19f8cd042ce2d6c7e6ad88fa631b78a4e3a83c2e572d", size = 1298566, upload-time = "2026-09-08T18:48:19.485Z" }, + { url = "https://files.pythonhosted.org/packages/9c/67/52de8bf3446e3d2d555b812d96673b31bb213b5b9d5804a666e5d0bba76e/hypothesis-6.168.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b3ce1cce70b25a37ed1a38a53ce7204785726c675c0f41a0f83c338a7e47b3d", size = 1335074, upload-time = "2026-09-08T18:48:01.561Z" }, + { url = "https://files.pythonhosted.org/packages/14/fd/e592773c1c0ce55e35d26ec55f75546bf1fd72ef5a5c520ed685969b40cb/hypothesis-6.168.0-cp312-cp312-win_amd64.whl", hash = "sha256:f62bdabf278db9ff61df5f3203d608949f0d893d0e30cdac3f2330e67e41ae68", size = 682026, upload-time = "2026-09-08T18:47:01.471Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e9/39bb8fcccfbafd10fcc777d583c6ebad5d2148e7d743cb350562f28e974f/hypothesis-6.168.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d55562bf8d41cfa18559c33f30cadf44ceac8e517509d7a022a9feace621f28", size = 793050, upload-time = "2026-09-08T18:47:56.045Z" }, + { url = "https://files.pythonhosted.org/packages/f7/dd/00fd32e8ec470535e0065cb8d6e175f9fc54b4d3a6f1269f6c487f8bd79d/hypothesis-6.168.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92cff497b92e2285ff6a94193fdee04aba483a4115d501c1f9a570bd103fcd20", size = 784558, upload-time = "2026-09-08T18:46:32.054Z" }, + { url = "https://files.pythonhosted.org/packages/d2/4d/553c47093f68bdbac0438e16c024ce97649b5804dc6972ef86b9bd2db8a1/hypothesis-6.168.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ff259260015f9be3756dcd4bc11c08e007314dec6b43d9a89084c4f34f94475", size = 1122841, upload-time = "2026-09-08T18:48:09.374Z" }, + { url = "https://files.pythonhosted.org/packages/43/d6/0b5940aa75e617c8fd12200bae24d1b71347362514e8210735c581d4d3d1/hypothesis-6.168.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35f1262831b5acc74ded15f629965daffcd657f6016ee04fc9605f6eb2b334c0", size = 1168825, upload-time = "2026-09-08T18:47:33.702Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c0/800de1231b2869b51409bbf85799d6f1bf49a00e0afff0aabc097aa8f1b7/hypothesis-6.168.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:046fe4bcfce2a2fa186ba9d96bbb62c25c2f6c2e4071f0783ed6b5cc481d0669", size = 1298433, upload-time = "2026-09-08T18:46:38.53Z" }, + { url = "https://files.pythonhosted.org/packages/be/35/9907667a30c1dbabbc44a09b4c25a0f575570937fcbbada924ae3a1dbf2a/hypothesis-6.168.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:24b52a2b1c8db6e1e516f9295c8e4ef7ef63303ff24fbbc5b35f4ff71dcd732c", size = 1334988, upload-time = "2026-09-08T18:48:29.395Z" }, + { url = "https://files.pythonhosted.org/packages/98/8a/7bf214e703fff532ed47cb52ab93ce4b7ea41e4e7084593fa02b740828b7/hypothesis-6.168.0-cp313-cp313-win_amd64.whl", hash = "sha256:ec0886fe0be9091669937989f9a662beca42ae14a4a6dab25491c2c63365f88d", size = 681990, upload-time = "2026-09-08T18:47:49.173Z" }, + { url = "https://files.pythonhosted.org/packages/2f/c1/64b36b250b1f66abb6ce8c81775d3373d149bc89cbea477ed71b57cf7d1b/hypothesis-6.168.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2df8afacf9261070795db36db4a394e3ccdbb663fd2d38c7a9fba0c836dcecc", size = 793101, upload-time = "2026-09-08T18:46:54.067Z" }, + { url = "https://files.pythonhosted.org/packages/6d/c4/494e42304b15f4ec649d36bbc3fc01cef1b63405cf30d4087ae07d048172/hypothesis-6.168.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9ba679f183c67adcb6f4ad93694beafb6da99fe691757f4e57b04ae77e581ba8", size = 784632, upload-time = "2026-09-08T18:48:31.551Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6d/90d874cb1d749f505749c9908f34e803b97b03457797d5893354980558bc/hypothesis-6.168.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d9a8574f80fc859313aee56167d202e8625c0eedd200971130f0839f06d1c93", size = 1123101, upload-time = "2026-09-08T18:48:17.362Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/ec893d0e5f4bcdd0121a8a4280f4e8aba3b4cdae01411f3236016ecd1f81/hypothesis-6.168.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deb02de608268928d779aa889b0a9d67794b1cc0c54a322cf19e386be8a46ca7", size = 1168962, upload-time = "2026-09-08T18:46:48.507Z" }, + { url = "https://files.pythonhosted.org/packages/11/f1/16ec2bddbaed461725d9aa5a80b43f1905ea50a08f689ec46f8966bb4f0f/hypothesis-6.168.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:076a2096c34448931c3cfeb2eb7a6b843a56ffdce5e4e3a025bfdf8f935666d9", size = 1298932, upload-time = "2026-09-08T18:47:40.632Z" }, + { url = "https://files.pythonhosted.org/packages/8f/12/7c2fe2706d092f12bd7b3e8565e1ca5d0c24b853751f2f970768086dbdeb/hypothesis-6.168.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f099b1c8fc49ec2d9d7944e661addb97d7c38e818fb8d1f78073c43895a87f6", size = 1335196, upload-time = "2026-09-08T18:47:57.775Z" }, + { url = "https://files.pythonhosted.org/packages/20/35/59f7ca2414ca39408d13f66a344affe0ffc64748dc01d8a1ca910009cdcb/hypothesis-6.168.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:93413d1b0af50a7b165d66278c529174bf2fd1773c78027735dc0b50d1d3fd27", size = 624102, upload-time = "2026-09-08T18:47:38.869Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1e/dcd9335ace916ffea40f2cb04ba4122094c2f7b928f3a73fc7d452ce5b71/hypothesis-6.168.0-cp314-cp314-win_amd64.whl", hash = "sha256:db2751c27bffc8491a96d72969649089d5400115e4b7c49bf7167ebbdcc84193", size = 681871, upload-time = "2026-09-08T18:47:59.697Z" }, + { url = "https://files.pythonhosted.org/packages/de/d0/bc50b0b91e40744b7caa56b8add85cef432f85b4d00108409e8eb17af830/hypothesis-6.168.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:cd0c1dcf308e919c8ae708054d0ad61921ae87634a9aea574a9851da584cebc1", size = 791695, upload-time = "2026-09-08T18:47:06.306Z" }, + { url = "https://files.pythonhosted.org/packages/4b/53/fc7537d50ff008dc5ea8598764935f93dd07bcedaf23ee4e635bdf7055f4/hypothesis-6.168.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d0bdb77f976740b8cd5ec697327ea343d02d052b9916d213b5d4c65d823415cd", size = 783239, upload-time = "2026-09-08T18:47:47.43Z" }, + { url = "https://files.pythonhosted.org/packages/71/2a/c7aac2efc06713f704d7e354755aff4a11608b9fe93d973ead374f3b81a3/hypothesis-6.168.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f7486bed33225d02f6aa78a4c4ba2b6f84992a82571cdda1bf08dce41d13507", size = 1121412, upload-time = "2026-09-08T18:47:07.927Z" }, + { url = "https://files.pythonhosted.org/packages/60/e2/668ab29e5096af682b17b8491f5427d7c5f17c1b991daa5577bde80c29ca/hypothesis-6.168.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ba3838c4a92e0b9730d1ed7e67e4950c152ad79d0a0c7594065262db84c55c4", size = 1167570, upload-time = "2026-09-08T18:47:17.94Z" }, + { url = "https://files.pythonhosted.org/packages/3a/17/c64635e4c988b5fa3d3b8be322e19e2fe4c731fa0ca074ca852aa70debea/hypothesis-6.168.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:891b2d281ede45130e7fa0a22fd65336cc77ef2f780ec3792e8de6fc274a02c8", size = 1297118, upload-time = "2026-09-08T18:48:13.407Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c5/c7a0d9a06bf5c3279386dd53161081a57b98c6faf60fbbf64d046315e9e6/hypothesis-6.168.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e86820053afad84677f301c0b892a226be1df49790800a65668ae7cc8a1ac571", size = 1334068, upload-time = "2026-09-08T18:48:15.462Z" }, + { url = "https://files.pythonhosted.org/packages/32/99/11a393a20a867e9b978308323d45022e96f5cd2edf391e4d9a65fb4e2cf7/hypothesis-6.168.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a4956f41ab1ec6e6ef9262a35970e9f3e2caaaa1cdafe0d413156c6934dd99d8", size = 681795, upload-time = "2026-09-08T18:47:14.415Z" }, + { url = "https://files.pythonhosted.org/packages/16/f7/5adae1bf1d4877aca2e8c8e007e57077237e9ac3765e430ffda490b17e19/hypothesis-6.168.0-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:754016594fe78cef91790e0922f60d183c52f531255fbfa30dac495b813e2128", size = 791056, upload-time = "2026-09-08T18:48:27.398Z" }, + { url = "https://files.pythonhosted.org/packages/f2/93/b1b2770b87591db5cf564b9aa0265cf21e9207d28645e0abdeb8df63225b/hypothesis-6.168.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:6f0dd437ec01140676192422b61f2f833b3ce6a3213da9b7e196ad6b3777e795", size = 782980, upload-time = "2026-09-08T18:48:34.087Z" }, + { url = "https://files.pythonhosted.org/packages/de/bd/673171c1d2423379a7d4a0f9f009cca735a422d0c4a4ac6422d1d1736cab/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f77af7721ff35a58fa8797decd14c932c350a2548686c6e9b844db710a3a2441", size = 1120964, upload-time = "2026-09-08T18:46:59.92Z" }, + { url = "https://files.pythonhosted.org/packages/7d/00/33a9bd941b22a4fd8a8c805b1563e0db17efc020422f5bd15bdd0fdf258f/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a0d28418c104d7268fdebcc09bc49f7b6569b5eb942430c6859f53ec8d4edf63", size = 1143869, upload-time = "2026-09-08T18:46:49.875Z" }, + { url = "https://files.pythonhosted.org/packages/0f/c1/963460976f41721eff8f67f30d31059ea407cc1b41c737c8859aabf37197/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:812a84c4cc7f7ae4fcb39a5647cc2698e6c18254f8423126425578f1dcdac782", size = 1146453, upload-time = "2026-09-08T18:46:25.757Z" }, + { url = "https://files.pythonhosted.org/packages/ce/53/09db238098ad66f4e6d2fe883f26c270c2595b90e21c8f969d4cb21cad7a/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6de30e559eb151de14a5f74bceb4d97792a9315ada2a1816b5da825cd7d28edc", size = 1166918, upload-time = "2026-09-08T18:48:23.436Z" }, + { url = "https://files.pythonhosted.org/packages/b9/31/e1b7b452c8a6166e445ba2ad80a864f6a9eee0fe4c8cecdb9af5c1ee0aa5/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:9018b20acdb061b2ef4b2fa7f558ca5db97ffea316e0a528bc003a24b2ac996e", size = 1126637, upload-time = "2026-09-08T18:47:50.878Z" }, + { url = "https://files.pythonhosted.org/packages/97/2b/4eceed248afb46fb6b2df21cf2239362de5b25d295c2dc67a82ec8657d5e/hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bc935a5d5f86fd8f5af951b8fbe00307f6f7c596f82a9a27c17d974f6ab0a26c", size = 1155682, upload-time = "2026-09-08T18:47:30.234Z" }, + { url = "https://files.pythonhosted.org/packages/b5/3e/f3414cda4f325004d5774485e8983b7d1b99e4b91013f013dd088fd778cf/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:45fcfa05f746e253350f55f216bcef59754f5f2b85745f1fc2bb8ba81dd517a9", size = 1296464, upload-time = "2026-09-08T18:46:34.833Z" }, + { url = "https://files.pythonhosted.org/packages/aa/40/ca79cf96545e1f172b36b8df56bfeb02b61f351f283026f9a57c4631e368/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f89d8e998d3c936ffbbd1c3686c96f0378f6558aecc5967a3035a857f2bab0ad", size = 1421853, upload-time = "2026-09-08T18:47:23.083Z" }, + { url = "https://files.pythonhosted.org/packages/77/bc/657d5386740c1f4ac518ff05e4c5b132f1e6df2e7c865ba8fda4279adfa5/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:d0620fa320fa66649e6bfd71e94f3f86115fffebb7e3c6dcece19d1aaff8e07f", size = 1278216, upload-time = "2026-09-08T18:46:30.871Z" }, + { url = "https://files.pythonhosted.org/packages/51/54/2328cdb70489a36634534478d9b238594a269d8bec4630ea0e6897048347/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:4085b61e25d3dcc6c9151d4115269870aee8cdb921611ee5c989b2786449be09", size = 1297593, upload-time = "2026-09-08T18:46:41.395Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ca/d803fa57e3ff7f460f6e262d2b74822cf143343d378501fd3da601b12040/hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:b5449a64eb37d9a4aa6ac9cd2ab0fd1a24145adf421ef1536884f73f39824887", size = 1333785, upload-time = "2026-09-08T18:48:25.434Z" }, + { url = "https://files.pythonhosted.org/packages/86/8f/b9799ae6ba6074f151db2c63f9f6f12d844512821ffaba1a3672d7e59f07/hypothesis-6.168.0-cp315-abi3.abi3t-win32.whl", hash = "sha256:91e3de666a6c4f7543000d1710e25055d63ef3032c98bd2ab338b3087bdaa780", size = 675173, upload-time = "2026-09-08T18:47:52.601Z" }, + { url = "https://files.pythonhosted.org/packages/61/54/14c3e277b451ff24128ecc2673cac59dd7e535bce1a433c466912fd682e1/hypothesis-6.168.0-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:9a2079cd09919956dd388f1a1f8ea5a79f2b2437650fbeda31d8661217ffefef", size = 681491, upload-time = "2026-09-08T18:46:51.437Z" }, + { url = "https://files.pythonhosted.org/packages/99/f3/827e4a48ffee7e40244b0bf064ba47c2171e053ab7edf1cf770105e23401/hypothesis-6.168.0-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:085c9aa246487c56a40ca89003d285cbffdbb5be4097ba6d0139f9c21003c04a", size = 679197, upload-time = "2026-09-08T18:47:32.112Z" }, ] [[package]] @@ -1973,75 +1973,75 @@ msgpack = [ [[package]] name = "numpy" -version = "2.5.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/80/db0b4559e57ec36362bedbb05530a87fafbcb6067708c946967a41d449e7/numpy-2.5.2.tar.gz", hash = "sha256:d482d171c406ae88c5b19cad3b6a1c4c5209f886ab74bc44c2c865c23f52d860", size = 20773161, upload-time = "2026-08-09T13:48:27.962Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/72/dccb0aaf40972777283303919f613964227266d0c13adebb79ac124f1c3e/numpy-2.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14e373cfc6387177e8409dac3c7159be8eb05cd77096cd7c950268b86f62831c", size = 16891693, upload-time = "2026-08-09T13:44:51.702Z" }, - { url = "https://files.pythonhosted.org/packages/60/2e/b5aee50a1f74ac815cf8331812cb8251e29024025de462e0c047641c614c/numpy-2.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbd96c833ecc8cc069ce518078fc8c60cb9cbfb0fea5b7a803ad65035596d03", size = 11903109, upload-time = "2026-08-09T13:44:55.501Z" }, - { url = "https://files.pythonhosted.org/packages/f3/f4/29e78102a80601cf034d4e9767022cffeca2c3b4c926e1754572ca95593d/numpy-2.5.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:6e8172ddfcf5cf74b811d372b570b83c60bd2de87a6fbfbebdadb4a9bd9c6cbb", size = 5350202, upload-time = "2026-08-09T13:44:58.401Z" }, - { url = "https://files.pythonhosted.org/packages/11/4b/dcd3b7eadaf4035d2c7a4289d232523a6964f602598ef7674e4bd7291f93/numpy-2.5.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:65f188481f1669e26f62b701e8205d19e460fa4a9b52a1414ba382330e4a3414", size = 6687736, upload-time = "2026-08-09T13:45:00.813Z" }, - { url = "https://files.pythonhosted.org/packages/e5/21/4947e0e9d6c9fc2e2ff15b8949049ee44f63adb9cacc729ab8793f97e712/numpy-2.5.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ee9c4eeb8454b3660a8b53493563c3e121c2fc94fbd72b848ef814ed7b676a9", size = 15612696, upload-time = "2026-08-09T13:45:04.151Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5f/62d28cf019460c7f1394105b4d49d9911a9c444cb77ab0bd95a204c5a6de/numpy-2.5.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3cdec01fa790a186d430433fdd4d4ffb70eed6f0eeb4bf05c8dbe2dce0a9bcb8", size = 16722264, upload-time = "2026-08-09T13:45:07.714Z" }, - { url = "https://files.pythonhosted.org/packages/14/25/3f0be4c1b9fdf5dd5e708a6806978564d7c46a055c000496309ff2a2f8af/numpy-2.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7999d4ddb0c4025018373fd787510d46e04c769467af22869707b3c1cfd459ab", size = 16974396, upload-time = "2026-08-09T13:45:11.316Z" }, - { url = "https://files.pythonhosted.org/packages/22/72/6262cbdeeb45da9d971e40715f579d791603ba8ec0b5e2db1ac55454421d/numpy-2.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c1f017dc0875c9209d219f97feceb7d54c2661bb243deb4114478e1295808af7", size = 18476044, upload-time = "2026-08-09T13:45:14.869Z" }, - { url = "https://files.pythonhosted.org/packages/36/33/29208b8b075bde62d26a81d14b358c42b0f69b6cabd98d4ff97f37f22b05/numpy-2.5.2-cp312-cp312-win32.whl", hash = "sha256:d6a48072864e3324e194a8fbb3c657bcc5b5c869dbc64c9537b1d5c862572c0a", size = 6072817, upload-time = "2026-08-09T13:45:17.867Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b9/87fea2769fe1c47c1b5b01d8310772c9d1a85d485de7cf386ef7a3332b02/numpy-2.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:28ac63476ec7651484215ee7fa15a1f78b57c14621f01e392afe17b9a1390ce4", size = 12464674, upload-time = "2026-08-09T13:45:20.734Z" }, - { url = "https://files.pythonhosted.org/packages/14/52/032b97e00461ab0809bbe4c588b035620e5a14b8cdee47ecddefc7b17d33/numpy-2.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:27650bb0e7140fa3d37b9923b4803645e0b125d190f326eecfd3f4dad8e8ade1", size = 10397131, upload-time = "2026-08-09T13:45:23.73Z" }, - { url = "https://files.pythonhosted.org/packages/f5/d2/6b24738a0ef4557d189b150046cd07823c50e4273e8aebd651222e24306f/numpy-2.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8e4cb9a754c8a0c62eaa88273a5fba3391f4a610d1dee893c0755da31c083f15", size = 16886595, upload-time = "2026-08-09T13:45:27.323Z" }, - { url = "https://files.pythonhosted.org/packages/65/60/f2d208d366f263f39c6e69ed309290717aab41078b6d04c9be2a84fa2a07/numpy-2.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:52c808f96484f5571a5cc863775ce50247c17dfb3b0361f8ed6b4b0456f80080", size = 11896845, upload-time = "2026-08-09T13:45:31.638Z" }, - { url = "https://files.pythonhosted.org/packages/3c/79/81e0bf24f4d020a2b1d5cd297a9f60c3f24eeb116f9bba5870443f7b6a4a/numpy-2.5.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:29d81e97f668489cba8ebfd796b9bdd453525d35dd9e162e2daec94bf3fc7740", size = 5343880, upload-time = "2026-08-09T13:45:34.373Z" }, - { url = "https://files.pythonhosted.org/packages/ba/cc/e3141cf06d1a8a2c7e107543fe1269c1d1af760d4d683c0794a4ee1127c2/numpy-2.5.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:afb3f0632d6b2e3ba04dbce8d1e48d321b369138b73830b5ca371a0e8d479d56", size = 6682264, upload-time = "2026-08-09T13:45:36.7Z" }, - { url = "https://files.pythonhosted.org/packages/29/f1/2a64a307d92c5d98f5255a4014eb43bb6103ee477087b61ecae44a3aa9b9/numpy-2.5.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0aadf13b60048d501e05fa699efaf7734e2494f3498a4c2a5521d822640324f3", size = 15609566, upload-time = "2026-08-09T13:45:39.518Z" }, - { url = "https://files.pythonhosted.org/packages/7b/44/59a1eb68e773c4098d107ef34a0dbdeca501d72ffcfbff9a7707343921ce/numpy-2.5.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29b86ff8a6cc556b47ec6b64b194815cc80e6bf5eedcc6cddfd65318cb0b4eee", size = 16709995, upload-time = "2026-08-09T13:45:43.661Z" }, - { url = "https://files.pythonhosted.org/packages/8a/4c/3e54d4ddbc359a1295f8b633e8106bcd4d7d4a206e82df051bdfb3058755/numpy-2.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6950c4b7dd562453090548ba7f5da7e59f57f85663f15d5dcc60e249192f7e59", size = 16972511, upload-time = "2026-08-09T13:45:47.094Z" }, - { url = "https://files.pythonhosted.org/packages/f2/9f/02e371638ebf19b66d46231e4be52999e87f32d1961b113bc45656608b22/numpy-2.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b9727f472d2f3888053b8a75ab0cb94745a9de224bb5846dbadc0092101bc71d", size = 18465609, upload-time = "2026-08-09T13:45:50.808Z" }, - { url = "https://files.pythonhosted.org/packages/eb/ae/ad6645abc7a3510fe48e8ea1ab4598166f500057ef4ebf38bfad4f1577de/numpy-2.5.2-cp313-cp313-win32.whl", hash = "sha256:4f9744f9fbdcea0bc552e8f19e1f141f811a3f9bc2be2cc6e86d982cab23e3f4", size = 6070204, upload-time = "2026-08-09T13:45:54.111Z" }, - { url = "https://files.pythonhosted.org/packages/15/20/f3489f86d81ea460b2bcdceaed094142ca6579f6be0ec527b781d39afe68/numpy-2.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:85aaccb24182c25df891ad0ec333585967e115269d5f1b17f2c9ae005bc96657", size = 12460532, upload-time = "2026-08-09T13:45:57.167Z" }, - { url = "https://files.pythonhosted.org/packages/d5/21/35b31dde1b283b79de828b80f876afd8c94e28fe1e9c375f89e261cc4c0d/numpy-2.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:bd68ece1553d2023c09a4226d9e41c586ad2d20594d1a456186c33513d2cb3f2", size = 10396725, upload-time = "2026-08-09T13:46:00.478Z" }, - { url = "https://files.pythonhosted.org/packages/ac/f8/c3b222bf075b50afd8e949a07a15c4b312a4a84bd8102a332bcd953cbbb4/numpy-2.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d787cf769c3baeb5f6235e778edb52c08dfa923789b5958f28e6450f96107cb1", size = 16885180, upload-time = "2026-08-09T13:46:03.939Z" }, - { url = "https://files.pythonhosted.org/packages/17/e1/2c1d4b1987795a92b5bbf7c24fe249ab96aa2573ab0d7604802c189d7b86/numpy-2.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:24b9dc2e3d84aa58523798805194e23e736f3f6ce2d1a5b92583ae734e6dbda8", size = 11907878, upload-time = "2026-08-09T13:46:07.045Z" }, - { url = "https://files.pythonhosted.org/packages/b9/ee/d08226fc858044355983a6e5b94f08ff6f3969e0a2b160a4a89f0ddb3445/numpy-2.5.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:9e9413326d726c2545bfa65d2c0876871e8d8386e77f992c1d426e180bbd4323", size = 5354922, upload-time = "2026-08-09T13:46:10.04Z" }, - { url = "https://files.pythonhosted.org/packages/94/f0/6d3d933056440ebbc5e6bad92065fc6c26a48a84a36b1208580e94eea76c/numpy-2.5.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:60e902ac295855348a5ca2ea4c89108989a9f5fddfad3dfc0a8f36b10358567e", size = 6679168, upload-time = "2026-08-09T13:46:12.275Z" }, - { url = "https://files.pythonhosted.org/packages/c4/3b/ecd49dd90033cceb2704d88ca905d4d7d89b0e8c739608754ffd325fa820/numpy-2.5.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e500dc868e9313530ce12ba470fe50ff3afe3d62993ed6eff652dacd555b65", size = 15624501, upload-time = "2026-08-09T13:46:15.322Z" }, - { url = "https://files.pythonhosted.org/packages/c7/99/461bd36dbdfac6c1c53efa370bd55a83227542d0d118f1677dbf1a3dacd5/numpy-2.5.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318b9a4c845dbea06708a29c84ee429cc3065048db34cdb799047643492050ee", size = 16713701, upload-time = "2026-08-09T13:46:18.949Z" }, - { url = "https://files.pythonhosted.org/packages/f9/9c/2b251df9e8a5d647b62b0cbc1b90a91850c1cf4859ecb532fd0b4eacff6c/numpy-2.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:34c319e2963be042673fb46570501b2f06c41924e17e3563d58646b4380dfb68", size = 16986065, upload-time = "2026-08-09T13:46:23.006Z" }, - { url = "https://files.pythonhosted.org/packages/8f/25/20de43f53ff1390534a124475055a19f01fe10c920a0fd11b8e18d6d6052/numpy-2.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f06571a052127dc1b4e8b83029b4d1b20daa2b64a31cdd181fc6bc774e9000eb", size = 18470031, upload-time = "2026-08-09T13:46:27.102Z" }, - { url = "https://files.pythonhosted.org/packages/56/5e/0c577ca308d6da5eb79b546ba10bbe5b60148192194e2da060913b1de4f1/numpy-2.5.2-cp314-cp314-win32.whl", hash = "sha256:2cc779226e476d1e1f08c74068c419e60f41a9e0e069c92f6671d31d5c985e98", size = 6121028, upload-time = "2026-08-09T13:46:30.046Z" }, - { url = "https://files.pythonhosted.org/packages/15/5c/7bcbd5b11f94199073320410cddcbb80cee62415bfeb540874b265c2d922/numpy-2.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:7587f53dfbd5edc0f7b87c6217b4c6d2d1f2ef9c3da70bc1315e7db5f8d7ec9d", size = 12597627, upload-time = "2026-08-09T13:46:32.886Z" }, - { url = "https://files.pythonhosted.org/packages/87/bc/4d0b06fba0da90ccc75af62823cb9dcedb6c9ea0cffa058cb2c9ee773a77/numpy-2.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:3e4c367352d3747784248a227fbec218e193b56f7e6692e3b64fc805478ecfdf", size = 10680414, upload-time = "2026-08-09T13:46:36.036Z" }, - { url = "https://files.pythonhosted.org/packages/cd/17/f429aac9dc08833a0d0f188eba38c532a751b1a1f2ca6018a37b455cb321/numpy-2.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b879fb674276e331513fb136b78dbc6bd3c848309e0d841cfd63be3896c4cfc1", size = 12026967, upload-time = "2026-08-09T13:46:39.084Z" }, - { url = "https://files.pythonhosted.org/packages/ca/9f/d0849de96a2a4ceaa16662f18ee13eaa9c0aa418269fdc8c4857c56b11da/numpy-2.5.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:fd0d703772bba096843785bd38371e31bb4a0c1151497ad5739d182114a73f7f", size = 5473874, upload-time = "2026-08-09T13:46:42.075Z" }, - { url = "https://files.pythonhosted.org/packages/89/3c/8df216d4a4a5422a3de045301cf7df8ea47286d76f5cb7160b0128ac26b7/numpy-2.5.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:3a2f061cebd9e3d23bdcfaaded5e2293a4c6a5b60fa42df85d410a725ce621bf", size = 6789276, upload-time = "2026-08-09T13:46:44.387Z" }, - { url = "https://files.pythonhosted.org/packages/e6/3a/20d7e9891c4ddfadd6ff8d95bf4b29f353d8e1770553de2099880551dfb9/numpy-2.5.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6df895598c0edcb41030126c89e0f353b07d93238116143b7405e937359736c4", size = 15659154, upload-time = "2026-08-09T13:46:47.538Z" }, - { url = "https://files.pythonhosted.org/packages/aa/d6/f3aa3d2688bf501b858835c6bd087ae9b51a56ae6fca8e2b0990abd177af/numpy-2.5.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1ab3d4a901f844ea836c3e80bf463c6a27d7f3c14e8e292fcf28d348b25b9bce", size = 16748909, upload-time = "2026-08-09T13:46:51.442Z" }, - { url = "https://files.pythonhosted.org/packages/7d/8f/1c5cae8d2baf86ab802ae97a00be55bc7e21ebc11b12bbc33376c5f05342/numpy-2.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cebc2d6dbb605a7703d59751dea4bd6b0ab127a5a4338a6f432df1936fef8b26", size = 17027685, upload-time = "2026-08-09T13:46:55.095Z" }, - { url = "https://files.pythonhosted.org/packages/5c/27/71d3467404aedc1c24ce79610f91b52b0b0f466c43a701aa56fc75c145ab/numpy-2.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:eaca7ff36f0f52e2111ec71f169d8fd3e889e7ddc0d2592e0d703fd8d3ce8fac", size = 18501181, upload-time = "2026-08-09T13:46:59.09Z" }, - { url = "https://files.pythonhosted.org/packages/14/2f/42921d27c40aea7e077f4a423ae509fd9220b028cd787bafefd8ab2b3a5f/numpy-2.5.2-cp314-cp314t-win32.whl", hash = "sha256:ddf47472af2e4280d79bac82304f5e80150211f1b9e614b760061d5fdfbb6eba", size = 6271085, upload-time = "2026-08-09T13:47:01.903Z" }, - { url = "https://files.pythonhosted.org/packages/75/e6/bad5f5d56de9b1971bac959963dda276d35c40f1854475005434bbe08692/numpy-2.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:44ef9675d908e65f9953063837c3277730f3f4437615a4cdab67b366cabaf884", size = 12787971, upload-time = "2026-08-09T13:47:04.963Z" }, - { url = "https://files.pythonhosted.org/packages/df/05/f608795cb34391acd67e38d94a3c36abd8d8576293a3a80727d7595c372c/numpy-2.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:eaa088384c46f519dacb93b7ec483a6d6b19a4a2085ae4f25ab9b1c43d387d1e", size = 10750306, upload-time = "2026-08-09T13:47:07.976Z" }, - { url = "https://files.pythonhosted.org/packages/33/c6/28de0191c5f82b7d42a0a51390ba98587048aa93a39fafb05bdbe6e8d00c/numpy-2.5.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:078f9b027b478c9379b9677babbf0f8b8f1ecfada27636d7b9a93990c638739f", size = 16885274, upload-time = "2026-08-09T13:47:11.439Z" }, - { url = "https://files.pythonhosted.org/packages/dd/d1/973ca116000d244897e468ea1aff30b589e5022e3c8744b71706fe33bd57/numpy-2.5.2-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:50a68f4bacd8a2b33d8da3d2269d0d78500f86ea582e4786dc10f5ef2c2c6842", size = 11907846, upload-time = "2026-08-09T13:47:15.128Z" }, - { url = "https://files.pythonhosted.org/packages/78/d9/8c4b3937ef204cb2fd88d389ccd0f265a2ffb11f35a01d2064cf46714bd6/numpy-2.5.2-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:e79aba74ffaf5f78a050d777c184cddf8fdffabab38acf5f3ef1fecbc17895d6", size = 5354892, upload-time = "2026-08-09T13:47:18.07Z" }, - { url = "https://files.pythonhosted.org/packages/74/9b/b6ee65ea2999fdb7023935e108e6fb776ee4082aa15f159acfa857e578c8/numpy-2.5.2-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:9a0731745a72a184490a582fb4af2533512bd071ace67785b5fdffc0ae58dce8", size = 6679309, upload-time = "2026-08-09T13:47:20.456Z" }, - { url = "https://files.pythonhosted.org/packages/43/f3/acb18d8b137a393c8e7803a8c994c9e64bde3930692a69d826993113a159/numpy-2.5.2-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4ec954036759bcee3aa484f8603bd9c14f3e776293b85578b8734c2d72777c69", size = 15625850, upload-time = "2026-08-09T13:47:24.365Z" }, - { url = "https://files.pythonhosted.org/packages/a9/bf/a8e9bb0db815a0e265b5744ebedd3af0bd5faad8604e5b50a1cd012f3c91/numpy-2.5.2-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc649493697006bc90614a5f0bbc8cb3cb1866715c474e473694968d7e6b99ab", size = 16713664, upload-time = "2026-08-09T13:47:27.965Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c3/6e913736b3dd6582344af32418b5fb9dab34282e8a8174ae1d54ceb0fc13/numpy-2.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:cf7de32f486e4ac9e2d93b810f9e9ac72a728dd46a32a0bb403222f27f653514", size = 16986749, upload-time = "2026-08-09T13:47:31.541Z" }, - { url = "https://files.pythonhosted.org/packages/80/09/7d3b23eff5c7428ef6c01e6f7052bb60d504c4d33e317b36b8959c24ad97/numpy-2.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2ffa7bacab3e2ee1b19ed31766bb60bb380b68c23f051e199c5cc598afd68710", size = 18470495, upload-time = "2026-08-09T13:47:35.364Z" }, - { url = "https://files.pythonhosted.org/packages/a5/a4/68a321d825374f6eb677ffe8ef8c6b9a328304e6fd2e39d9530822776607/numpy-2.5.2-cp315-cp315-win32.whl", hash = "sha256:6b588cc8f902d6bff201c19fd00c43ab8545671e3554d014e12e14139e5e8617", size = 6120696, upload-time = "2026-08-09T13:47:38.561Z" }, - { url = "https://files.pythonhosted.org/packages/c8/23/deafbb1700f79fae9cd1e91220f133d124cc267de1b584da3fbf6db2f6cd/numpy-2.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:07d4e89f3a9ab0a9ba24264ccdb642b3dd951b2281e8883a5481a4aa79cc31a7", size = 12597324, upload-time = "2026-08-09T13:47:41.401Z" }, - { url = "https://files.pythonhosted.org/packages/33/cd/3272ba105e3bbbdaeb11357eda31e7a6825ffe159e8171665660299a948f/numpy-2.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:a610dc7e3c52edd39c2bc2375ff9c3fd59cb3ad00e4472d36f83bc1457145788", size = 10680466, upload-time = "2026-08-09T13:47:44.873Z" }, - { url = "https://files.pythonhosted.org/packages/0e/0e/58370637b1bb70a5c9ce2b43f4b521ccb224e36ccb76a6596b17ae4b447c/numpy-2.5.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:40f4d451aed46a8046a1aae41c4e55fb3612273df9c502480135e1501576a34b", size = 16993947, upload-time = "2026-08-09T13:47:48.97Z" }, - { url = "https://files.pythonhosted.org/packages/10/93/2abcb807712b289d6d60fe4cf30532f98974a8396d885650f3ba5a13026e/numpy-2.5.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:c081cbe16ba1ab53078e5ff29013621e33c509eedab055775d956427712c236e", size = 12025331, upload-time = "2026-08-09T13:47:52.646Z" }, - { url = "https://files.pythonhosted.org/packages/8b/3a/2898e003a5fbaf87e76c039b4ee1f5eb390471b4ffe74887c1f34c4e791e/numpy-2.5.2-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:0090ccdd57ec2703e9b49d0bf554767370581c1dd0a6b2bb2b2d9def317d042a", size = 5472336, upload-time = "2026-08-09T13:47:55.403Z" }, - { url = "https://files.pythonhosted.org/packages/61/a5/23f69d07c544597b29758b31b55c27dc9d541012a2c1496189fef702aec2/numpy-2.5.2-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:6a9bb119fb8dd21ba30b3f0e555b7e2b081bd9883af21ec9c1c633d161cda3a8", size = 6788387, upload-time = "2026-08-09T13:47:58.192Z" }, - { url = "https://files.pythonhosted.org/packages/15/ea/c0dbdbcf22f43782510a3e492dd3da73c6112b69cac8929d16d127536fc4/numpy-2.5.2-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a839318485284a6fb31be4f8f2c91c8f2cb22f4543c4a8903f12b0671ffe07cc", size = 15667096, upload-time = "2026-08-09T13:48:01.562Z" }, - { url = "https://files.pythonhosted.org/packages/fc/5e/29c73c31748cdb0f7566642125ba17fd5b56780cddf891b085dab27e4466/numpy-2.5.2-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba0a474801b8dc67b66bf465548abc90e82b44d2611b5770f33008dcabffe8ec", size = 16751730, upload-time = "2026-08-09T13:48:05.706Z" }, - { url = "https://files.pythonhosted.org/packages/47/95/02501e8454796bb58dadf7a99d3181e0b464bf264e1003039572f9779fac/numpy-2.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0a4035ae1129ff8777f08bfbd44f1e5d8e9c049ce0c2dd78fc0d92c13e7251c0", size = 17038686, upload-time = "2026-08-09T13:48:09.627Z" }, - { url = "https://files.pythonhosted.org/packages/0e/b5/53a681d91b5c82687067d8ea5035e02d917b5509d6f334cb06484a954714/numpy-2.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:77843ca236b777e67f8d6b3660ea116e499612703a0ecd7093f316201eb9d8e2", size = 18507727, upload-time = "2026-08-09T13:48:13.744Z" }, - { url = "https://files.pythonhosted.org/packages/42/06/6e11443f7b64ee376c860506091103bf68f92d2cab9e8d96d4501babf07c/numpy-2.5.2-cp315-cp315t-win32.whl", hash = "sha256:7354826bc6f8f69402e9b7fe28d15fcd34feebd74f856f111585c5b0c9fb0251", size = 6269775, upload-time = "2026-08-09T13:48:17.543Z" }, - { url = "https://files.pythonhosted.org/packages/f1/18/195d6b86cd72dbbc501edfa778005fa6b87afd34c153e46028cd3a0938f4/numpy-2.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:e5651f3f87add730ee6608d915009e19c911fba0cb000c7e3ea994b7d768eb12", size = 12782559, upload-time = "2026-08-09T13:48:21.023Z" }, - { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, +version = "2.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/01/11703282db468b85f6f7b8c7f22d058de5970d5c7e60a3a8aaa313c3de36/numpy-2.5.3.tar.gz", hash = "sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563", size = 20791231, upload-time = "2026-09-06T16:27:47.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/50/8fdbb16af64895706a45f06a4068e29db732ec180f3c1375f14123359138/numpy-2.5.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9", size = 16994982, upload-time = "2026-09-06T16:24:29.244Z" }, + { url = "https://files.pythonhosted.org/packages/60/39/789131c1188c078dcb3a1692e72e1e050c68b88ffe72c9ccaac9bcd7a9cd/numpy-2.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c", size = 12009327, upload-time = "2026-09-06T16:24:32.491Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/a312e95696e5f601914dd8b6dd844692ba61670807417e24b68e337b5c70/numpy-2.5.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0", size = 5445405, upload-time = "2026-09-06T16:24:35.071Z" }, + { url = "https://files.pythonhosted.org/packages/30/d0/5623a1707ed4fe16e3909fe3cf5ee3da004ae677ad23d83bbf3adf1a6faf/numpy-2.5.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e", size = 6783213, upload-time = "2026-09-06T16:24:37.253Z" }, + { url = "https://files.pythonhosted.org/packages/f1/32/84146fc020ad3c25f805f70ab60da46fe3c540a21369754a7e4369754b6f/numpy-2.5.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58", size = 15687872, upload-time = "2026-09-06T16:24:39.751Z" }, + { url = "https://files.pythonhosted.org/packages/65/af/aa78d1a88805456e212b65461354cd943197fb9acecc4c90fd12295123a3/numpy-2.5.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3", size = 16717410, upload-time = "2026-09-06T16:24:42.745Z" }, + { url = "https://files.pythonhosted.org/packages/3b/24/faa79d865e69a97ba17473b23a1b74094b2259c03e820c70297293b9ea49/numpy-2.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff", size = 17040975, upload-time = "2026-09-06T16:24:45.961Z" }, + { url = "https://files.pythonhosted.org/packages/62/4a/8877e629445a7176297dffcaf9c485faa96a95d81728a62521ad55bd4c0f/numpy-2.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034", size = 18476479, upload-time = "2026-09-06T16:24:49.35Z" }, + { url = "https://files.pythonhosted.org/packages/c8/db/35e1c2d38b04cbd5b731f9d71495e055e813197669d22b612f11748d2ff9/numpy-2.5.3-cp312-cp312-win32.whl", hash = "sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4", size = 6133378, upload-time = "2026-09-06T16:24:51.915Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a1/accf6d4f0c80c5d9ba9735d6b1550e444180599f34dec69ca01360f717ad/numpy-2.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def", size = 12567828, upload-time = "2026-09-06T16:24:54.255Z" }, + { url = "https://files.pythonhosted.org/packages/22/43/1764aff32e4652526ae2f71fa8b3efd8d25c8a3d6926914454e47138ed1e/numpy-2.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034", size = 10485432, upload-time = "2026-09-06T16:24:57.278Z" }, + { url = "https://files.pythonhosted.org/packages/79/e5/8fb89cd46d14e35699d13bf943a5f5f441ecee8667120a1f6105ab89e349/numpy-2.5.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034", size = 16991061, upload-time = "2026-09-06T16:25:00.411Z" }, + { url = "https://files.pythonhosted.org/packages/2f/06/9dc9e48b5e5e941c8b10350c5ff2d721da42a20517d911d15544246775ff/numpy-2.5.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e", size = 12003676, upload-time = "2026-09-06T16:25:03.475Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2a/98282aa5b8f58b1157d440bb6282eed47e3632a5de53a714fbab17e659fe/numpy-2.5.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09", size = 5439695, upload-time = "2026-09-06T16:25:05.978Z" }, + { url = "https://files.pythonhosted.org/packages/a1/f9/b6533d777be9d6ffd29dc1be0867e563e6e8cc9a220ff1b716adc317f060/numpy-2.5.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958", size = 6779395, upload-time = "2026-09-06T16:25:08.599Z" }, + { url = "https://files.pythonhosted.org/packages/73/85/735720d04ec197c5dcfacdfc9922667c7f1f5f496a279b7ba4d7c74c4cc7/numpy-2.5.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b", size = 15681750, upload-time = "2026-09-06T16:25:11.173Z" }, + { url = "https://files.pythonhosted.org/packages/3a/1b/3b16a9bc514a440a7a0883684111dcb1ef1aee960af2ca95da8fc775f124/numpy-2.5.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321", size = 16708577, upload-time = "2026-09-06T16:25:14.171Z" }, + { url = "https://files.pythonhosted.org/packages/69/c4/386f397831b07328b639c96c5b62719346cf4baf07c68d927239752b1534/numpy-2.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231", size = 17042047, upload-time = "2026-09-06T16:25:17.582Z" }, + { url = "https://files.pythonhosted.org/packages/5f/3e/a700ecbf36e85ae8328fd3b0e12eeddc22ed6358a64cb2bd913e0d195d65/numpy-2.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0", size = 18465724, upload-time = "2026-09-06T16:25:20.949Z" }, + { url = "https://files.pythonhosted.org/packages/41/ee/38e785e88a4045f6ad1d1f2808dcdfafdca48c760260c0587bf171e29fc9/numpy-2.5.3-cp313-cp313-win32.whl", hash = "sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f", size = 6129003, upload-time = "2026-09-06T16:25:23.664Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ec/100f2b1794ede74a9b3d7ec6b9736927f56713414c1dfe19ab6c383494bf/numpy-2.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec", size = 12560965, upload-time = "2026-09-06T16:25:26.602Z" }, + { url = "https://files.pythonhosted.org/packages/80/b1/7dc825ca94c12acebbce4c37caa5e198695eb31424bc579679f32b1bb49d/numpy-2.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204", size = 10482343, upload-time = "2026-09-06T16:25:29.772Z" }, + { url = "https://files.pythonhosted.org/packages/70/78/cf416f15dc29375a229d9dfebf8db6e313f291580b39fa1a568b6052bb07/numpy-2.5.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a", size = 16998686, upload-time = "2026-09-06T16:25:33.171Z" }, + { url = "https://files.pythonhosted.org/packages/9e/59/abcc2d8def4fd60eec7d87f92d27c13448ffd9ab14339bcc63a0d7a2fdea/numpy-2.5.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c", size = 12013862, upload-time = "2026-09-06T16:25:36.748Z" }, + { url = "https://files.pythonhosted.org/packages/94/75/4640d2d6e4b64a049e48425a82728a41ef4adb61332d2cba68055774878b/numpy-2.5.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07", size = 5449793, upload-time = "2026-09-06T16:25:39.476Z" }, + { url = "https://files.pythonhosted.org/packages/96/cd/625b57ae33d4ca560f32cc0b47b4a5922146d9beb998ddf773900d440a73/numpy-2.5.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0", size = 6785176, upload-time = "2026-09-06T16:25:42.069Z" }, + { url = "https://files.pythonhosted.org/packages/9c/72/12918652e7912ef9751e8694c88820fcd1908e0618cb23f5f3caa6004b7b/numpy-2.5.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc", size = 15703377, upload-time = "2026-09-06T16:25:45.135Z" }, + { url = "https://files.pythonhosted.org/packages/45/8f/9beacf79ca7c650688ad0baa80931adb988fe6e6e5d5903c23cc3dbd70eb/numpy-2.5.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b", size = 16711928, upload-time = "2026-09-06T16:25:48.461Z" }, + { url = "https://files.pythonhosted.org/packages/09/8d/41d0a56e1ac4c87495c897a211b1368691b7237aadabec8b3b8f3a74d48f/numpy-2.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104", size = 17059507, upload-time = "2026-09-06T16:25:51.873Z" }, + { url = "https://files.pythonhosted.org/packages/08/1e/0dfbc5cc251d54e2af790f254d24ec38637fa97ec7d5d11de7ffed787098/numpy-2.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694", size = 18471002, upload-time = "2026-09-06T16:25:55.233Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2c/dfa40f6991f8185c8c30ffd023dfcbb11888e823cfab9557b920f3bb7bed/numpy-2.5.3-cp314-cp314-win32.whl", hash = "sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e", size = 6180485, upload-time = "2026-09-06T16:25:58.157Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/d2c08231e4fde7e415501fd02c715d96e98599b2d8384445933944152984/numpy-2.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10", size = 12698179, upload-time = "2026-09-06T16:26:00.789Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e9/dcdcc9b95cf5f49815055573aee1b11cfbf5299f38a180e437ded050810f/numpy-2.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6", size = 10769383, upload-time = "2026-09-06T16:26:04.011Z" }, + { url = "https://files.pythonhosted.org/packages/49/c4/af8bc08a7ef4e1529a7c0cf24969accce316b783999802089a581ec99272/numpy-2.5.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297", size = 12132668, upload-time = "2026-09-06T16:26:07.138Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ae/0f15eb56d4ec5e13c1f7ff04ff407f997d1acbadb45d3e1f2e2645a8f43c/numpy-2.5.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841", size = 5568580, upload-time = "2026-09-06T16:26:09.828Z" }, + { url = "https://files.pythonhosted.org/packages/23/fb/c72a8f25d4b6e96c354e7ab45ace3b27dc11e5d6a13b6c7d0cd6b08bf112/numpy-2.5.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891", size = 6882634, upload-time = "2026-09-06T16:26:12.524Z" }, + { url = "https://files.pythonhosted.org/packages/07/a9/968c90ed2ab15060c338e8137f1215b5a60756ae07328e0a60d1c6734df4/numpy-2.5.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6", size = 15748923, upload-time = "2026-09-06T16:26:15.092Z" }, + { url = "https://files.pythonhosted.org/packages/59/08/9df04103947b95e3b6b1f2ed1a70521f325647a31b82da6a2aae3a485508/numpy-2.5.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211", size = 16746748, upload-time = "2026-09-06T16:26:18.43Z" }, + { url = "https://files.pythonhosted.org/packages/41/a0/14c8d5fe5b53a334aabb653deb391c0fef49558f491880ea300ed6785224/numpy-2.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653", size = 17111561, upload-time = "2026-09-06T16:26:22.113Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a6/d7e96e42f01522e154c32489640f16dfc4f6181d165d05fc3bec8c2c4999/numpy-2.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33", size = 18513945, upload-time = "2026-09-06T16:26:25.401Z" }, + { url = "https://files.pythonhosted.org/packages/25/39/3453afb7119d0449ef11c886874120ff180e2c337760e0e2d88f70f1a945/numpy-2.5.3-cp314-cp314t-win32.whl", hash = "sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c", size = 6335421, upload-time = "2026-09-06T16:26:28.175Z" }, + { url = "https://files.pythonhosted.org/packages/99/01/22815d2b19a1a746b1d45205cffebb3fe511a18acb75fba6c88491fc9894/numpy-2.5.3-cp314-cp314t-win_amd64.whl", hash = "sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435", size = 12896420, upload-time = "2026-09-06T16:26:31.265Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ee/a7cbba67eeaff038dc29ca8b98a88396c8b0cc9c89d4924f4a27a5c9150b/numpy-2.5.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a", size = 10857177, upload-time = "2026-09-06T16:26:34.167Z" }, + { url = "https://files.pythonhosted.org/packages/45/56/78194492883ff5eec90423fe56a3a44b154da047d88a6307f629713c584f/numpy-2.5.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058", size = 16996531, upload-time = "2026-09-06T16:26:37.287Z" }, + { url = "https://files.pythonhosted.org/packages/11/39/dd55c0af90bbab564b09ae3b0aa60ec5c02b900fa4f1ba23440525c8b32d/numpy-2.5.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be", size = 12012569, upload-time = "2026-09-06T16:26:40.707Z" }, + { url = "https://files.pythonhosted.org/packages/b6/51/04f67d32e4862b281b1cb84ceeaed3421189a84fb6fb51a391cd6d5009f7/numpy-2.5.3-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5", size = 5448498, upload-time = "2026-09-06T16:26:43.435Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c9/25b4dc0dd1344ec26c7319e84fd4e9809d2b5628f4e12decd618036e5178/numpy-2.5.3-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90", size = 6783026, upload-time = "2026-09-06T16:26:46.374Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c7/29285be1e5232a6e7ee3268a33c85843f5a8ee93350c6465cddd66ebbf76/numpy-2.5.3-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159", size = 15697322, upload-time = "2026-09-06T16:26:49.415Z" }, + { url = "https://files.pythonhosted.org/packages/55/49/bbad5335fb4996a16881f853ff3e0ba582f01720e55c89b1c06b8fc42a90/numpy-2.5.3-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab", size = 16708995, upload-time = "2026-09-06T16:26:53.127Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e9/1df35483760b04a65ea44669f89dc64f30e5aca098b48ceb8b1310b0e0fe/numpy-2.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2", size = 17052508, upload-time = "2026-09-06T16:26:56.464Z" }, + { url = "https://files.pythonhosted.org/packages/b8/99/66e54da8265cc8be8a7382bf96edce17aaa2837d6f484432025932a3caa5/numpy-2.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7", size = 18468224, upload-time = "2026-09-06T16:26:59.966Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/b5e90a91c115168d793dfd2ad9c69c438c2fe7a13a437e770bc5b078e732/numpy-2.5.3-cp315-cp315-win32.whl", hash = "sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034", size = 6179919, upload-time = "2026-09-06T16:27:03.166Z" }, + { url = "https://files.pythonhosted.org/packages/37/ea/780748fd3985109075514ef8fc64cd25f943e40dde13a6d59141eb268fc8/numpy-2.5.3-cp315-cp315-win_amd64.whl", hash = "sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae", size = 12697656, upload-time = "2026-09-06T16:27:06.153Z" }, + { url = "https://files.pythonhosted.org/packages/b3/16/407be69a2a87c8cab64d95975a8977a426a29e138f07e276ec258f0fe4e5/numpy-2.5.3-cp315-cp315-win_arm64.whl", hash = "sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5", size = 10767601, upload-time = "2026-09-06T16:27:09.297Z" }, + { url = "https://files.pythonhosted.org/packages/44/bf/a97ffb01e41d50a32a9177aef942a4d0e389a3daf451d04e5f38ef6afb87/numpy-2.5.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a", size = 17090092, upload-time = "2026-09-06T16:27:12.907Z" }, + { url = "https://files.pythonhosted.org/packages/d1/24/136c02f2c2af9a067a84d0c3aa10c99012c0476fa5066732fa4a4202557d/numpy-2.5.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832", size = 12129429, upload-time = "2026-09-06T16:27:16.089Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6c/b47582d6597789bf946d5efbeb6b9e56fd8bcbd5efc6fbf51dbe1ea31eb3/numpy-2.5.3-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0", size = 5565452, upload-time = "2026-09-06T16:27:19.868Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/ef3cc6da73774202d4deae16bb321fd8298a4e0561e3539f8c4be237d916/numpy-2.5.3-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997", size = 6876736, upload-time = "2026-09-06T16:27:22.232Z" }, + { url = "https://files.pythonhosted.org/packages/9e/24/e3813329498596cb842703dcacac1741612ed9fb9c4e6a3e0c7e2ebbc597/numpy-2.5.3-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85", size = 15745777, upload-time = "2026-09-06T16:27:25.181Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9e/4e7a07fd0776dc2210cdacf2010be8665194d094defc10c419d7dea794cc/numpy-2.5.3-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4", size = 16746949, upload-time = "2026-09-06T16:27:28.576Z" }, + { url = "https://files.pythonhosted.org/packages/91/db/01674c0e20335057813a00c2ebd546ed25bff9ed7914f9bced00f8c55d94/numpy-2.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c", size = 17108994, upload-time = "2026-09-06T16:27:31.946Z" }, + { url = "https://files.pythonhosted.org/packages/45/7a/584c5e71f8d378e57cac0b033891ed65c683ef90573ba4854e8c28203db0/numpy-2.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0", size = 18512266, upload-time = "2026-09-06T16:27:35.196Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d2/4e1014173aa3c55e6a756e0e567290743a6ab33a288460374d7ef6bcd239/numpy-2.5.3-cp315-cp315t-win32.whl", hash = "sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469", size = 6330292, upload-time = "2026-09-06T16:27:38.149Z" }, + { url = "https://files.pythonhosted.org/packages/6c/b0/ff5658a58199b7bcaad87bf260eef6713d9d42cca4e028f935b4fc5fbac6/numpy-2.5.3-cp315-cp315t-win_amd64.whl", hash = "sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551", size = 12884918, upload-time = "2026-09-06T16:27:40.965Z" }, + { url = "https://files.pythonhosted.org/packages/fb/0b/b12a2df5d1b774bd9007a6fdff9381145b6223d37f11afc9c37ab0efd9a1/numpy-2.5.3-cp315-cp315t-win_arm64.whl", hash = "sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d", size = 10850807, upload-time = "2026-09-06T16:27:43.868Z" }, ] [[package]] @@ -3016,27 +3016,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/85/c8e12473c93018f92d19dd988a294202e1c27426c47ec4de53ffb847b8d8/ruff-0.16.5.tar.gz", hash = "sha256:1b88500f9ffbcab3dedb0082c9f9492e91ec3d618aac1236a3e0189938f7040b", size = 4912003, upload-time = "2026-08-27T16:34:18.258Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/b6/77c90a970fe2dae17a723acbd011043ea97c98d7deacccefdc4ba74ec512/ruff-0.16.5-py3-none-linux_armv6l.whl", hash = "sha256:12e5f673e774c35fbb62f288809c7653b73445f8ecec6b6063fd6ea3521aa14b", size = 10011941, upload-time = "2026-08-27T16:33:41.287Z" }, - { url = "https://files.pythonhosted.org/packages/4b/46/6cf67cf6411885a1d6f7f6d801682f155536a85176d10b605e2ceffed8bd/ruff-0.16.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:eda58a5802de40e7ed5b32b64e0b32539338cc6fcd2c78f61e3ad6a0d79f51c3", size = 10204049, upload-time = "2026-08-27T16:33:44.056Z" }, - { url = "https://files.pythonhosted.org/packages/46/fd/c8720ca7a090abf0c2fef4abe8a5ef6e5127ed15196d8886ff75a2b370e2/ruff-0.16.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5ae9a7b9a8875131f40f8fe967cc86abf899779efd663cb7ce3d572d01da7eb", size = 9809037, upload-time = "2026-08-27T16:33:46.257Z" }, - { url = "https://files.pythonhosted.org/packages/43/45/a684caacdedaca180f52bacccc40bf0789d2c5a7c75f25324853e9eaedb5/ruff-0.16.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b719b0a1f4d59710d283ab2965f621684a108a9e41da622e3b23f0326cd0025", size = 9964129, upload-time = "2026-08-27T16:33:48.352Z" }, - { url = "https://files.pythonhosted.org/packages/9e/f2/5d2bcdaca6b5b93d1b4dfc166cd2aebf7680143a1b38a28759df13a94d31/ruff-0.16.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2298f2780ed1be0c5cb1361e32ab7b1467f3cce7dabe101d2210a314f2fe42e9", size = 9821518, upload-time = "2026-08-27T16:33:50.57Z" }, - { url = "https://files.pythonhosted.org/packages/aa/ff/011cce29accf9257d5974145b733fc653a37985ed6825413a3987cefbfe0/ruff-0.16.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:258f29035a2dd021e7861e631b227a5b3f14e50c1184c9a6a122c5f4576154d7", size = 10534835, upload-time = "2026-08-27T16:33:52.522Z" }, - { url = "https://files.pythonhosted.org/packages/d7/5a/f0cf109bada9bba0e96c90c21c9f9251803f57225c32d293327a03c710d6/ruff-0.16.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9a4f0432966834019c74d1b7e5c51224305d7713f3d7faf3e7451f1a3be3cde", size = 11252550, upload-time = "2026-08-27T16:33:54.521Z" }, - { url = "https://files.pythonhosted.org/packages/63/4d/1d481aaea2046c6a7ed7c291f9004c669cce3c087b6b376ed5b08271e3fe/ruff-0.16.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5eb3a8c3d0ade9cea42b591fd530368e8798380e30e0a308b85a5cf718f09ea", size = 10777949, upload-time = "2026-08-27T16:33:56.88Z" }, - { url = "https://files.pythonhosted.org/packages/ee/34/ee245ca55f64443233034b3d02b03236b19242004281247c079390b7facd/ruff-0.16.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef0f69e191a13a3c9816f63163c88790cb12cd157bbbb384e9c44745702ab105", size = 10311656, upload-time = "2026-08-27T16:33:59.12Z" }, - { url = "https://files.pythonhosted.org/packages/a7/4d/c33a333e341c0a2b96c715b52d89a606f5a34cd4ac493cd9b8d0187186b8/ruff-0.16.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0eeab41fbea2c42f98dfb9822cdccda9d24ba38d49f6dc945b5c236d48f0ef29", size = 10532125, upload-time = "2026-08-27T16:34:01.166Z" }, - { url = "https://files.pythonhosted.org/packages/30/e1/a64cef78b40192497bb98a27a8aa8f2c98ee9ee15bc97f7712d94ef32937/ruff-0.16.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f0768e9df4300713fff30733c87575f68b6f1d8de41184e505b7fdd9c0c95eaf", size = 10097648, upload-time = "2026-08-27T16:34:03.16Z" }, - { url = "https://files.pythonhosted.org/packages/cc/4e/4cdc9ed3c3e109d2f71e62572a37457298d7bc7501ec3138babb7ed32bbd/ruff-0.16.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:95cc70cdc7aa80c338de356279d2adbeb2de0f520b9ecd8aba75b94e95e02f91", size = 9829344, upload-time = "2026-08-27T16:34:05.134Z" }, - { url = "https://files.pythonhosted.org/packages/39/4a/31ed35ce31729955fc583ee0d176d6e784c1290cb0b0a75cb2134c1ab72a/ruff-0.16.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d185c8398ded1bfd91c0c2cb258346307571eccc473a8490af8c3977399c384a", size = 10277117, upload-time = "2026-08-27T16:34:07.425Z" }, - { url = "https://files.pythonhosted.org/packages/a8/a0/60356d86687b4b666d593df213f4dc3041750d024cb7bf2cfa81cfd65c2e/ruff-0.16.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fb8e3a3c4c6a784150a7ced53b015f4b253fc2bf97a610886419ead64b4756ef", size = 10711653, upload-time = "2026-08-27T16:34:09.712Z" }, - { url = "https://files.pythonhosted.org/packages/ed/20/656d67f5b25ca9bda4e02b1de25867b2954e1d19e03648060f167ad0f4cc/ruff-0.16.5-py3-none-win32.whl", hash = "sha256:288b0a5f080492fe5635db849f9e2e84aa3cce7b7f0e955997d416c507c76a26", size = 10034250, upload-time = "2026-08-27T16:34:11.8Z" }, - { url = "https://files.pythonhosted.org/packages/5b/42/ee8e68a207b9127fcde6c3d7e197def432f346cb1af159e1fa14ca0d1cdc/ruff-0.16.5-py3-none-win_amd64.whl", hash = "sha256:ddc6385fb2137f616357ca03d6c74f4be987f80fed4008566b754f6032b8546f", size = 10516714, upload-time = "2026-08-27T16:34:13.963Z" }, - { url = "https://files.pythonhosted.org/packages/73/e3/7df5a396e445b9ba49ce9a9437439a4d80042c61c0ade199abf8d16de1ac/ruff-0.16.5-py3-none-win_arm64.whl", hash = "sha256:a64abe90968719b851bb7cedffaa8753fbdbdadab483089682db623f3edc587e", size = 10391564, upload-time = "2026-08-27T16:34:16.064Z" }, +version = "0.16.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/7c/6adb35d70e7c027e308274557901c7e00fb3407750faf3620c184ae058cb/ruff-0.16.6.tar.gz", hash = "sha256:dcf8a73d2ff77e99dde91244b4da16feba7f14e6beeb4015dee7c5a909e99050", size = 4921251, upload-time = "2026-09-03T16:57:29.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/28/9cc1b79639e284ec103f43c88c644db4eb58cbd0ea1ca11f1193435369ac/ruff-0.16.6-py3-none-linux_armv6l.whl", hash = "sha256:61c368c26bf8e973e5ab14a2772de587bc068ea3f9a277f673380749b4898fb8", size = 10015638, upload-time = "2026-09-03T16:56:40.986Z" }, + { url = "https://files.pythonhosted.org/packages/71/11/627d342ef727ea7794edf74fe23d60a074b02c3acc2e9436684e782286ca/ruff-0.16.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ecf4f068e2e123e43a26e9db4e19524cc56563912404e83bbfca375757e45a32", size = 10220762, upload-time = "2026-09-03T16:56:44.681Z" }, + { url = "https://files.pythonhosted.org/packages/43/d9/b75668ce41e4c8d073d18d6d08672ba6906ce45d5c06ea4fdb2e84ce3853/ruff-0.16.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:99b62ea33baf130f50368798d841f0d95527b6d817bf31817b65dd058f1d314c", size = 9835082, upload-time = "2026-09-03T16:56:47.142Z" }, + { url = "https://files.pythonhosted.org/packages/99/97/123ab10b05cde889c107c20f5a9774955104b5552796a2a8584b089ae8eb/ruff-0.16.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fbf89013f2bb3f6835a6038ff658dc8a1b38c98dc8e724b964168ad4e881876", size = 9949304, upload-time = "2026-09-03T16:56:49.813Z" }, + { url = "https://files.pythonhosted.org/packages/3e/58/a4a2c59dd2e5b85929c912d9cac3056eb9ee8c7e75e9b9fe3e109174966b/ruff-0.16.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56a67065e22efa6bc4d498299d3bb06c0c90aace8fac2068b5a12f9dc4d8d51d", size = 9840612, upload-time = "2026-09-03T16:56:52.368Z" }, + { url = "https://files.pythonhosted.org/packages/61/6a/ff8c8626a786c4f49d48ced4a752dadbca65f5263005f9c2416578194694/ruff-0.16.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e25cc89174874b176a157e4428d66761c2c0c006654419bf384f967f361ff1b1", size = 10543465, upload-time = "2026-09-03T16:56:55.089Z" }, + { url = "https://files.pythonhosted.org/packages/ad/bb/c47535923365f337b82e28192e4e9eef2176511007cfd99a62fc22df5dad/ruff-0.16.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0700580ed5303723cb3c11c2f1d2a8913ce77b7ea86646dddb887f5417a9ba70", size = 11267576, upload-time = "2026-09-03T16:56:57.791Z" }, + { url = "https://files.pythonhosted.org/packages/ba/50/e5119a5212b5cd63b51e1f4b25e7bd636a6668fc069a3160b108ad7e3c16/ruff-0.16.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15f1d0b6e165a6e56567befb6629f8209271311d990bae0f37e6d065035ef5f3", size = 10781993, upload-time = "2026-09-03T16:57:00.666Z" }, + { url = "https://files.pythonhosted.org/packages/8b/98/083d8b4ef3c51a0d19db84367791cbe9f44e4b53343d19dfa83556e1cd9a/ruff-0.16.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d72c591a96986ee4268860e2b7235082129ca5e4cb9cbba653a4b57c11893757", size = 10317748, upload-time = "2026-09-03T16:57:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/9a/29/68f7ff2c5ad95f19f00627ac2de95644e25fe47371ea60b2db1fd952315e/ruff-0.16.6-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:65a006baa18f33324325814c864daef03541d51564b98c517610ea756ab7003e", size = 10540096, upload-time = "2026-09-03T16:57:06.182Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f9/79a8f6de85968641d68a7863aeec577551924ef066a990a48ff93167beab/ruff-0.16.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:cd02a7bf1a21a8735228a3e8c95a9dc5cf86bd2a52194f4aaae2a5755b4de0f4", size = 10100494, upload-time = "2026-09-03T16:57:09.194Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e8/b81a22d9b90c00b892ccf2fa2ac36fa95de4c13ab85aea3e73795cfe4651/ruff-0.16.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:31b36f1e5ad85e0737f09d2be4e512e2e283583c14015da3b9dc07359ac0fc88", size = 9843663, upload-time = "2026-09-03T16:57:12.168Z" }, + { url = "https://files.pythonhosted.org/packages/39/aa/54f516ec5e5a11c4afdceb1c454ebb054ffb96e4f4a1705580b4346abd35/ruff-0.16.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:61029b4ab4aa723fd3064fab96b1d814492596bf0c792679fffcbde1e1679953", size = 10282461, upload-time = "2026-09-03T16:57:15.077Z" }, + { url = "https://files.pythonhosted.org/packages/52/0b/38d0aa8aa32372b96dc44f97b22e576c4147808271aab7b2cb1e353d4445/ruff-0.16.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9ac8998457832c2061709d900856b7ad271dace0cb41f346588d540162bfa718", size = 10728808, upload-time = "2026-09-03T16:57:17.797Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e5/9e274e24eeb027640ffc7442f21239f16d17f47acec15ae34f32e03a5c79/ruff-0.16.6-py3-none-win32.whl", hash = "sha256:0b87d9d16fcb63e8018423ca1d50b7260f15cb2da33e30db4baad4183a948c25", size = 10049212, upload-time = "2026-09-03T16:57:20.55Z" }, + { url = "https://files.pythonhosted.org/packages/22/31/72472449414223ed1a2da236b992adbb1a2ae59e34794574810f60ce068e/ruff-0.16.6-py3-none-win_amd64.whl", hash = "sha256:10d21c51c3495d8eaea7b703a16592117ea6eb1d649e36335aa965ff1173eb39", size = 10556402, upload-time = "2026-09-03T16:57:23.501Z" }, + { url = "https://files.pythonhosted.org/packages/fc/07/d781f8f8e1ac24bef9f3269cf62ffb1407ca24c3a8f12e5e22874f90528c/ruff-0.16.6-py3-none-win_arm64.whl", hash = "sha256:7a976c79b958f94e50a022a19f0f8c87387448020935ec14fc74331bd0a7f2c5", size = 10412850, upload-time = "2026-09-03T16:57:26.416Z" }, ] [[package]] @@ -3227,15 +3227,15 @@ wheels = [ [[package]] name = "towncrier" -version = "25.8.0" +version = "26.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "jinja2" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c2/eb/5bf25a34123698d3bbab39c5bc5375f8f8bcbcc5a136964ade66935b8b9d/towncrier-25.8.0.tar.gz", hash = "sha256:eef16d29f831ad57abb3ae32a0565739866219f1ebfbdd297d32894eb9940eb1", size = 76322, upload-time = "2025-08-30T11:41:55.393Z" } +sdist = { url = "https://files.pythonhosted.org/packages/92/d4/e85ce614bf45b1a7fe13b82cc687819c2eaac5a1dd94b84834322095fd98/towncrier-26.9.0.tar.gz", hash = "sha256:ace9031631c718cc0709107b8755e6e31350eb8acdc877af43df3177addb9a55", size = 79338, upload-time = "2026-09-04T12:57:25.341Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/06/8ba22ec32c74ac1be3baa26116e3c28bc0e76a5387476921d20b6fdade11/towncrier-25.8.0-py3-none-any.whl", hash = "sha256:b953d133d98f9aeae9084b56a3563fd2519dfc6ec33f61c9cd2c61ff243fb513", size = 65101, upload-time = "2025-08-30T11:41:53.644Z" }, + { url = "https://files.pythonhosted.org/packages/82/88/f9340b545dafa64de053798e760515a2736fbb60092cf9aa6b146ccf3ede/towncrier-26.9.0-py3-none-any.whl", hash = "sha256:ae4d223e6aadaff98d29b7f09e58d9c9ccf4cd3b455a2d2e0486d432cd8bf660", size = 67030, upload-time = "2026-09-04T12:57:23.986Z" }, ] [[package]] @@ -3298,28 +3298,28 @@ wheels = [ [[package]] name = "uv" -version = "0.12.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/42/6d2be569fd280ebd982a54ee2e3d3754fedde5cf557fd7f858eb40d161cf/uv-0.12.9.tar.gz", hash = "sha256:55b8920edb2a29eeef31e246fb60d2145985e17f51ad14b26643575b82759302", size = 7134801, upload-time = "2026-09-01T21:57:47.305Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/4a/208c1ed443a5024bdb83cedf1e09b322e68ccc9f7e11d1f53bb1f0690174/uv-0.12.9-py3-none-linux_armv6l.whl", hash = "sha256:97062b2160a97fcfa45f10bbbe02a9c6ba4f5a50cf4e02c0f2babb80b47c1ea8", size = 22183326, upload-time = "2026-09-01T21:57:01.443Z" }, - { url = "https://files.pythonhosted.org/packages/7a/4f/7cc3d45951bb01089bf948d91a89c5b433789df8ceeba4f4cbcce0b1ef5c/uv-0.12.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0cb7e8a732d81f25d364de8d6ac69f4d00d56b581918ac242eeaeffb53977588", size = 20481767, upload-time = "2026-09-01T21:57:04.573Z" }, - { url = "https://files.pythonhosted.org/packages/5f/2b/28f22620d336069038e850ff662be979866c4944126d17a19bed738e3dc1/uv-0.12.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:0af988f800778913d36f9bf6905eb21c917dccc403bb364ad739bfd383acf491", size = 17338955, upload-time = "2026-09-01T21:57:06.884Z" }, - { url = "https://files.pythonhosted.org/packages/cc/29/649ff920dcd2373a819a4e0d9c1989ae770698e634c277e5a6fbbe19e9b4/uv-0.12.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:cce6e716df290b7f4a31f76ef39866e1dc006e947520a660459dea00b5d55234", size = 21598566, upload-time = "2026-09-01T21:57:09.223Z" }, - { url = "https://files.pythonhosted.org/packages/35/0d/0db7cfdd5bf11ce91c53558449f0e753d3cc578d9e94bd350c10b43f17ef/uv-0.12.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:00ff9f34c04055ebc3e1599ee459f5ae0d42c17cd6a1d8d029f0389e64a1419c", size = 21690729, upload-time = "2026-09-01T21:57:11.866Z" }, - { url = "https://files.pythonhosted.org/packages/69/bf/a63391a9d0f930f5ec23ff6a1daa9f7806a430fe41fb1e3bfb6cb10ef737/uv-0.12.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2d4a1d4c067f9e2edf703e181a71db8fc9d7af5fcdbd0053103d0a1260e04ce6", size = 21716103, upload-time = "2026-09-01T21:57:14.407Z" }, - { url = "https://files.pythonhosted.org/packages/f5/4c/4b1fea7c22abac3297fa7fc222f3703c33ffe45aa7553fa9d8bd2b0b70bb/uv-0.12.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cafa85338ce1d27b7caa4aadc83ab21f350e2cd2185462fbb10a232b6e93152e", size = 22340621, upload-time = "2026-09-01T21:57:17.109Z" }, - { url = "https://files.pythonhosted.org/packages/25/57/7ac6fd2b97ea9f608b65bb9b47d686e5d64f8142b190d3c4590d74cf8524/uv-0.12.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bbdcb9db8b44d80e3fcebc4205604d52ea145404efb179958560bb2a0157743a", size = 23586367, upload-time = "2026-09-01T21:57:19.494Z" }, - { url = "https://files.pythonhosted.org/packages/ff/87/3ef6bf32db7af74e7c230ef6c06c387d9be2363dfa63eb524065031c010a/uv-0.12.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cfb512d0e284e9fc42c3bbe9b701f430083fca23dbb67a20daeea897fb793069", size = 23279627, upload-time = "2026-09-01T21:57:22.146Z" }, - { url = "https://files.pythonhosted.org/packages/37/4b/cd04809c7ad5149faac55160925bd67aab43f803f6c01be1c32aec7d24d9/uv-0.12.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5badfd805fd88bf99b4b4f044f6e8f762f1892cab27477f4427bb473e93dd049", size = 20056367, upload-time = "2026-09-01T21:57:24.49Z" }, - { url = "https://files.pythonhosted.org/packages/29/6b/59cddb48c47124428d329ac3ed4ce91ccd8ab757ebeed3d482115d8b82ef/uv-0.12.9-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:c0f7e2b46e4c503f9497c0f41bba3570aa233685db814cd011225118f2d72b6a", size = 19383069, upload-time = "2026-09-01T21:57:27.019Z" }, - { url = "https://files.pythonhosted.org/packages/99/a6/22e4650cc9a2d018165fda5a53ca7addedc6d4f083ab7c833acc12fbe1e1/uv-0.12.9-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:445c572202243229043abaeab2fecb1b3691477c71c79feb07f638ce34febd39", size = 22392774, upload-time = "2026-09-01T21:57:29.323Z" }, - { url = "https://files.pythonhosted.org/packages/d4/37/68166b0acb7f8ff30316bf74c0e9840c7c6cbc86ee3e236382bb107d42db/uv-0.12.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ce4ad780d3cefe5a8f965f3ddf942c01e71aa37e36f4fc431e72e4ee8ce5f378", size = 22517532, upload-time = "2026-09-01T21:57:31.758Z" }, - { url = "https://files.pythonhosted.org/packages/28/ac/8801d60db574aa15fba53944e0ee2e298605bf9076a5389f5e56ed51206a/uv-0.12.9-py3-none-musllinux_1_1_i686.whl", hash = "sha256:f639f8415f08918fb4ba2022bfd94d039e01c6c1eca28583125d439f7bd635fb", size = 21498480, upload-time = "2026-09-01T21:57:34.402Z" }, - { url = "https://files.pythonhosted.org/packages/ec/47/663bbc05f26c6c518c086da5df1e38fc5b2650fa6768cb36c6418d3f9559/uv-0.12.9-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:5f9325495fa0bf3e71e6b09f72e68b0d4a000067888b48a659536a8c7191a112", size = 22889848, upload-time = "2026-09-01T21:57:37.284Z" }, - { url = "https://files.pythonhosted.org/packages/58/ce/a4b1ae2a80264231a4dd0a070ffd0fe7f2bdf1d78e99d3c0c68ce315262b/uv-0.12.9-py3-none-win32.whl", hash = "sha256:f4ab95475562884fc8e72fbbf0c3da0e025d3bb25e7348544cd02b19bf77f3e1", size = 19858143, upload-time = "2026-09-01T21:57:39.571Z" }, - { url = "https://files.pythonhosted.org/packages/f5/b2/55c7fdbdacb0b110f92804cceea449dfcd5be4354ae1e997aca700145a65/uv-0.12.9-py3-none-win_amd64.whl", hash = "sha256:871e884c3329f69c12bb7426edb9e727745c3bb2db6d9e0538ce0e1bdfb64916", size = 17991226, upload-time = "2026-09-01T21:57:42.282Z" }, - { url = "https://files.pythonhosted.org/packages/85/67/cac985492192ff4a95a76b3854143062f1b3290aca3a404a812b7aadc9cf/uv-0.12.9-py3-none-win_arm64.whl", hash = "sha256:31fddf25d756a51fe33f7831ff697a122628dd3121c37c435c39c678428f937b", size = 19472343, upload-time = "2026-09-01T21:57:44.877Z" }, +version = "0.12.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/1e/7827991def6006d84adc5113bc000956e83d190da00f836167775f276af5/uv-0.12.12.tar.gz", hash = "sha256:981a00db03357d9e9eec2b0fd128ac8b6786bf9265a56b77b9d9fc22f30fec3b", size = 7162711, upload-time = "2026-09-09T16:44:12.259Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/5e/705f15eaf979e50e1f6ff7238fa491196944a6137f724aaf69ab93f13fd5/uv-0.12.12-py3-none-linux_armv6l.whl", hash = "sha256:93deea45061d858760f8756fcca54a3e694978fe7f96e4987ad95ab47f72d554", size = 22185914, upload-time = "2026-09-09T16:43:29.816Z" }, + { url = "https://files.pythonhosted.org/packages/c5/49/1e5c867d088fd6641f15db3f50e07770a466f54309b818242bfd21e11469/uv-0.12.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d3fa745404dd52a1763bc159073deacc5c0d501cb5e5c0baeeb53bf26f6d3482", size = 20476071, upload-time = "2026-09-09T16:43:32.473Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8d/ddec2493ee930ebf32676f7b3715d75915eda45c981f6621ce3e99b69254/uv-0.12.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e29bf70f889e801f7dc440b51b859f005311416aa3b5c42ba9dc387c7e877c00", size = 16743115, upload-time = "2026-09-09T16:43:34.574Z" }, + { url = "https://files.pythonhosted.org/packages/77/27/b7b7486ebc998ab0893169a02d87d540c47bc6dd821de7e11156e343697a/uv-0.12.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:bff88c94d0b09149c1017b87b9eeca16764df37e1a6717f2b64a4104e3fb6795", size = 21666817, upload-time = "2026-09-09T16:43:36.699Z" }, + { url = "https://files.pythonhosted.org/packages/af/c1/131f9de447f8f7299ebe3f5cb5089f7c7f2a62b257d81a5cffee5bacd237/uv-0.12.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:75c36925481ab9a3f0f31a737636597653ca228d14152420a6047a85c6a16fb0", size = 21668514, upload-time = "2026-09-09T16:43:38.962Z" }, + { url = "https://files.pythonhosted.org/packages/12/56/94253e4f0e3806236393415b3b941625f3466c0e4091330827e3f07c27a6/uv-0.12.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0635b6d6601cf93d442353fa54f8dddfc3129f3f74f09721fc402983a57dc172", size = 21708940, upload-time = "2026-09-09T16:43:41.327Z" }, + { url = "https://files.pythonhosted.org/packages/30/be/cd8d050f00b11f01b59b13b4e89801f8f4bfbda599fad5858703b8b81fd3/uv-0.12.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e87f83293e6b3521660aa5c9aa5ab2a34fdc87845c90a466563cfbd53acf07de", size = 22402933, upload-time = "2026-09-09T16:43:43.686Z" }, + { url = "https://files.pythonhosted.org/packages/1a/84/c131548ba3e7726ad105d2022a9e761e09225736678d771cfac502141674/uv-0.12.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6bd1bf48fbc973c0700bf35717204b1e2a659e54fb68a010621b4949128f4a7e", size = 23701106, upload-time = "2026-09-09T16:43:46.16Z" }, + { url = "https://files.pythonhosted.org/packages/6c/0e/f9b3feda985e48d87a6c8375a1255af04af3e759dc92329387215830cd15/uv-0.12.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4a8535ca0ebd23d03a5a19b7c7a1c7567aac2aeaa06ee76716e133339e091c56", size = 23357681, upload-time = "2026-09-09T16:43:48.623Z" }, + { url = "https://files.pythonhosted.org/packages/be/c2/e8ff20e5f0bf1011995688784bc71ed7edbc7ccd3836e4967895680b2968/uv-0.12.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa5df02fc619a3cc7a58810d6ffeb80ca1e01404b8ef7239bd1cf2103c02cacf", size = 20041994, upload-time = "2026-09-09T16:43:50.98Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b6/3c3e39422e507de31270ea3f82d66c8dcf010ed35cb8559dbdfb65078413/uv-0.12.12-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:091e942f0677f88389265c9028d64108c0641593dd8993166eae71e33fc5b5aa", size = 19458417, upload-time = "2026-09-09T16:43:53.209Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3a/f39ffe3cd7f3fbc33b3f62eef26274ed3f42a01ceb18296ec6fd65a36080/uv-0.12.12-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:5b22f74a63fae875baf9af5818b08901f5a7addd5440b121245e8c4c7bb84084", size = 22474163, upload-time = "2026-09-09T16:43:55.363Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9c/6a0105b5102486da1ad8d0dbfb438adc61a5e5d8397757a24fd5f6d17a43/uv-0.12.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0e27713b2441d9204a35e0799c6e44ffeef9978e72ea5530cafcf91f8b24986c", size = 22599041, upload-time = "2026-09-09T16:43:57.702Z" }, + { url = "https://files.pythonhosted.org/packages/c3/14/a86c94d61245f88dfd825aef961f680d6dd7ba331b4ac5254a6de44aab36/uv-0.12.12-py3-none-musllinux_1_1_i686.whl", hash = "sha256:206446685f487f21c7eff07f74513693dd53a2374934b4324ebb1a1d665b595d", size = 21570488, upload-time = "2026-09-09T16:44:00.391Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c2/f81d871bb447fcf28667fda4a15b5ad6c2719ba2cd34e51b674167dfab83/uv-0.12.12-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:98d9659e080c42c289e4552d4f01b758cea544a96e3d9e5ac8565faacf84f970", size = 22996969, upload-time = "2026-09-09T16:44:02.703Z" }, + { url = "https://files.pythonhosted.org/packages/ff/5f/e1190ec0810e3d785b14f6d9c43b744a7869d3ecec3f5cda3dd47cdcf0f7/uv-0.12.12-py3-none-win32.whl", hash = "sha256:febbb31b52cd47061e496ca12e34cf083981e167d1d0ebadf4a59a0ce369d86a", size = 19441083, upload-time = "2026-09-09T16:44:04.984Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b7/aebd3504d559315be6137ebfe1f6d2258e8e2f48e528d7cc799aea3441ef/uv-0.12.12-py3-none-win_amd64.whl", hash = "sha256:4a5acd0c6087970bdf210eeb1203835117f6fb8b84828604205de510f68aa6a7", size = 17654279, upload-time = "2026-09-09T16:44:07.495Z" }, + { url = "https://files.pythonhosted.org/packages/34/f5/70f1cb8ad7dba238489a86a341000c133db54aa12cdd81e67c98ebe9a59a/uv-0.12.12-py3-none-win_arm64.whl", hash = "sha256:3577343756d8bf686c92eaca89a79fb0c1b6a85c2f0b579d535db22a1b49eb4b", size = 18863039, upload-time = "2026-09-09T16:44:09.961Z" }, ] [[package]] @@ -3701,7 +3701,7 @@ dev = [ { name = "coverage", specifier = "==7.16.0" }, { name = "fsspec", specifier = ">=2023.10.0" }, { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, - { name = "hypothesis", specifier = "==6.167.1" }, + { name = "hypothesis", specifier = "==6.168.0" }, { name = "markdown-exec", extras = ["ansi"], specifier = "==1.12.3" }, { name = "mike", specifier = "==2.2.0" }, { name = "mkdocs", specifier = "==1.6.1" }, @@ -3723,12 +3723,12 @@ dev = [ { name = "pytest-reportlog", specifier = "==1.0.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "requests", specifier = "==2.34.2" }, - { name = "ruff", specifier = "==0.16.5" }, + { name = "ruff", specifier = "==0.16.6" }, { name = "s3fs", specifier = ">=2023.10.0" }, { name = "tomlkit", specifier = "==0.15.1" }, - { name = "towncrier", specifier = "==25.8.0" }, + { name = "towncrier", specifier = "==26.9.0" }, { name = "universal-pathlib" }, - { name = "uv", specifier = "==0.12.9" }, + { name = "uv", specifier = "==0.12.12" }, ] docs = [ { name = "astroid", specifier = "==4.3.1" }, @@ -3742,16 +3742,16 @@ docs = [ { name = "mkdocstrings-python", specifier = "==2.0.8" }, { name = "numcodecs", extras = ["msgpack"] }, { name = "pytest", specifier = "==9.1.1" }, - { name = "ruff", specifier = "==0.16.5" }, + { name = "ruff", specifier = "==0.16.6" }, { name = "s3fs", specifier = ">=2023.10.0" }, - { name = "towncrier", specifier = "==25.8.0" }, + { name = "towncrier", specifier = "==26.9.0" }, ] -release = [{ name = "towncrier", specifier = "==25.8.0" }] +release = [{ name = "towncrier", specifier = "==26.9.0" }] remote-tests = [ { name = "botocore" }, { name = "coverage", specifier = "==7.16.0" }, { name = "fsspec", specifier = ">=2023.10.0" }, - { name = "hypothesis", specifier = "==6.167.1" }, + { name = "hypothesis", specifier = "==6.168.0" }, { name = "moto", extras = ["s3", "server"], specifier = "==5.2.3" }, { name = "numpydoc", specifier = "==1.10.0" }, { name = "obstore", specifier = ">=0.5.1" }, @@ -3766,11 +3766,11 @@ remote-tests = [ { name = "requests", specifier = "==2.34.2" }, { name = "s3fs", specifier = ">=2023.10.0" }, { name = "tomlkit", specifier = "==0.15.1" }, - { name = "uv", specifier = "==0.12.9" }, + { name = "uv", specifier = "==0.12.12" }, ] test = [ { name = "coverage", specifier = "==7.16.0" }, - { name = "hypothesis", specifier = "==6.167.1" }, + { name = "hypothesis", specifier = "==6.168.0" }, { name = "numpydoc", specifier = "==1.10.0" }, { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-accept", specifier = "==0.3.0" }, @@ -3781,5 +3781,5 @@ test = [ { name = "pytest-reportlog", specifier = "==1.0.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "tomlkit", specifier = "==0.15.1" }, - { name = "uv", specifier = "==0.12.9" }, + { name = "uv", specifier = "==0.12.12" }, ] From f2ab3d5ba4a37f43078b4a693ce4aad73d895641 Mon Sep 17 00:00:00 2001 From: glaziermag <130600081+glaziermag@users.noreply.github.com> Date: Thu, 17 Sep 2026 01:15:58 -0700 Subject: [PATCH 21/29] fix(ci): preserve branch zarr in xarray downstream tests (#4368) * fix(ci): preserve branch zarr in xarray downstream tests * chore: use pull request number for changelog fragment --------- Co-authored-by: glaziermag Co-authored-by: Davis Bennett --- .github/workflows/downstream.yml | 5 +++-- changes/4368.misc.md | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changes/4368.misc.md diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 30210cd452..eb0cd4a414 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -59,10 +59,11 @@ jobs: working-directory: xarray run: uv pip install --no-deps .. + # Keep the branch override: syncing xarray would reinstall zarr from PyPI. - name: Show versions working-directory: xarray run: | - uv run python -c " + uv run --no-sync python -c " import zarr; print(f'zarr {zarr.__version__}') import xarray; print(f'xarray {xarray.__version__}') " @@ -70,7 +71,7 @@ jobs: - name: Run xarray zarr backend tests working-directory: xarray run: | - uv run python -m pytest --no-header -q \ + uv run --no-sync python -m pytest --no-header -q \ xarray/tests/test_backends.py \ xarray/tests/test_backends_api.py \ xarray/tests/test_backends_datatree.py diff --git a/changes/4368.misc.md b/changes/4368.misc.md new file mode 100644 index 0000000000..bdb87c70cf --- /dev/null +++ b/changes/4368.misc.md @@ -0,0 +1 @@ +Prevent the downstream xarray CI job from replacing the branch version of Zarr with a PyPI release before reporting versions and running tests. From df1ff8794508d37d8fa7de2003720375ff82789e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 10:18:24 +0200 Subject: [PATCH 22/29] fix(ci): harden the just migration's environment, pinning and release path Review follow-ups to the Justfile centralization: - `just gpu` read HATCH_ENV, so an exported CPU test environment silently redirected `pytest -m gpu` into an env built without the gpu feature. It now reads GPU_HATCH_ENV, which nothing else sets. - Pin `uvx prek` to 0.5.3 and restore hook-environment caching in the lint workflow, so lint no longer floats on whatever prek PyPI serves that day. - Restore the mypy hook's `uv run --frozen mypy` entry: routing it through `just typecheck` made just a prerequisite for committing in every existing clone, and expanded to the identical command. - Run the changelog filename check with `uv run --no-project python` instead of building the `dev` hatch env (test + remote-tests + docs + mypy) for a script that imports only sys and pathlib. - Extend `just just-check` over packages/*/justfile, which the root delegation recipes depend on and the formatting gate did not cover. - Set up Python in the codspeed job before pip-installing into it, matching the other workflows; that runner image was never given one. - Revert the release and nightly-wheel jobs to `hatch build`, so the publishing path does not fetch rust-just to run a pure alias. - Renumber the changelog fragment to this pull request and credit #4096 in the body, so the rendered note does not link to an unrelated upstream PR. Assisted-by: ClaudeCode:claude-opus-5 --- .github/workflows/check_changelogs.yml | 6 ++---- .github/workflows/codspeed.yml | 4 ++++ .github/workflows/gpu_test.yml | 5 ++++- .github/workflows/lint.yml | 10 ++++++++++ .github/workflows/nightly_wheels.yml | 4 +--- .github/workflows/releases.yml | 4 +--- .pre-commit-config.yaml | 2 +- Justfile | 21 ++++++++++++++------- changes/339.misc.md | 1 + changes/4096.misc.md | 1 - docs/contributing.md | 6 ++++-- 11 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 changes/339.misc.md delete mode 100644 changes/4096.misc.md diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index a8b84da7ba..7c81fbaba3 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -24,10 +24,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - - name: Install task runners - run: | - uv tool install hatch==1.16.5 - uv tool install rust-just==1.58.0 + - name: Install just + run: uv tool install rust-just==1.58.0 - name: Check zarr-python changelog entries run: just check-changelogs diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 4694c8dbc7..8e3ecdd82f 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -27,6 +27,10 @@ jobs: with: fetch-depth: 0 persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index c4417b7a46..bcf933fe9c 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -69,6 +69,9 @@ jobs: version: '1.16.5' - name: Install just run: python -m pip install rust-just==1.58.0 + # Two names for the same environment on purpose: `just setup` builds whatever + # HATCH_ENV points at, while `just gpu` reads GPU_HATCH_ENV so that an + # ambient HATCH_ENV can never redirect `pytest -m gpu` into a CPU environment. - name: Set Up Hatch Env env: HATCH_ENV: gputest.py${{ matrix.python-version }} @@ -76,7 +79,7 @@ jobs: just setup - name: Run Tests env: - HATCH_ENV: gputest.py${{ matrix.python-version }} + GPU_HATCH_ENV: gputest.py${{ matrix.python-version }} run: | just gpu diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 04b0167313..a41f83124c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,6 +32,16 @@ jobs: enable-cache: true - name: Install just run: uv tool install rust-just==1.58.0 + # `uvx prek` builds each hook's environment from scratch, so cache them the + # way the prek action used to. Keyed on the hook config: a new pinned rev or + # a new hook is exactly when the cached environments stop being valid. + - name: Cache prek hook environments + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/prek + key: prek-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + prek-${{ runner.os }}- - name: Check justfile formatting run: just just-check - name: Lint diff --git a/.github/workflows/nightly_wheels.yml b/.github/workflows/nightly_wheels.yml index de625a8847..2cb913511e 100644 --- a/.github/workflows/nightly_wheels.yml +++ b/.github/workflows/nightly_wheels.yml @@ -39,9 +39,7 @@ jobs: version: '1.16.5' - name: Build wheel and sdist - run: | - pip install rust-just==1.58.0 - just build + run: hatch build - name: Upload nightly wheels uses: scientific-python/upload-nightly-action@e76cfec8a4611fd02808a801b0ff5a7d7c1b2d99 diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index b87c1fa7f7..0865314915 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -42,9 +42,7 @@ jobs: with: version: '1.16.5' - name: Build wheel and sdist - run: | - pip install rust-just==1.58.0 - just build + run: hatch build - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: releases diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b5d210511d..54345c819e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: - id: mypy name: mypy language: system - entry: just typecheck + entry: uv run --frozen mypy pass_filenames: false always_run: true types_or: [python, pyi] diff --git a/Justfile b/Justfile index dab90ee5e3..0ebeb39ff0 100644 --- a/Justfile +++ b/Justfile @@ -7,6 +7,12 @@ set windows-shell := ["bash", "-eu", "-o", "pipefail", "-c"] set positional-arguments hatch_env := env("HATCH_ENV", "test.py3.12-optional") +# Deliberately a different variable from HATCH_ENV: `just gpu` must not inherit a +# CPU test environment that happens to be exported in the caller's shell, which +# would run `pytest -m gpu` against an environment built without the gpu feature. +gpu_env := env("GPU_HATCH_ENV", "gputest.py3.12") +# Pinned so a prek release cannot change what CI lints without a commit here. +prek_version := "0.5.3" # List available recipes default: @@ -60,9 +66,9 @@ benchmark *args: benchmark-codspeed *args: hatch run {{ quote(hatch_env) }}:pytest tests/benchmarks --codspeed "$@" -# Run GPU tests with coverage (default environment: gputest.py3.12) +# Run GPU tests with coverage; select the environment with GPU_HATCH_ENV gpu *args: - HATCH_ENV={{ quote(env("HATCH_ENV", "gputest.py3.12")) }} just coverage -m gpu "$@" + HATCH_ENV={{ quote(gpu_env) }} just coverage -m gpu "$@" # Build documentation (warnings are errors) docs-build *args: @@ -89,15 +95,15 @@ docs-check: check-doc-exports lint-docs docs-build # Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...) lint *args: - uvx prek run --all-files "$@" + uvx prek@{{ prek_version }} run --all-files "$@" # Run hooks with a custom selection, e.g. just hooks run --last-commit hooks +args: - uvx prek "$@" + uvx prek@{{ prek_version }} "$@" # Install local pre-commit hooks hooks-install: - uvx prek install + uvx prek@{{ prek_version }} install # Type-check the library using the locked tooling environment typecheck *args: @@ -129,11 +135,12 @@ changelog-build *args: # Check changelog filenames (default: changes/; accepts a package changes directory) check-changelogs *args: - hatch run dev:python ci/check_changelog_entries.py "$@" + uv run --no-project python ci/check_changelog_entries.py "$@" -# Check recipe formatting +# Check recipe formatting of the root Justfile and every package justfile just-check: just --fmt --check + for f in packages/*/justfile; do just --justfile "$f" --fmt --check; done # Run a zarr-metadata recipe, or list its recipes with no arguments zarr-metadata *args: diff --git a/changes/339.misc.md b/changes/339.misc.md new file mode 100644 index 0000000000..7b55499cf1 --- /dev/null +++ b/changes/339.misc.md @@ -0,0 +1 @@ +Define development and CI commands in a root Justfile, with Hatch managing Python environments. Extracted from [#4096](https://github.com/zarr-developers/zarr-python/pull/4096). diff --git a/changes/4096.misc.md b/changes/4096.misc.md deleted file mode 100644 index ff4cf41a17..0000000000 --- a/changes/4096.misc.md +++ /dev/null @@ -1 +0,0 @@ -Define development and CI commands in a root Justfile, with Hatch managing Python environments. diff --git a/docs/contributing.md b/docs/contributing.md index 19ee571d6f..42e7d71dfd 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -93,14 +93,16 @@ just test ``` Test recipes default to `test.py3.12-optional`. Set `HATCH_ENV` to select a different -interpreter or dependency set, just as CI does. On Windows, run these commands in +interpreter or dependency set, just as CI does. `just gpu` reads `GPU_HATCH_ENV` +instead, so an exported `HATCH_ENV` cannot silently send GPU tests to an +environment built without the `gpu` feature. On Windows, run these commands in Git Bash. ```bash HATCH_ENV=test.py3.13-minimal just test HATCH_ENV=min_deps just coverage HATCH_ENV=upstream just coverage -HATCH_ENV=gputest.py3.12 just gpu +GPU_HATCH_ENV=gputest.py3.12 just gpu just test tests/test_array.py -k 'resize and not async' ``` From 050e42dac1412646030af2e3b0e77b8e263a220f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 10:24:52 +0200 Subject: [PATCH 23/29] fix(ci): install prek persistently for git hooks; run setup steps in one just process `just hooks-install` ran `uvx prek install`, and prek writes a hook shim that hard-codes the binary it was installed from, falling back to `prek` on PATH. Under uvx that path is an entry in uv's archive cache, so the first `uv cache prune` (or a bump of the pinned version) broke every commit with `prek: not found`, and nothing put `prek` on PATH for the fallback. Install it as a pinned `uv tool` instead: a changed pin upgrades in place, a repeated run is a no-op, and the shim points at a path that stays. `setup` re-invoked `just list-env`, which dropped a `just hatch_env=... setup` override on the floor because the child process re-read HATCH_ENV. Run it as a subsequent dependency so both steps see the same value. Also retire the last two comments naming the removed hatch scripts. Assisted-by: ClaudeCode:claude-fable-5-1 --- Justfile | 11 +++++++---- docs/contributing.md | 4 ++-- lychee.toml | 2 +- tests/test_docs.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Justfile b/Justfile index 0ebeb39ff0..641671e6ac 100644 --- a/Justfile +++ b/Justfile @@ -23,9 +23,8 @@ envs: hatch env show # Create the selected Python environment and list its installed packages -setup: +setup: && list-env hatch env create {{ quote(hatch_env) }} - just list-env # List packages in the selected Python environment list-env: @@ -101,9 +100,13 @@ lint *args: hooks +args: uvx prek@{{ prek_version }} "$@" -# Install local pre-commit hooks +# Install local pre-commit hooks. prek is installed as a persistent uv tool, not run +# through uvx: the hook shim prek writes into .git/hooks hard-codes the binary path +# it was installed from and falls back to `prek` on PATH, and a uvx archive path +# stops existing at the next `uv cache prune`. hooks-install: - uvx prek@{{ prek_version }} install + uv tool install prek=={{ prek_version }} + prek install # Type-check the library using the locked tooling environment typecheck *args: diff --git a/docs/contributing.md b/docs/contributing.md index 42e7d71dfd..5c3cfbe8b0 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -82,7 +82,7 @@ git remote add upstream git@github.com:zarr-developers/zarr-python.git The root `Justfile` defines development and CI commands. [just](https://just.systems/) runs these commands, while [Hatch](https://hatch.pypa.io/latest/index.html) manages -the Python environments declared in `pyproject.toml`. Install the task tools and uv (used by the lint and lock commands): +the Python environments declared in `pyproject.toml`. Install the task tools and uv (several recipes run tooling through it): ```bash pip install hatch==1.16.5 rust-just==1.58.0 uv @@ -165,7 +165,7 @@ All tests are automatically run via GitHub Actions for every pull request and mu All code must conform to the PEP8 standard. Regarding line length, lines up to 100 characters are allowed, although please try to keep under 90 wherever possible. -`Zarr` uses a set of git hooks managed by [`prek`](https://github.com/j178/prek), a fast, Rust-based pre-commit hook manager compatible with `.pre-commit-config.yaml`. The just recipes use `uvx` to run prek, installing it on demand. +`Zarr` uses a set of git hooks managed by [`prek`](https://github.com/j178/prek), a fast, Rust-based pre-commit hook manager compatible with `.pre-commit-config.yaml`. The recipes pin the prek version: `just lint` and `just hooks` run it through `uvx`, and `just hooks-install` installs it as a persistent `uv tool` so the git hook can find it on later commits. The hooks can be installed locally by running: diff --git a/lychee.toml b/lychee.toml index dccb3001dc..9bb9be5caf 100644 --- a/lychee.toml +++ b/lychee.toml @@ -16,7 +16,7 @@ exclude_path = [ # URL patterns to ignore (regex, matched against the full URL). exclude = [ - # Local docs preview server shown in the contributing guide ("hatch run serve"), + # Local docs preview server shown in the contributing guide ("just docs-serve"), # documentation of a command rather than a reachable link. '^https?://0\.0\.0\.0', '^https?://(localhost|127\.0\.0\.1)(:\d+)?', diff --git a/tests/test_docs.py b/tests/test_docs.py index 874d90d4e5..6361591a5e 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -210,7 +210,7 @@ def test_test_only_blocks_come_last() -> None: Because we cannot statically tell which later blocks are state-dependent, this guard enforces the simple, safe convention only for the blocks we author this way (test="true" marker-bound examples like s3/gpu). It is NOT a complete build-hazard - check -- the authoritative check is `mkdocs build --strict` (the docs:check CI job), + check -- the authoritative check is `mkdocs build --strict` (`just docs-build` in CI), which catches the exec="false" case too. This guard just turns the common test-only case into a fast, local failure.""" # Collect, per published-docs file, the start lines of test-only and exec blocks. From 4aa6b9565e5d3f33ebc7793f95f6d1066801ae3c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 10:43:33 +0200 Subject: [PATCH 24/29] chore: renumber the changelog fragment to the upstream pull request towncrier renders the fragment name as a link into zarr-developers/zarr-python, so the fork's PR number would have pointed the released note at an unrelated upstream pull request. #4372 is the PR that merges this work. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- changes/{339.misc.md => 4372.misc.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{339.misc.md => 4372.misc.md} (100%) diff --git a/changes/339.misc.md b/changes/4372.misc.md similarity index 100% rename from changes/339.misc.md rename to changes/4372.misc.md From 20f82f1fd8357578034b5958a0c7c6f21d98cab9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:27:21 +0000 Subject: [PATCH 25/29] chore: update pre-commit hooks (#4231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: update pre-commit hooks updates: - [github.com/astral-sh/ruff-pre-commit: v0.16.0 → v0.16.6](https://github.com/astral-sh/ruff-pre-commit/compare/v0.16.0...v0.16.6) - [github.com/codespell-project/codespell: v2.4.2 → v2.4.3](https://github.com/codespell-project/codespell/compare/v2.4.2...v2.4.3) - [github.com/DavidAnson/markdownlint-cli2: v0.22.1 → v0.23.2](https://github.com/DavidAnson/markdownlint-cli2/compare/v0.22.1...v0.23.2) - [github.com/scientific-python/cookie: 2026.06.18 → 2026.08.14](https://github.com/scientific-python/cookie/compare/2026.06.18...2026.08.14) - [github.com/zizmorcore/zizmor-pre-commit: v1.26.1 → v1.30.0](https://github.com/zizmorcore/zizmor-pre-commit/compare/v1.26.1...v1.30.0) - [github.com/twisted/towncrier: 25.8.0 → 26.9.0](https://github.com/twisted/towncrier/compare/25.8.0...26.9.0) * style: reformat a README example for ruff 0.16.6 ruff-format v0.16.6 formats Python code blocks in Markdown, and normalizes this inline comment to PEP 8's two spaces. Fallout from the hook bump in this pull request, not a behavior change. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 * ci: pin just-version in setup-just steps zizmor v1.30.0 adds the `unpinned-tools` audit, which flags all twelve `extractions/setup-just` steps across the three package workflows: pinning the action by SHA still leaves the action free to install whatever just is newest at run time, so a just release can change CI without a commit here. Pinned to 1.58.0, which is what the action resolves to today, so this changes nothing about the current builds while making that choice explicit. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Davis Bennett Co-authored-by: Claude Opus 5 --- .github/workflows/zarr-http-server.yml | 16 ++++++++++++++++ .github/workflows/zarr-indexing.yml | 16 ++++++++++++++++ .github/workflows/zarr-metadata.yml | 16 ++++++++++++++++ .pre-commit-config.yaml | 12 ++++++------ packages/zarr-http-server/README.md | 2 +- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/.github/workflows/zarr-http-server.yml b/.github/workflows/zarr-http-server.yml index a30d990a33..f3530ebc33 100644 --- a/.github/workflows/zarr-http-server.yml +++ b/.github/workflows/zarr-http-server.yml @@ -54,6 +54,10 @@ jobs: run: uv python install ${{ matrix.python-version }} - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Sync test dependency groups # The examples group carries the deps the README examples need, so the # test that reads a served array back with a zarr client runs here @@ -77,6 +81,10 @@ jobs: uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Run ruff run: just lint @@ -99,6 +107,10 @@ jobs: run: uv python install 3.12 - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Sync test dependency group run: uv sync --group test --python 3.12 - name: Run mypy @@ -121,6 +133,10 @@ jobs: enable-cache: true - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Build docs run: just docs-check diff --git a/.github/workflows/zarr-indexing.yml b/.github/workflows/zarr-indexing.yml index 3b106e16aa..c77c9a9741 100644 --- a/.github/workflows/zarr-indexing.yml +++ b/.github/workflows/zarr-indexing.yml @@ -41,6 +41,10 @@ jobs: enable-cache: true - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} # The suite imports nothing from `zarr`; it runs against the repo-root @@ -70,6 +74,10 @@ jobs: uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Run ruff # The ruff version pin lives in packages/zarr-indexing/justfile. run: just lint @@ -95,6 +103,10 @@ jobs: run: uv sync --group test --python 3.12 - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Run pyright # The pyright invocation lives in packages/zarr-indexing/justfile. run: just typecheck @@ -116,6 +128,10 @@ jobs: enable-cache: true - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Build docs # The strict mkdocs build lives in packages/zarr-indexing/justfile. run: just docs-check diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 069f6d5060..87065e5913 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -44,6 +44,10 @@ jobs: enable-cache: true - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Sync test dependency group @@ -66,6 +70,10 @@ jobs: uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Run ruff run: just lint @@ -86,6 +94,10 @@ jobs: enable-cache: true - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Run pyright # The pyright version and interpreter pins live in the justfile. run: just typecheck @@ -107,6 +119,10 @@ jobs: enable-cache: true - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + # Pin the tool, not just the action: without this the action + # installs whatever just is newest at run time. + just-version: 1.58.0 - name: Build docs run: just docs-check diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54345c819e..be96f1551f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,13 +17,13 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.0 + rev: v0.16.6 hooks: - id: ruff-check args: ["--fix", "--show-fixes"] - id: ruff-format - repo: https://github.com/codespell-project/codespell - rev: v2.4.2 + rev: v2.4.3 hooks: - id: codespell args: ["-L", "fo,ihs,kake,te", "-S", "fixture"] @@ -34,7 +34,7 @@ repos: exclude: mkdocs.yml - id: trailing-whitespace - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.22.1 + rev: v0.23.2 hooks: # Markdown structure/hygiene. Rule selection and ignores are in # .markdownlint-cli2.jsonc; complements ci/lint_docs.py (RST residue, @@ -72,7 +72,7 @@ repos: files: ^packages/zarr-http-server/ stages: [pre-push] - repo: https://github.com/scientific-python/cookie - rev: 2026.06.18 + rev: 2026.08.14 hooks: - id: sp-repo-review - repo: https://github.com/numpy/numpydoc @@ -89,10 +89,10 @@ repos: types: [python] files: ^(src|tests)/ - repo: https://github.com/zizmorcore/zizmor-pre-commit - rev: v1.26.1 + rev: v1.30.0 hooks: - id: zizmor - repo: https://github.com/twisted/towncrier - rev: 25.8.0 + rev: 26.9.0 hooks: - id: towncrier-check diff --git a/packages/zarr-http-server/README.md b/packages/zarr-http-server/README.md index 53df0d2f50..c68f2c9ed2 100644 --- a/packages/zarr-http-server/README.md +++ b/packages/zarr-http-server/README.md @@ -30,7 +30,7 @@ store = zarr.storage.MemoryStore() array = zarr.create_array(store, shape=(100,), chunks=(10,), dtype="float64") with serve_background(node_app(array)) as server: - print(server.url) # e.g. http://127.0.0.1:8000 + print(server.url) # e.g. http://127.0.0.1:8000 ``` Building an app and running it are separate steps, and either app works with From 7014615850650a588a30e55a3f28a499b412eeed Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 11:37:56 +0200 Subject: [PATCH 26/29] ci: install just as a binary in the benchmark job Adding `actions/setup-python` to this job so that `pip install rust-just` had an interpreter also changed which interpreter hatch built `test.py3.12-minimal` on: the runner's own Python was replaced by /opt/hostedtoolcache/Python/3.12.5/arm64. CodSpeed measured the result as a 12.97% degradation across 26 benchmarks, spread -12.76% to -13.99% -- a range of 1.2 points over slice indexing, sharded Morton indexing, three array shapes and two store types. Nothing in this branch touches src/, tests/benchmarks/ or packages/; that uniformity is an interpreter swap, not a regression, and CodSpeed flagged it as "different runtime environments detected". Install just with extractions/setup-just instead, matching the three package workflows. No interpreter is set up, so hatch resolves the same Python the baseline used, and `just-version` is pinned to satisfy zizmor's unpinned-tools audit. The benchmarks need a re-run to produce a comparison against a matching environment; the numbers on the previous run should not be acknowledged as a regression. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .github/workflows/codspeed.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 8e3ecdd82f..03a7c0d9b6 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -27,16 +27,19 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' + # Installed as a binary rather than with `pip install rust-just`, which would + # need an interpreter set up in this job. Benchmarks are only comparable across + # runs if the interpreter underneath them does not move: pinning a Python here + # measured a uniform ~13% slowdown on every benchmark, because hatch then built + # the environment on that interpreter instead of the runner's own. - name: Install just - run: python -m pip install rust-just==1.58.0 + uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + just-version: 1.58.0 - name: Run the benchmarks uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 env: From 1187a43de9d3aa467d92f317f39415bde245928d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:47:33 +0000 Subject: [PATCH 27/29] chore(deps): bump the actions group across 1 directory with 4 updates (#4370) Bumps the actions group with 4 updates in the / directory: [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv), [github-community-projects/issue-metrics](https://github.com/github-community-projects/issue-metrics), [scientific-python/upload-nightly-action](https://github.com/scientific-python/upload-nightly-action) and [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action). Updates `astral-sh/setup-uv` from 10.0.1 to 10.1.0 - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/20cfd1bf945f4377ade1205e4dbc17946fc9a30d...bec219d24cd3e171d82865faccec33120bb574f4) Updates `github-community-projects/issue-metrics` from 5.0.1 to 5.0.2 - [Release notes](https://github.com/github-community-projects/issue-metrics/releases) - [Commits](https://github.com/github-community-projects/issue-metrics/compare/61084fa9599a62c7821f06602e180a42d1c7a205...a7dc2fb675661e208d4fc6fa321a6b95fc4a06f9) Updates `scientific-python/upload-nightly-action` from 0.6.4 to 0.6.5 - [Release notes](https://github.com/scientific-python/upload-nightly-action/releases) - [Commits](https://github.com/scientific-python/upload-nightly-action/compare/e76cfec8a4611fd02808a801b0ff5a7d7c1b2d99...16fa02eacee1655195143de09f03676e60ef2bf5) Updates `zizmorcore/zizmor-action` from 0.6.3 to 0.6.4 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/70fb788f84895a7701f5643d103d587e460b5c99...cc914d7f3750a2d13d75c7f184a1060aa0e9d482) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: 10.1.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: github-community-projects/issue-metrics dependency-version: 5.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: scientific-python/upload-nightly-action dependency-version: 0.6.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/check_changelogs.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/downstream.yml | 4 ++-- .github/workflows/gpu_test.yml | 2 +- .github/workflows/hypothesis.yaml | 2 +- .github/workflows/issue-metrics.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/nightly_wheels.yml | 2 +- .github/workflows/test.yml | 8 ++++---- .github/workflows/zarr-http-server-release.yml | 2 +- .github/workflows/zarr-http-server.yml | 8 ++++---- .github/workflows/zarr-indexing-release.yml | 2 +- .github/workflows/zarr-indexing.yml | 8 ++++---- .github/workflows/zarr-metadata-release.yml | 2 +- .github/workflows/zarr-metadata.yml | 8 ++++---- .github/workflows/zizmor.yml | 2 +- 16 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index f642eb17ca..f0d7ab1471 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -22,7 +22,7 @@ jobs: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Check zarr-python changelog entries run: uv run --no-sync python ci/check_changelog_entries.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0f9c711a60..5e93efe230 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - run: uv sync --group docs # Fast source-level guards that need no built site, so they run before the (slower) # build for a quick failure: every public export is in the API reference, and no diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index eb0cd4a414..76cfaa0aa8 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -45,7 +45,7 @@ jobs: python-version: '3.13' - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install xarray and test dependencies working-directory: xarray @@ -103,7 +103,7 @@ jobs: python-version: '3.13' - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install numcodecs with test-zarr-main group working-directory: numcodecs diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index c902bfdaa8..4c4d49a65c 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -62,7 +62,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 352888f749..ff5382b4d4 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -60,7 +60,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: diff --git a/.github/workflows/issue-metrics.yml b/.github/workflows/issue-metrics.yml index 7b4cd08a0b..507451e04a 100644 --- a/.github/workflows/issue-metrics.yml +++ b/.github/workflows/issue-metrics.yml @@ -33,7 +33,7 @@ jobs: echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" - name: Run issue-metrics tool - uses: github-community-projects/issue-metrics@61084fa9599a62c7821f06602e180a42d1c7a205 # v5.0.1 + uses: github-community-projects/issue-metrics@a7dc2fb675661e208d4fc6fa321a6b95fc4a06f9 # v5.0.2 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 24521aadc5..05354dd503 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,7 +27,7 @@ jobs: with: python-version: "3.12" - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 diff --git a/.github/workflows/nightly_wheels.yml b/.github/workflows/nightly_wheels.yml index 2cb913511e..3d7fc60bb4 100644 --- a/.github/workflows/nightly_wheels.yml +++ b/.github/workflows/nightly_wheels.yml @@ -42,7 +42,7 @@ jobs: run: hatch build - name: Upload nightly wheels - uses: scientific-python/upload-nightly-action@e76cfec8a4611fd02808a801b0ff5a7d7c1b2d99 + uses: scientific-python/upload-nightly-action@16fa02eacee1655195143de09f03676e60ef2bf5 with: artifacts_path: dist anaconda_nightly_upload_token: ${{ secrets.ANACONDA_ORG_UPLOAD_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 650932309c..eb029bd153 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Set Up Hatch Env @@ -115,7 +115,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Set Up Hatch Env @@ -150,7 +150,7 @@ jobs: python-version: '3.13' cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Set Up Hatch Env @@ -174,7 +174,7 @@ jobs: python-version: '3.13' cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Run Benchmarks diff --git a/.github/workflows/zarr-http-server-release.yml b/.github/workflows/zarr-http-server-release.yml index fb5e0fc21f..703cb7fd26 100644 --- a/.github/workflows/zarr-http-server-release.yml +++ b/.github/workflows/zarr-http-server-release.yml @@ -51,7 +51,7 @@ jobs: path: dist - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: false diff --git a/.github/workflows/zarr-http-server.yml b/.github/workflows/zarr-http-server.yml index f3530ebc33..b90528d40f 100644 --- a/.github/workflows/zarr-http-server.yml +++ b/.github/workflows/zarr-http-server.yml @@ -47,7 +47,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - name: Set up Python ${{ matrix.python-version }} @@ -78,7 +78,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 with: @@ -100,7 +100,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - name: Set up Python @@ -128,7 +128,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - name: Install just diff --git a/.github/workflows/zarr-indexing-release.yml b/.github/workflows/zarr-indexing-release.yml index 5d95eb1996..2ae363294d 100644 --- a/.github/workflows/zarr-indexing-release.yml +++ b/.github/workflows/zarr-indexing-release.yml @@ -51,7 +51,7 @@ jobs: path: dist - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: false diff --git a/.github/workflows/zarr-indexing.yml b/.github/workflows/zarr-indexing.yml index c77c9a9741..5d74c6a5e9 100644 --- a/.github/workflows/zarr-indexing.yml +++ b/.github/workflows/zarr-indexing.yml @@ -36,7 +36,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - name: Install just @@ -71,7 +71,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 with: @@ -94,7 +94,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - name: Set up Python @@ -123,7 +123,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - name: Install just diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index 3125d2529a..d3409a1176 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -51,7 +51,7 @@ jobs: path: dist - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: false diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 87065e5913..521a4f270c 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -39,7 +39,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - name: Install just @@ -67,7 +67,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 with: @@ -89,7 +89,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - name: Install just @@ -114,7 +114,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - name: Install just diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index eb9a68d07d..19bdb461ea 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -32,4 +32,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@70fb788f84895a7701f5643d103d587e460b5c99 # v0.6.3 + uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 From ea821011fcc40d123fd70a4faec657afdd9dcef2 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 16:14:14 +0200 Subject: [PATCH 28/29] ci: install just from official release binaries; lock the docs toolchain Follow-ups from an adversarial audit of this branch. Supply chain. `rust-just` is a third-party repackaging of just on PyPI (Repository: github.com/gnpaone/rust-just) with no attestation on any of its 17 wheels, and this branch had introduced it in ten places including the self-hosted GPU runner and Read the Docs. All 23 just installs across the repo now come from casey/just's own release binaries via extractions/setup-crate@7577c1bd (v2.0.1), a node action whose SHA pins every line of executed code -- rather than extractions/setup-just, a composite wrapper that astral-sh/python-build-standalone#771 moved off for that reason. This also retires the 12 pre-existing setup-just call sites, so the repo has one mechanism instead of four. setup-crate does not verify checksums (extractions/setup-just#20 is open); Read the Docs, where no action is available, fetches the release tarball and verifies it against the SHA256SUMS casey/just publishes. Lockfile. docs.yml had stopped resolving from uv.lock: `uv sync --group docs` became `hatch run docs:`, and hatch never reads the lockfile, so the entire docs toolchain floated -- including numcodecs[msgpack], which carries no version constraint. Dependabot's uv ecosystem exists to keep that lock fresh and no job exercised it any more. The eight docs and changelog recipes now run `uv run --frozen --group docs`, so single-version tooling comes from the lock (as mypy already did) while hatch keeps the test environments, which exist per interpreter and per dependency set and cannot live in one lockfile. The hatch docs environment is removed as unused, and neither docs.yml nor Read the Docs installs hatch at all now. Argument forwarding. The three package justfiles splatted `{{ args }}` unquoted, so `just zarr-metadata test -k 'a and b'` reached pytest as `-k a` plus two stray paths, and command substitution in an argument executed. They now use `set positional-arguments` and `"$@"` like the root, which is what the contributing guide already claimed. Also, regressions from my own earlier commits on this branch: `just lint` had dropped `--show-diff-on-failure --color=always`, which j178/prek-action passed by default, so a hook that rewrote a file failed with no diff; a four-line comment above `hooks-install` displaced its `just --list` description, since just reads only the last comment line; the prek cache key did not cover prek_version and had a restore-keys fallback that could reuse a store built by a different prek; `just just-check` ran before `just lint`, so a cosmetic formatting nit hid every real lint result; and the `packages/*/justfile` glob lacked nullglob, which breaks in the sdist, where /Justfile ships but /packages deliberately does not. Docs: the install instructions used `pip install`, which fails on an externally-managed interpreter; `just just-check` was an undocumented required check; two sentences still told contributors to activate an environment that no longer exists; and the changelog fragment did not mention that the Hatch script tables were removed. Adds a .gitattributes rule so a Windows checkout does not get a CRLF Justfile that `just --fmt --check` rejects on every line. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .gitattributes | 6 +++++ .github/workflows/check_changelogs.yml | 5 +++- .github/workflows/codspeed.yml | 14 +++++----- .github/workflows/docs.yml | 7 +++-- .github/workflows/gpu_test.yml | 5 +++- .github/workflows/hypothesis.yaml | 5 +++- .github/workflows/lint.yml | 19 ++++++++----- .github/workflows/test.yml | 28 ++++++++++++++++--- .github/workflows/zarr-http-server.yml | 28 +++++++++---------- .github/workflows/zarr-indexing.yml | 28 +++++++++---------- .github/workflows/zarr-metadata.yml | 28 +++++++++---------- .readthedocs.yaml | 11 +++++++- Justfile | 37 +++++++++++++++----------- changes/4372.misc.md | 2 +- docs/contributing.md | 18 +++++++++---- packages/zarr-http-server/justfile | 5 +++- packages/zarr-indexing/justfile | 7 +++-- packages/zarr-metadata/justfile | 5 +++- pyproject.toml | 8 ------ 19 files changed, 162 insertions(+), 104 deletions(-) diff --git a/.gitattributes b/.gitattributes index 57eb8a8807..f507ce96ca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,9 @@ *.py linguist-language=python *.ipynb linguist-documentation .git_archival.txt export-subst + +# just refuses to format a CRLF file, so `just just-check` would fail on a Windows +# checkout with core.autocrlf=true before the contributor has changed anything. +Justfile text eol=lf +*/justfile text eol=lf +packages/*/justfile text eol=lf diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index 740c0ae346..960451cc4c 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -25,7 +25,10 @@ jobs: uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - run: uv tool install rust-just==1.58.0 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Check zarr-python changelog entries run: just check-changelogs diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 03a7c0d9b6..a58ce40a9d 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -31,15 +31,15 @@ jobs: uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' - # Installed as a binary rather than with `pip install rust-just`, which would - # need an interpreter set up in this job. Benchmarks are only comparable across - # runs if the interpreter underneath them does not move: pinning a Python here - # measured a uniform ~13% slowdown on every benchmark, because hatch then built - # the environment on that interpreter instead of the runner's own. + # No interpreter is set up in this job on purpose. Benchmarks are only comparable + # across runs if the interpreter underneath them does not move: adding a pinned + # Python here measured a uniform ~13% slowdown on every benchmark, because hatch + # then built the environment on that interpreter instead of the runner's own. - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run the benchmarks uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 env: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dc30c7f210..990866a109 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,8 +23,11 @@ jobs: with: persist-credentials: false - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - - run: uv tool install hatch==1.16.5 - - run: uv tool install rust-just==1.58.0 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 # Fast source-level guards that need no built site, so they run before the (slower) # build for a quick failure: every public export is in the API reference, and no # docstring/Markdown carries reStructuredText markup that MkDocs won't render. diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 4432d512fd..a8e989f580 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -68,7 +68,10 @@ jobs: with: version: '1.16.5' - name: Install just - run: python -m pip install rust-just==1.58.0 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 # Two names for the same environment on purpose: `just setup` builds whatever # HATCH_ENV points at, while `just gpu` reads GPU_HATCH_ENV so that an # ambient HATCH_ENV can never redirect `pytest -m gpu` into a CPU environment. diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 5293557b11..3be6f3ba60 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -66,7 +66,10 @@ jobs: with: version: '1.16.5' - name: Install just - run: python -m pip install rust-just==1.58.0 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ffd0ae4e87..f47fbcac11 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -31,7 +31,10 @@ jobs: with: enable-cache: true - name: Install just - run: uv tool install rust-just==1.58.0 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 # `uvx prek` builds each hook's environment from scratch, so cache them the # way the prek action used to. Keyed on the hook config: a new pinned rev or # a new hook is exactly when the cached environments stop being valid. @@ -39,10 +42,14 @@ jobs: uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/prek - key: prek-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} - restore-keys: | - prek-${{ runner.os }}- - - name: Check justfile formatting - run: just just-check + # Justfile is hashed too: prek_version lives there, and a bumped prek + # must not restore a store built by the previous one. No restore-keys + # fallback for the same reason. + key: prek-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml', 'Justfile') }} - name: Lint run: just lint + - name: Check justfile formatting + # After the linters, and never masking them: this is cosmetic, and a + # mis-formatted recipe should not cost someone their ruff/mypy results. + if: always() + run: just just-check diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 566cc79927..0d7d396fff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch - run: python -m pip install hatch==1.16.5 rust-just==1.58.0 + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} @@ -116,7 +121,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch - run: python -m pip install hatch==1.16.5 rust-just==1.58.0 + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: ${{ matrix.dependency-set }} @@ -150,7 +160,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch - run: python -m pip install hatch==1.16.5 rust-just==1.58.0 + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env run: | HATCH_ENV=doctest just setup @@ -174,7 +189,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch - run: python -m pip install hatch==1.16.5 rust-just==1.58.0 + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Run Benchmarks env: ZARR_BENCHMARK_CLEAR_CACHE: '1' diff --git a/.github/workflows/zarr-http-server.yml b/.github/workflows/zarr-http-server.yml index b90528d40f..4a8fcca101 100644 --- a/.github/workflows/zarr-http-server.yml +++ b/.github/workflows/zarr-http-server.yml @@ -53,11 +53,10 @@ jobs: - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Sync test dependency groups # The examples group carries the deps the README examples need, so the # test that reads a served array back with a zarr client runs here @@ -80,11 +79,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run ruff run: just lint @@ -106,11 +104,10 @@ jobs: - name: Set up Python run: uv python install 3.12 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Sync test dependency group run: uv sync --group test --python 3.12 - name: Run mypy @@ -132,11 +129,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Build docs run: just docs-check diff --git a/.github/workflows/zarr-indexing.yml b/.github/workflows/zarr-indexing.yml index 5d74c6a5e9..9af8ed1e3d 100644 --- a/.github/workflows/zarr-indexing.yml +++ b/.github/workflows/zarr-indexing.yml @@ -40,11 +40,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} # The suite imports nothing from `zarr`; it runs against the repo-root @@ -73,11 +72,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run ruff # The ruff version pin lives in packages/zarr-indexing/justfile. run: just lint @@ -102,11 +100,10 @@ jobs: - name: Sync test dependency group run: uv sync --group test --python 3.12 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run pyright # The pyright invocation lives in packages/zarr-indexing/justfile. run: just typecheck @@ -127,11 +124,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Build docs # The strict mkdocs build lives in packages/zarr-indexing/justfile. run: just docs-check diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 521a4f270c..3c42f4810f 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -43,11 +43,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Sync test dependency group @@ -69,11 +68,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run ruff run: just lint @@ -93,11 +91,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run pyright # The pyright version and interpreter pins live in the justfile. run: just typecheck @@ -118,11 +115,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Build docs run: just docs-check diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3190c7180b..872eb2be80 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,7 +19,16 @@ build: fi install: - pip install --upgrade pip - - pip install hatch==1.16.5 rust-just==1.58.0 + # The docs recipes resolve their toolchain from uv.lock, so uv is all that is + # needed here besides just itself. + - pip install uv==0.12.9 + # No GitHub Action is available on Read the Docs, so fetch the official just + # release and verify it against the checksum casey/just publishes. Installed + # into the build virtualenv's bin, which is already on PATH. + - | + curl -fsSL -o /tmp/just.tar.gz https://github.com/casey/just/releases/download/1.58.0/just-1.58.0-x86_64-unknown-linux-musl.tar.gz + echo "4a5cc2f53e6f0f8c59092a6cc38291eb729d46a7dd95d3ae582008881b84931d /tmp/just.tar.gz" | sha256sum -c - + tar -xzf /tmp/just.tar.gz -C "$READTHEDOCS_VIRTUALENV_PATH/bin" just pre_build: - | if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ]; diff --git a/Justfile b/Justfile index 641671e6ac..6e56644207 100644 --- a/Justfile +++ b/Justfile @@ -1,5 +1,5 @@ # Development and CI verbs live here; Hatch owns Python environments in pyproject.toml. -# Install: pip install hatch==1.16.5 rust-just==1.58.0 uv +# Install: uv tool install hatch==1.16.5 && uv tool install rust-just==1.58.0 # Select test dependencies/interpreter: HATCH_ENV=test.py3.13-minimal just test # On Windows, use Git Bash (the same shell used by the test workflow). set shell := ["bash", "-eu", "-o", "pipefail", "-c"] @@ -13,6 +13,13 @@ hatch_env := env("HATCH_ENV", "test.py3.12-optional") gpu_env := env("GPU_HATCH_ENV", "gputest.py3.12") # Pinned so a prek release cannot change what CI lints without a commit here. prek_version := "0.5.3" +# Documentation and changelog tooling resolves from uv.lock, not a hatch environment. +# There is only ever one docs toolchain, so it can be locked and hash-verified, and +# dependabot's uv ecosystem keeps it current. Hatch still owns the test environments, +# which exist per interpreter and per dependency set and cannot live in one lockfile. +docs_run := "uv run --frozen --group docs" +# The hatch docs environment used to set these; they belong with the mkdocs calls now. +mkdocs_env := "DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true" # List available recipes default: @@ -71,39 +78,39 @@ gpu *args: # Build documentation (warnings are errors) docs-build *args: - hatch run docs:mkdocs build --strict "$@" + {{ mkdocs_env }} {{ docs_run }} mkdocs build --strict "$@" # Serve documentation with live reload docs-serve *args: - hatch run docs:mkdocs serve --watch src "$@" + {{ mkdocs_env }} {{ docs_run }} mkdocs serve --watch src "$@" # Check that every public export has API documentation check-doc-exports *args: - hatch run docs:python ci/check_documented_exports.py docs/api "$@" + {{ docs_run }} python ci/check_documented_exports.py docs/api "$@" # Check documentation source conventions lint-docs *args: - hatch run docs:python ci/lint_docs.py "$@" + {{ docs_run }} python ci/lint_docs.py "$@" # Report unlinked types in built documentation check-doc-links *args: - hatch run docs:python ci/check_unlinked_types.py "$@" + {{ docs_run }} python ci/check_unlinked_types.py "$@" # Run source documentation checks followed by a strict build docs-check: check-doc-exports lint-docs docs-build # Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...) lint *args: - uvx prek@{{ prek_version }} run --all-files "$@" + uvx prek@{{ prek_version }} run --show-diff-on-failure --color=always --all-files "$@" # Run hooks with a custom selection, e.g. just hooks run --last-commit hooks +args: uvx prek@{{ prek_version }} "$@" -# Install local pre-commit hooks. prek is installed as a persistent uv tool, not run -# through uvx: the hook shim prek writes into .git/hooks hard-codes the binary path -# it was installed from and falls back to `prek` on PATH, and a uvx archive path -# stops existing at the next `uv cache prune`. +# prek is installed as a persistent uv tool rather than run through uvx: the hook shim +# prek writes into .git/hooks hard-codes the binary path it was installed from and falls +# back to `prek` on PATH, and a uvx archive path stops existing at the next cache prune. +# Install local pre-commit hooks hooks-install: uv tool install prek=={{ prek_version }} prek install @@ -126,15 +133,15 @@ build *args: # Create a changelog fragment (interactive without arguments) changelog *args: - hatch run docs:towncrier create "$@" + {{ docs_run }} towncrier create "$@" # Preview the next release's changelog changelog-draft *args: - hatch run docs:towncrier build --draft --version Unreleased "$@" + {{ docs_run }} towncrier build --draft --version Unreleased "$@" # Build release notes; pass --version and --yes when preparing a release changelog-build *args: - hatch run docs:towncrier build "$@" + {{ docs_run }} towncrier build "$@" # Check changelog filenames (default: changes/; accepts a package changes directory) check-changelogs *args: @@ -143,7 +150,7 @@ check-changelogs *args: # Check recipe formatting of the root Justfile and every package justfile just-check: just --fmt --check - for f in packages/*/justfile; do just --justfile "$f" --fmt --check; done + shopt -s nullglob; for f in packages/*/justfile; do just --justfile "$f" --fmt --check; done # Run a zarr-metadata recipe, or list its recipes with no arguments zarr-metadata *args: diff --git a/changes/4372.misc.md b/changes/4372.misc.md index 7b55499cf1..55a800a3f3 100644 --- a/changes/4372.misc.md +++ b/changes/4372.misc.md @@ -1 +1 @@ -Define development and CI commands in a root Justfile, with Hatch managing Python environments. Extracted from [#4096](https://github.com/zarr-developers/zarr-python/pull/4096). +Define development and CI commands in a root Justfile, with Hatch managing Python environments and documentation tooling resolving from `uv.lock`. The Hatch script tables are removed, so invocations like `hatch env run --env test.py3.12-optional run-coverage` become `just coverage`; see the contributing guide for the full set. Extracted from [#4096](https://github.com/zarr-developers/zarr-python/pull/4096). diff --git a/docs/contributing.md b/docs/contributing.md index 5c3cfbe8b0..45e401e064 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -82,10 +82,14 @@ git remote add upstream git@github.com:zarr-developers/zarr-python.git The root `Justfile` defines development and CI commands. [just](https://just.systems/) runs these commands, while [Hatch](https://hatch.pypa.io/latest/index.html) manages -the Python environments declared in `pyproject.toml`. Install the task tools and uv (several recipes run tooling through it): +the Python environments declared in `pyproject.toml`. Install +[uv](https://docs.astral.sh/uv/getting-started/installation/) first, then the two task +tools. `uv tool install` puts them in their own environments, which a plain `pip install` +cannot do on a Python that marks itself externally managed (Debian, Ubuntu, Homebrew): ```bash -pip install hatch==1.16.5 rust-just==1.58.0 uv +uv tool install hatch==1.16.5 +uv tool install rust-just==1.58.0 # or any option from https://just.systems/man/en/packages.html just # list available commands just envs # list Python environments just setup # create the default test environment @@ -95,7 +99,8 @@ just test Test recipes default to `test.py3.12-optional`. Set `HATCH_ENV` to select a different interpreter or dependency set, just as CI does. `just gpu` reads `GPU_HATCH_ENV` instead, so an exported `HATCH_ENV` cannot silently send GPU tests to an -environment built without the `gpu` feature. On Windows, run these commands in +environment built without the `gpu` feature, and `just doctest` always runs in the +`doctest` environment. On Windows, run these commands in Git Bash. ```bash @@ -153,7 +158,7 @@ Again, any conflicts need to be resolved before submitting a pull request. ### Running the test suite -Zarr includes a suite of unit tests. The simplest way to run the unit tests is to activate your development environment (see [creating a development environment](#creating-a-development-environment) above) and invoke: +Zarr includes a suite of unit tests. The simplest way to run the unit tests is to invoke: ```bash just test @@ -340,7 +345,7 @@ Sometimes, you may want the documentation to build quicker. You can disable code ### Changelog -zarr-python uses [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) to manage release notes. Most pull requests should include at least one news fragment describing the changes. To add a release note, you'll need the GitHub issue or pull request number and the type of your change (`feature`, `bugfix`, `doc`, `removal`, `misc`). With that, run `just changelog` with your development environment, which will prompt you for the issue number, change type, and the news text: +zarr-python uses [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) to manage release notes. Most pull requests should include at least one news fragment describing the changes. To add a release note, you'll need the GitHub issue or pull request number and the type of your change (`feature`, `bugfix`, `doc`, `removal`, `misc`). With that, run `just changelog`, which will prompt you for the issue number, change type, and the news text: ```bash just changelog @@ -464,6 +469,9 @@ The benchmarks are run as part of the continuous integration suite through [cods ## Building distributions and maintaining dependencies +`just just-check` verifies that the root `Justfile` and each package `justfile` are +formatted the way CI expects; `just --fmt` rewrites them in place if it complains. + Run `just build` to produce a source distribution and wheel in `dist/`. Use `just lock-check` to check the dependency lockfile, or `just lock` to update it. `just typecheck` runs the type checker independently of the other lint hooks. diff --git a/packages/zarr-http-server/justfile b/packages/zarr-http-server/justfile index d65b1b53b7..5daac00ebc 100644 --- a/packages/zarr-http-server/justfile +++ b/packages/zarr-http-server/justfile @@ -10,6 +10,9 @@ # newest release: when ruff 0.16 began selecting BLE001 under the root # config's `B` prefix, this job failed on rules the pinned ruff never enforced, # with no code change to blame. Bump alongside the pre-commit rev. +# Quoted arguments must survive delegation from the root Justfile. +set positional-arguments + ruff_version := "0.16.0" # List available recipes @@ -21,7 +24,7 @@ default: # silently skipping. # Run the test suite; extra args are passed to pytest test *args: - uv run --group test --group examples pytest tests {{ args }} + uv run --group test --group examples pytest tests "$@" # Lint the package sources and tests lint: diff --git a/packages/zarr-indexing/justfile b/packages/zarr-indexing/justfile index e32f177ff0..29f6ce885f 100644 --- a/packages/zarr-indexing/justfile +++ b/packages/zarr-indexing/justfile @@ -2,6 +2,9 @@ # directory as the working directory regardless of where `just` is invoked. # List available recipes +# Quoted arguments must survive delegation from the root Justfile. +set positional-arguments + default: @just --list @@ -10,7 +13,7 @@ default: # overlay, using the same test invocation as CI. # Run the test suite; extra args are passed to pytest test *args: - uv run --project ../.. --group test --with-editable . python -m pytest tests src/zarr_indexing {{ args }} + uv run --project ../.. --group test --with-editable . python -m pytest tests src/zarr_indexing "$@" # TensorStore is the oracle for the parity suites, which skip without it. It # ships binary wheels only, so it rides in as a run-time overlay rather than @@ -18,7 +21,7 @@ test *args: # gate the CI job that calls this on the matrix version. # Run the tensorstore parity suites; extra args are passed to pytest test-tensorstore *args: - uv run --project ../.. --group test --with-editable . --with 'tensorstore>=0.1.84' python -m pytest tests/test_ndsel_tensorstore.py tests/test_tensorstore_parity.py {{ args }} + uv run --project ../.. --group test --with-editable . --with 'tensorstore>=0.1.84' python -m pytest tests/test_ndsel_tensorstore.py tests/test_tensorstore_parity.py "$@" # Lint with the same invocation CI uses. Ruff is pinned to the repo-wide # version in the root .pre-commit-config.yaml; bump together. diff --git a/packages/zarr-metadata/justfile b/packages/zarr-metadata/justfile index 0f1861ed7d..843286b89f 100644 --- a/packages/zarr-metadata/justfile +++ b/packages/zarr-metadata/justfile @@ -2,12 +2,15 @@ # directory as the working directory regardless of where `just` is invoked. # List available recipes +# Quoted arguments must survive delegation from the root Justfile. +set positional-arguments + default: @just --list # Run the test suite; extra args are passed to pytest test *args: - uv run --group test pytest tests {{ args }} + uv run --group test pytest tests "$@" # Lint with the same invocation CI uses lint: diff --git a/pyproject.toml b/pyproject.toml index 75802b0a06..a4dafe4d6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -265,14 +265,6 @@ extra-dependencies = [ [tool.hatch.envs.default] installer = "uv" -[tool.hatch.envs.docs] -features = ['remote'] -dependency-groups = ['docs'] - -[tool.hatch.envs.docs.env-vars] -DISABLE_MKDOCS_2_WARNING = "true" -NO_MKDOCS_2_WARNING = "true" - [tool.hatch.envs.doctest] description = "Test environment for validating executable code blocks in documentation" features = ['remote'] From ae41d6eb3354ff4364c8f04ab1a0dccbf0d0f5b1 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 20 Sep 2026 10:46:01 +0200 Subject: [PATCH 29/29] fix(zarr-metadata)!: make entity types assignable to metadata fields; validators return tuples (#4378) * fix(zarr-metadata)!: make entity types assignable to metadata fields; validators return tuples `ZarrV3NamedConfigJSON.name` and `.configuration` are `ReadOnly` and the envelope is `closed`, so the concrete codec / chunk-grid / chunk-key-encoding / data-type TypedDicts are assignable to the fields they describe. Every concrete `*Object` / `*Configuration` is `closed` and object forms declare `must_understand: NotRequired[bool]`. Every `validate_*` in `zarr_metadata.model` returns `tuple[ValidationProblem, ...]`, `MetadataValidationError.problems` is a tuple, and `load_store_json` returns `object` rather than `Any`. `ANN401` is enforced package-wide. `ZarrV2ConsolidatedMetadataJSON.zarr_consolidated_format` is `Literal[1]`. Split from d-v-b/zarr-python#296 (part 1 of 3). Assisted-by: ClaudeCode:claude-fable-5-1 * chore(zarr-metadata): number changelog fragments for #317 Assisted-by: ClaudeCode:claude-fable-5-1 * test(zarr-metadata): expect tuples from validate_array_metadata_v2 in #4365's tests Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 * chore(zarr-metadata): number changelog fragments for #4378 Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- packages/zarr-metadata/changes/4378.bugfix.md | 25 +++ packages/zarr-metadata/changes/4378.misc.1.md | 5 + packages/zarr-metadata/changes/4378.misc.md | 6 + packages/zarr-metadata/pyproject.toml | 8 + .../src/zarr_metadata/_common.py | 52 +++--- .../src/zarr_metadata/model/_array.py | 16 +- .../src/zarr_metadata/model/_group.py | 8 +- .../src/zarr_metadata/model/_validation.py | 160 ++++++++++-------- .../src/zarr_metadata/pydantic.py | 25 +-- .../src/zarr_metadata/v2/consolidated.py | 2 +- .../v3/chunk_grid/rectilinear.py | 7 +- .../zarr_metadata/v3/chunk_grid/regular.py | 7 +- .../v3/chunk_key_encoding/default.py | 5 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 5 +- .../src/zarr_metadata/v3/codec/blosc.py | 5 +- .../src/zarr_metadata/v3/codec/bytes.py | 5 +- .../src/zarr_metadata/v3/codec/cast_value.py | 7 +- .../src/zarr_metadata/v3/codec/crc32c.py | 3 +- .../src/zarr_metadata/v3/codec/gzip.py | 7 +- .../zarr_metadata/v3/codec/scale_offset.py | 5 +- .../v3/codec/sharding_indexed.py | 5 +- .../src/zarr_metadata/v3/codec/transpose.py | 7 +- .../src/zarr_metadata/v3/codec/zstd.py | 5 +- .../src/zarr_metadata/v3/consolidated.py | 2 +- .../v3/data_type/numpy_datetime64.py | 7 +- .../v3/data_type/numpy_timedelta64.py | 7 +- .../src/zarr_metadata/v3/data_type/struct.py | 9 +- .../zarr-metadata/tests/model/test_array.py | 44 ++--- .../zarr-metadata/tests/model/test_group.py | 4 +- .../tests/test_partial_equivalence.py | 4 +- 30 files changed, 267 insertions(+), 190 deletions(-) create mode 100644 packages/zarr-metadata/changes/4378.bugfix.md create mode 100644 packages/zarr-metadata/changes/4378.misc.1.md create mode 100644 packages/zarr-metadata/changes/4378.misc.md diff --git a/packages/zarr-metadata/changes/4378.bugfix.md b/packages/zarr-metadata/changes/4378.bugfix.md new file mode 100644 index 0000000000..cbe3a1a7ed --- /dev/null +++ b/packages/zarr-metadata/changes/4378.bugfix.md @@ -0,0 +1,25 @@ +Concrete v3 entity types are now assignable to the fields they describe. +Previously, none of the package's canonical codec / chunk-grid / +chunk-key-encoding / data-type types (e.g. `BloscCodecMetadata`, +`RegularChunkGridMetadata`) satisfied `ZarrV3MetadataFieldJSON`, so a +type checker rejected putting them into the very fields they document +(`codecs`, `chunk_grid`, `data_type`, ...). Three changes fix this: + +- `ZarrV3NamedConfigJSON.name` and `.configuration` are now `ReadOnly` + (PEP 705), making them covariant so concrete `name: Literal[...]` and + required-`configuration` shapes are accepted. +- `ZarrV3NamedConfigJSON` is now `closed` (PEP 728): the spec's + named-configuration envelope has exactly `name` / `configuration` / + `must_understand`, and closing the type also makes it usable as a + `JSONValue` (needed for e.g. the `sharding_indexed` inner `codecs`). +- Every concrete `*Object` / `*Configuration` TypedDict is now `closed`, + and object forms declare `must_understand: NotRequired[bool]` (any v3 + metadata field may carry the extension member). + +**Soft-breaking** for type-checking consumers: dicts with keys beyond the +declared shape no longer satisfy the closed types, and `name` / +`configuration` can no longer be mutated through `ZarrV3NamedConfigJSON`. +Both were previously accepted by type checkers but produced documents +outside the spec's shapes. `zarr_metadata.pydantic` serializers now +declare their return schema via the pydantic-facing shadow types, so +pydantic schema generation stays warning-free. diff --git a/packages/zarr-metadata/changes/4378.misc.1.md b/packages/zarr-metadata/changes/4378.misc.1.md new file mode 100644 index 0000000000..5f5eb1f238 --- /dev/null +++ b/packages/zarr-metadata/changes/4378.misc.1.md @@ -0,0 +1,5 @@ +`ZarrV2ConsolidatedMetadataJSON.zarr_consolidated_format` is typed +`Literal[1]` rather than `int`. Format 1 is the only defined `.zmetadata` +format and the runtime validator already rejected anything else, so the +type now carries the constraint the validator enforces instead of +contradicting it. diff --git a/packages/zarr-metadata/changes/4378.misc.md b/packages/zarr-metadata/changes/4378.misc.md new file mode 100644 index 0000000000..ed9167064f --- /dev/null +++ b/packages/zarr-metadata/changes/4378.misc.md @@ -0,0 +1,6 @@ +**Breaking:** every `validate_*` function in `zarr_metadata.model` now +returns `tuple[ValidationProblem, ...]` instead of `list[ValidationProblem]`, +and `MetadataValidationError.problems` is a tuple. Iteration and indexing are +unchanged; callers that mutate reports must first copy them with +`list(problems)`. `MetadataValidationError` still accepts any problem +sequence. diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 8900f96dc5..6e7b0e4f52 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -95,6 +95,14 @@ include = [ extend = "../../pyproject.toml" target-version = "py311" +[tool.ruff.lint] +# `Any` defeats the point of a package whose product is precise types, so it +# is banned in annotations here rather than merely discouraged. Use `object` +# for "any value" (the caller must narrow) and the document TypedDicts where +# the shape is known; a genuinely dynamic annotation needs an explicit noqa +# saying why. +extend-select = ["ANN401"] + [tool.pytest.ini_options] minversion = "7" testpaths = ["tests"] diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 08c143107f..0751d56563 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -6,14 +6,21 @@ `zarr_metadata.v3.data_type`. """ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from typing import NotRequired -from typing_extensions import TypeAliasType, TypedDict +from typing_extensions import ReadOnly, TypeAliasType, TypedDict JSONValue = TypeAliasType( "JSONValue", - int | float | bool | str | Sequence["JSONValue"] | Mapping[str, "JSONValue"] | None, + int + | float + | bool + | str + | list["JSONValue"] + | tuple["JSONValue", ...] + | Mapping[str, "JSONValue"] + | None, ) """A recursive type alias for JSON-encodable values. @@ -21,30 +28,37 @@ self-reference is a named recursion point that pydantic can resolve when building a `TypeAdapter`; a bare recursive `TypeAlias` raises `PydanticUserError`/`RecursionError` at validation time. - -The array arm is the covariant `Sequence` rather than the invariant -`list["JSONValue"] | tuple["JSONValue", ...]`, so values typed with a -*narrower* element type still count as JSON values: a `list[str]` field on a -TypedDict is assignable to `JSONValue` under `Sequence` but not under -`list[JSONValue]` (`list` is invariant in its element type, and pyright's -diagnostic for that failure suggests exactly this change). This is what lets -downstream TypedDicts give their fields precise types (`Sequence[str]`, -`list[int]`, ...) while remaining assignable to `Mapping[str, JSONValue]`. -The type-level cost, accepted deliberately: `Sequence` says nothing about the -concrete container, and it admits `str`/`bytes` (`str` was already a union -arm); runtime code narrowing a JSON array must exclude `str`/`bytes`/ -`bytearray` regardless of how this alias is spelled. """ -class ZarrV3NamedConfigJSON(TypedDict): +class ZarrV3NamedConfigJSON(TypedDict, closed=True): """ Externally-tagged union member for a metadata field. The optional `configuration` mapping holds arbitrary JSON-encodable values. `must_understand` is implicitly true when absent. + + `name` and `configuration` are `ReadOnly` (PEP 705) so that concrete + entity types — `BloscCodecObject`, `RegularChunkGridObject`, and the + rest — are assignable to this type, and therefore to + `ZarrV3MetadataFieldJSON`. Without `ReadOnly` both items are invariant, + so a concrete `name: Literal["blosc"]` does not satisfy `name: str`, and + a required `configuration` does not satisfy a `NotRequired` one. That + made the package's own codec types unusable in the very fields they + describe (`codecs`, `data_type`, `chunk_grid`, ...), and made + `TypeIs`-based codec classification impossible to declare, since `TypeIs` + requires the narrowed type to be assignable to the input type. + + `must_understand` stays writable: nothing needs to narrow it, and + keeping it mutable lets writers set it on an already-constructed field. + + The type is `closed` (PEP 728): the spec's named-configuration envelope + has exactly these three members, and closing it is also what makes this + type — and every concrete entity type embedding it, e.g. the + `sharding_indexed` configuration's inner `codecs` list — assignable to + `Mapping[str, JSONValue]` (i.e. usable as a `JSONValue`). """ - name: str - configuration: NotRequired[Mapping[str, JSONValue]] + name: ReadOnly[str] + configuration: NotRequired[ReadOnly[Mapping[str, JSONValue]]] must_understand: NotRequired[bool] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index db708c0152..26a69a304a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -61,10 +61,14 @@ class ZarrV3NamedConfig: def to_json(self) -> ZarrV3MetadataFieldJSON: if not self.configuration and self.must_understand: return self.name - out: ZarrV3NamedConfigJSON = {"name": self.name} - if self.configuration: - # to_json output shares no mutable state with the model. - out["configuration"] = copy.deepcopy(self.configuration) + # `configuration` is ReadOnly, so it is set in the literal rather than + # assigned afterwards. to_json output shares no mutable state with the + # model. + out: ZarrV3NamedConfigJSON = ( + {"name": self.name, "configuration": copy.deepcopy(self.configuration)} + if self.configuration + else {"name": self.name} + ) if not self.must_understand: out["must_understand"] = False return out @@ -465,7 +469,7 @@ def from_json(cls, data: object) -> ZarrV2ArrayMetadata: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: - zarray_raw = cast("object", load_store_json(mapping, ZARR_V2_ARRAY_METADATA_STORE_KEY)) + zarray_raw = load_store_json(mapping, ZARR_V2_ARRAY_METADATA_STORE_KEY) if not isinstance(zarray_raw, Mapping): return cls.from_json(zarray_raw) zarray = cast("Mapping[str, object]", zarray_raw) @@ -480,7 +484,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: ] ) if ZARR_V2_ATTRIBUTES_STORE_KEY in mapping: - zattrs = cast("object", load_store_json(mapping, ZARR_V2_ATTRIBUTES_STORE_KEY)) + zattrs = load_store_json(mapping, ZARR_V2_ATTRIBUTES_STORE_KEY) return cls.from_json({**zarray, "attributes": zattrs}) return cls.from_json(zarray) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index bb2e14da1b..5519e6fbb9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -226,7 +226,7 @@ def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: def from_json(cls, data: object) -> ZarrV3ConsolidatedMetadata: normalized = arrays_to_tuples(data) problems = validate_consolidated_metadata_v3(normalized) - if problems: + if len(problems) != 0: raise MetadataValidationError(problems) env = cast("Mapping[str, object]", normalized) entries: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] = {} @@ -315,7 +315,7 @@ def from_json(cls, data: object) -> ZarrV2GroupMetadata: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: - zgroup_raw = cast("object", load_store_json(mapping, ZARR_V2_GROUP_METADATA_STORE_KEY)) + zgroup_raw = load_store_json(mapping, ZARR_V2_GROUP_METADATA_STORE_KEY) if not isinstance(zgroup_raw, Mapping): return cls.from_json(zgroup_raw) zgroup = cast("Mapping[str, object]", zgroup_raw) @@ -330,7 +330,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: ] ) if ZARR_V2_ATTRIBUTES_STORE_KEY in mapping: - zattrs = cast("object", load_store_json(mapping, ZARR_V2_ATTRIBUTES_STORE_KEY)) + zattrs = load_store_json(mapping, ZARR_V2_ATTRIBUTES_STORE_KEY) return cls.from_json({**zgroup, "attributes": zattrs}) return cls.from_json(zgroup) @@ -417,7 +417,7 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: ) for problem in validate_json(value) ) - if problems: + if len(problems) != 0: raise MetadataValidationError(problems) entries_tupled = cast( "dict[str, JSONValue]", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index d1cd74b719..f927851e4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -17,7 +17,7 @@ import math from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Any, Final, Literal, NoReturn, cast +from typing import Final, Literal, NoReturn, cast from typing_extensions import TypeIs @@ -62,27 +62,33 @@ def __str__(self) -> str: class MetadataValidationError(ValueError): """Raised when a value fails structural metadata validation. - Carries every problem found (not just the first) in `.problems`. + Carries every problem found (not just the first) in `.problems`, as an + immutable tuple: a raised error is a finished report, and a caller + inspecting it must not be able to edit the record. """ - def __init__(self, problems: list[ValidationProblem]) -> None: - self.problems = problems - super().__init__("\n".join(str(problem) for problem in problems)) + problems: tuple[ValidationProblem, ...] + def __init__(self, problems: Sequence[ValidationProblem]) -> None: + self.problems = tuple(problems) + super().__init__("\n".join(str(problem) for problem in self.problems)) -def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: + +def _prefix( + loc_head: str | int, problems: Sequence[ValidationProblem] +) -> tuple[ValidationProblem, ...]: """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" - return [ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems] + return tuple(ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems) -def validate_json(value: object) -> list[ValidationProblem]: +def validate_json(value: object) -> tuple[ValidationProblem, ...]: """Return every reason `value` is not JSON-serializable (recursively).""" if isinstance(value, float): if math.isfinite(value): - return [] - return [ValidationProblem((), f"non-finite float {value!r} is not JSON", "invalid_value")] + return () + return (ValidationProblem((), f"non-finite float {value!r} is not JSON", "invalid_value"),) if isinstance(value, (str, int, bool)) or value is None: - return [] + return () problems: list[ValidationProblem] = [] if isinstance(value, Mapping): for key, item in cast("Mapping[object, object]", value).items(): @@ -92,12 +98,12 @@ def validate_json(value: object) -> list[ValidationProblem]: ) continue problems.extend(_prefix(key, validate_json(item))) - return problems + return tuple(problems) if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): for index, item in enumerate(cast("Sequence[object]", value)): problems.extend(_prefix(index, validate_json(item))) - return problems - return [ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type")] + return tuple(problems) + return (ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type"),) def _is_canonical_json(value: object) -> TypeIs[JSONValue]: @@ -126,7 +132,7 @@ def parse_json(value: object) -> JSONValue: """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" normalized = arrays_to_tuples(value) problems = validate_json(normalized) - if problems: + if len(problems) != 0: raise MetadataValidationError(problems) return cast(JSONValue, normalized) @@ -177,17 +183,19 @@ def parse_json(value: object) -> JSONValue: ) -def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[ValidationProblem]: +def _missing_keys( + required: frozenset[str], doc: Mapping[str, object] +) -> tuple[ValidationProblem, ...]: """One `missing_key` problem per required key absent from `doc`.""" - return [ + return tuple( ValidationProblem((key,), "missing required key", "missing_key") for key in sorted(required - doc.keys()) - ] + ) def _unexpected_keys( allowed: frozenset[str], doc: Mapping[object, object] -) -> list[ValidationProblem]: +) -> tuple[ValidationProblem, ...]: """One problem per member outside a closed document's declared shape.""" problems: list[ValidationProblem] = [] for key in doc: @@ -199,18 +207,18 @@ def _unexpected_keys( problems.append( ValidationProblem((key,), "unexpected document member", "invalid_value") ) - return problems + return tuple(problems) def _check_literal( doc: Mapping[str, object], key: str, expected: object -) -> list[ValidationProblem]: +) -> tuple[ValidationProblem, ...]: """One `invalid_value` problem if `doc[key]` is present but not `expected`.""" if key in doc and (type(doc[key]) is not type(expected) or doc[key] != expected): - return [ - ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value") - ] - return [] + return ( + ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value"), + ) + return () def _validate_extension_fields_v3( @@ -218,7 +226,7 @@ def _validate_extension_fields_v3( standard_keys: frozenset[str], *, additional_reserved_keys: frozenset[str] = frozenset(), -) -> list[ValidationProblem]: +) -> tuple[ValidationProblem, ...]: """Validate v3 top-level key types and unknown-field JSON payloads.""" problems: list[ValidationProblem] = [] reserved_keys = standard_keys | additional_reserved_keys @@ -231,27 +239,27 @@ def _validate_extension_fields_v3( if key in reserved_keys: continue problems.extend(_prefix(key, validate_json(value))) - return problems + return tuple(problems) def validate_metadata_field_v3( value: object, *, allow_must_understand_false: bool = True -) -> list[ValidationProblem]: +) -> tuple[ValidationProblem, ...]: """Return every reason `value` is not a v3 metadata field. A metadata field is a bare name string or a mapping containing `name` and optional `configuration` and `must_understand` members. """ if isinstance(value, str): - return [] + return () if not isinstance(value, Mapping): - return [ + return ( ValidationProblem( (), "expected a metadata field (string or extension object)", "invalid_type", - ) - ] + ), + ) field = cast("Mapping[object, object]", value) problems: list[ValidationProblem] = [] allowed_keys = frozenset({"name", "configuration", "must_understand"}) @@ -293,7 +301,7 @@ def validate_metadata_field_v3( "invalid_value", ) ) - return problems + return tuple(problems) def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: @@ -310,7 +318,7 @@ def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: """Return `value` narrowed to `ZarrV3MetadataFieldJSON`, or raise `MetadataValidationError`.""" normalized = arrays_to_tuples(value) problems = validate_metadata_field_v3(normalized) - if problems: + if len(problems) != 0: raise MetadataValidationError(problems) return cast(ZarrV3MetadataFieldJSON, normalized) @@ -331,19 +339,19 @@ def _is_int_sequence(value: object) -> bool: ) -def _validate_dim_sequence(doc: Mapping[str, object], key: str) -> list[ValidationProblem]: +def _validate_dim_sequence(doc: Mapping[str, object], key: str) -> tuple[ValidationProblem, ...]: """Validate a dimension sequence (`shape` / `chunks`) if present in `doc`. Dimension lengths are non-negative integers. """ if key not in doc: - return [] + return () value = doc[key] if not _is_int_sequence(value): - return [ValidationProblem((key,), "expected a sequence of int", "invalid_type")] + return (ValidationProblem((key,), "expected a sequence of int", "invalid_type"),) if any(item < 0 for item in cast("Sequence[int]", value)): - return [ValidationProblem((key,), "expected non-negative integers", "invalid_value")] - return [] + return (ValidationProblem((key,), "expected non-negative integers", "invalid_value"),) + return () def _is_dtype_v2(value: object) -> bool: @@ -447,18 +455,18 @@ def _is_codec_v2(value: object) -> bool: ) -def _validate_codec_v2(value: object) -> list[ValidationProblem]: +def _validate_codec_v2(value: object) -> tuple[ValidationProblem, ...]: """Validate a v2 codec's required shape and JSON-valued configuration.""" if not _is_codec_v2(value): - return [ + return ( ValidationProblem( (), "expected a codec configuration with a string 'id'", "invalid_type" - ) - ] + ), + ) return validate_json(value) -def _validate_attributes(value: object) -> list[ValidationProblem]: +def _validate_attributes(value: object) -> tuple[ValidationProblem, ...]: """Validate an `attributes` value: a mapping with string keys. Returns a problem at `("attributes",)` if it is not, else `[]`. Shared by the @@ -470,27 +478,27 @@ def _validate_attributes(value: object) -> list[ValidationProblem]: if not isinstance(value, Mapping) or not all( isinstance(k, str) for k in cast("Mapping[object, object]", value) ): - return [ + return ( ValidationProblem( ("attributes",), "expected a mapping with string keys", "invalid_type" - ) - ] + ), + ) problems: list[ValidationProblem] = [] for key, item in cast("Mapping[str, object]", value).items(): problems.extend(_prefix("attributes", _prefix(key, validate_json(item)))) - return problems + return tuple(problems) -def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: +def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: """Return every reason `value` is not a structurally-valid v3 array doc. Checks structure, not domain validity. Unknown top-level keys are allowed (they map to `extra_fields`). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping", "invalid_type")] + return (ValidationProblem((), "expected a mapping", "invalid_type"),) doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) + problems: list[ValidationProblem] = list(_missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc)) problems.extend( _validate_extension_fields_v3( cast("Mapping[object, object]", value), ARRAY_METADATA_STANDARD_KEYS_V3 @@ -552,7 +560,7 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: "invalid_value", ) ) - return problems + return tuple(problems) def is_array_metadata_v3(value: object) -> TypeIs[ZarrV3ArrayMetadataJSON]: @@ -568,12 +576,12 @@ def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: """Return `value` as `ZarrV3ArrayMetadataJSON`, or raise `MetadataValidationError`.""" normalized = arrays_to_tuples(value) problems = validate_array_metadata_v3(normalized) - if problems: + if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV3ArrayMetadataJSON", normalized) -def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: +def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: """Return every reason `value` is not a structurally-valid v2 array doc. Checks structure, not domain validity: `dtype` must be a string or field @@ -582,22 +590,22 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: codec configurations (mappings with a string `id`). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping", "invalid_type")] + return (ValidationProblem((), "expected a mapping", "invalid_type"),) doc = cast("Mapping[str, object]", value) # Unlike the group document ("Other keys MUST NOT be present", # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), the v2 array document is open: other keys "SHOULD NOT be # present within the metadata object and SHOULD be ignored by # implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so members outside # ARRAY_METADATA_STANDARD_KEYS_V2 are not problems. - problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) + problems: list[ValidationProblem] = list(_missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc)) problems.extend(_check_literal(doc, "zarr_format", 2)) shape_problems = _validate_dim_sequence(doc, "shape") chunks_problems = _validate_dim_sequence(doc, "chunks") problems.extend(shape_problems) problems.extend(chunks_problems) if ( - not shape_problems - and not chunks_problems + len(shape_problems) == 0 + and len(chunks_problems) == 0 and _is_int_sequence(doc.get("shape")) and _is_int_sequence(doc.get("chunks")) ): @@ -660,7 +668,7 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) - return problems + return tuple(problems) def is_array_metadata_v2(value: object) -> TypeIs[ZarrV2ArrayMetadataJSON]: @@ -676,12 +684,12 @@ def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: """Return `value` as `ZarrV2ArrayMetadataJSON`, or raise `MetadataValidationError`.""" normalized = arrays_to_tuples(value) problems = validate_array_metadata_v2(normalized) - if problems: + if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV2ArrayMetadataJSON", normalized) -def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: +def validate_consolidated_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: """Return every reason `value` is not a valid inline consolidated envelope. Locs are value-relative (the caller prefixes with `consolidated_metadata` @@ -690,7 +698,7 @@ def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: `ZarrV3ConsolidatedMetadata.from_json` accepts. """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping", "invalid_type")] + return (ValidationProblem((), "expected a mapping", "invalid_type"),) env = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = [ ValidationProblem((key,), "missing required key", "missing_key") @@ -737,10 +745,10 @@ def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: "invalid_value", ) ) - return problems + return tuple(problems) -def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: +def validate_group_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: """Return every reason `value` is not a structurally-valid v3 group doc. Checks structure, not domain validity. Unknown top-level keys are allowed @@ -749,9 +757,9 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: `validate_consolidated_metadata_v3`. """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping", "invalid_type")] + return (ValidationProblem((), "expected a mapping", "invalid_type"),) doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc) + problems: list[ValidationProblem] = list(_missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc)) problems.extend( _validate_extension_fields_v3( cast("Mapping[object, object]", value), @@ -773,7 +781,7 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: validate_consolidated_metadata_v3(doc["consolidated_metadata"]), ) ) - return problems + return tuple(problems) def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: @@ -785,28 +793,28 @@ def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: """Return `value` narrowed to `ZarrV3GroupMetadataJSON`, or raise `MetadataValidationError`.""" normalized = arrays_to_tuples(value) problems = validate_group_metadata_v3(normalized) - if problems: + if len(problems) != 0: raise MetadataValidationError(problems) return cast(ZarrV3GroupMetadataJSON, normalized) -def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: +def validate_group_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: """Return every reason `value` is not a structurally-valid v2 group doc. Validates the in-memory merged form: the `.zgroup` fields plus an optional `attributes` mapping folded in from `.zattrs`. """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping", "invalid_type")] + return (ValidationProblem((), "expected a mapping", "invalid_type"),) doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc) + problems: list[ValidationProblem] = list(_missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc)) problems.extend( _unexpected_keys(GROUP_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) ) problems.extend(_check_literal(doc, "zarr_format", 2)) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) - return problems + return tuple(problems) def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: @@ -818,7 +826,7 @@ def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: """Return `value` narrowed to `ZarrV2GroupMetadataJSON`, or raise `MetadataValidationError`.""" normalized = arrays_to_tuples(value) problems = validate_group_metadata_v2(normalized) - if problems: + if len(problems) != 0: raise MetadataValidationError(problems) return cast(ZarrV2GroupMetadataJSON, normalized) @@ -828,9 +836,13 @@ def _reject_json_constant(constant: str) -> NoReturn: raise ValueError(f"non-standard JSON constant {constant!r}") -def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: +def load_store_json(mapping: Mapping[str, bytes], key: str) -> object: """Decode the JSON document stored at `key` in `mapping`. + Returns `object`, not `Any`: what a store holds is unknown until a + validator says otherwise, and `Any` would let unchecked values flow + into typed positions silently. Narrow the result with a `parse_*`. + Every ingestion failure surfaces as `MetadataValidationError`: a missing store key is a `missing_key` problem and undecodable bytes are an `invalid_json` problem, rather than leaking `KeyError` / diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 8584efa570..5b0c9e5b57 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -54,17 +54,6 @@ class ArrayManifest(BaseModel): from zarr_metadata._pydantic_schema import ( ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldSchema, ) -from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataJSON -from zarr_metadata.v2.consolidated import ( - ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataJSON, -) -from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataJSON -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldJSON -from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataJSON -from zarr_metadata.v3.consolidated import ( - ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataJSON, -) -from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataJSON if TYPE_CHECKING: from collections.abc import Callable @@ -89,7 +78,7 @@ def coerce(value: object) -> _M: _coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), json_schema_input_type=_ZarrV3ArrayMetadataSchema, ), - PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=_ZarrV3ArrayMetadataJSON), + PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=_ZarrV3ArrayMetadataSchema), ] """Field type for a v3 array metadata document (`zarr.json` content).""" @@ -99,7 +88,7 @@ def coerce(value: object) -> _M: _coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json), json_schema_input_type=_ZarrV2ArrayMetadataSchema, ), - PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=_ZarrV2ArrayMetadataJSON), + PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=_ZarrV2ArrayMetadataSchema), ] """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" @@ -109,7 +98,7 @@ def coerce(value: object) -> _M: _coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json), json_schema_input_type=_ZarrV3GroupMetadataSchema, ), - PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=_ZarrV3GroupMetadataJSON), + PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=_ZarrV3GroupMetadataSchema), ] """Field type for a v3 group metadata document (`zarr.json` content).""" @@ -119,7 +108,7 @@ def coerce(value: object) -> _M: _coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json), json_schema_input_type=_ZarrV2GroupMetadataSchema, ), - PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=_ZarrV2GroupMetadataJSON), + PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=_ZarrV2GroupMetadataSchema), ] """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" @@ -134,7 +123,7 @@ def coerce(value: object) -> _M: ), PlainSerializer( _model.ZarrV3ConsolidatedMetadata.to_json, - return_type=_ZarrV3ConsolidatedMetadataJSON, + return_type=_ZarrV3ConsolidatedMetadataSchema, ), ] """Field type for v3 inline consolidated metadata.""" @@ -150,7 +139,7 @@ def coerce(value: object) -> _M: ), PlainSerializer( _model.ZarrV2ConsolidatedMetadata.to_json, - return_type=_ZarrV2ConsolidatedMetadataJSON, + return_type=_ZarrV2ConsolidatedMetadataSchema, ), ] """Field type for a v2 `.zmetadata` document.""" @@ -161,7 +150,7 @@ def coerce(value: object) -> _M: _coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json), json_schema_input_type=_ZarrV3MetadataFieldSchema, ), - PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=_ZarrV3MetadataFieldJSON), + PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=_ZarrV3MetadataFieldSchema), ] """Field type for one normalized v3 metadata extension envelope.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py index 999c9131da..a6b8e4a58e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py @@ -34,7 +34,7 @@ class ZarrV2ConsolidatedMetadataJSON(TypedDict): `key.endswith(".zarray")` etc. """ - zarr_consolidated_format: int + zarr_consolidated_format: Literal[1] metadata: Mapping[str, ZarrV2ZArrayJSON | ZarrV2ZGroupJSON | ZarrV2ZAttrsJSON] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 480f5538fa..78c38b702a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -4,7 +4,7 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md """ -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import TypedDict @@ -23,18 +23,19 @@ """ -class RectilinearChunkGridConfiguration(TypedDict): +class RectilinearChunkGridConfiguration(TypedDict, closed=True): """Configuration for the rectilinear chunk grid.""" kind: Literal["inline"] chunk_shapes: tuple[RectilinearDimSpec, ...] -class RectilinearChunkGridObject(TypedDict): +class RectilinearChunkGridObject(TypedDict, closed=True): """Rectilinear chunk grid metadata in object form.""" name: RectilinearChunkGridName configuration: RectilinearChunkGridConfiguration + must_understand: NotRequired[bool] RectilinearChunkGridMetadata = RectilinearChunkGridObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index a0b33688c8..bdcd9e06c5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -4,7 +4,7 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#regular-grids """ -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import TypedDict @@ -15,17 +15,18 @@ """Literal type of the `name` field of the regular chunk grid.""" -class RegularChunkGridConfiguration(TypedDict): +class RegularChunkGridConfiguration(TypedDict, closed=True): """Configuration for the regular chunk grid.""" chunk_shape: tuple[int, ...] -class RegularChunkGridObject(TypedDict): +class RegularChunkGridObject(TypedDict, closed=True): """Regular chunk grid metadata in object form.""" name: RegularChunkGridName configuration: RegularChunkGridConfiguration + must_understand: NotRequired[bool] RegularChunkGridMetadata = RegularChunkGridObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 39c5385ce4..10a7c0cb55 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -27,7 +27,7 @@ """Tuple of permitted values for the `separator` field of the default chunk key encoding.""" -class DefaultChunkKeyEncodingConfiguration(TypedDict): +class DefaultChunkKeyEncodingConfiguration(TypedDict, closed=True): """Configuration for the default chunk key encoding. `separator` is optional and defaults to `"/"` per spec. @@ -37,11 +37,12 @@ class DefaultChunkKeyEncodingConfiguration(TypedDict): separator: NotRequired[DefaultChunkKeyEncodingSeparator] -class DefaultChunkKeyEncodingObject(TypedDict): +class DefaultChunkKeyEncodingObject(TypedDict, closed=True): """Default chunk key encoding metadata in object form.""" name: DefaultChunkKeyEncodingName configuration: NotRequired[DefaultChunkKeyEncodingConfiguration] + must_understand: NotRequired[bool] DefaultChunkKeyEncodingMetadata = DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 3face04f91..63cdf26783 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -33,7 +33,7 @@ """Tuple of permitted values for the `separator` field of the v2 chunk key encoding.""" -class V2ChunkKeyEncodingConfiguration(TypedDict): +class V2ChunkKeyEncodingConfiguration(TypedDict, closed=True): """Configuration for the v2 chunk key encoding. `separator` is optional and defaults to `"."` per spec. @@ -43,11 +43,12 @@ class V2ChunkKeyEncodingConfiguration(TypedDict): separator: NotRequired[V2ChunkKeyEncodingSeparator] -class V2ChunkKeyEncodingObject(TypedDict): +class V2ChunkKeyEncodingObject(TypedDict, closed=True): """v2-compatibility chunk key encoding metadata in object form.""" name: V2ChunkKeyEncodingName configuration: NotRequired[V2ChunkKeyEncodingConfiguration] + must_understand: NotRequired[bool] V2ChunkKeyEncodingMetadata = V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 4c3631a032..3387a7c285 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -27,7 +27,7 @@ """Tuple of permitted values for the `cname` field of the `blosc` codec.""" -class BloscCodecConfiguration(TypedDict): +class BloscCodecConfiguration(TypedDict, closed=True): """Configuration for the Zarr v3 `blosc` codec.""" cname: BloscCName @@ -37,11 +37,12 @@ class BloscCodecConfiguration(TypedDict): typesize: NotRequired[int] -class BloscCodecObject(TypedDict): +class BloscCodecObject(TypedDict, closed=True): """`blosc` codec metadata in object form.""" name: BloscCodecName configuration: BloscCodecConfiguration + must_understand: NotRequired[bool] BloscCodecMetadata = BloscCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 43b165f9e6..4feb6b8c1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -21,7 +21,7 @@ """Tuple of permitted values for the `endian` field of the `bytes` codec.""" -class BytesCodecConfiguration(TypedDict): +class BytesCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `bytes` codec. @@ -31,7 +31,7 @@ class BytesCodecConfiguration(TypedDict): endian: NotRequired[Endianness] -class BytesCodecObject(TypedDict): +class BytesCodecObject(TypedDict, closed=True): """`bytes` codec metadata in object form. `configuration` is itself optional — when no configuration fields are @@ -42,6 +42,7 @@ class BytesCodecObject(TypedDict): name: BytesCodecName configuration: NotRequired[BytesCodecConfiguration] + must_understand: NotRequired[bool] BytesCodecMetadata = BytesCodecObject | BytesCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index d1b878c95c..656a509ed7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -55,14 +55,14 @@ """ -class ScalarMap(TypedDict): +class ScalarMap(TypedDict, closed=True): """Optional encode/decode scalar overrides for the cast_value codec.""" encode: NotRequired[tuple[ScalarMapEntry, ...]] decode: NotRequired[tuple[ScalarMapEntry, ...]] -class CastValueCodecConfiguration(TypedDict): +class CastValueCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `cast_value` codec. @@ -77,11 +77,12 @@ class CastValueCodecConfiguration(TypedDict): scalar_map: NotRequired[ScalarMap] -class CastValueCodecObject(TypedDict): +class CastValueCodecObject(TypedDict, closed=True): """`cast_value` codec metadata in object form.""" name: CastValueCodecName configuration: CastValueCodecConfiguration + must_understand: NotRequired[bool] CastValueCodecMetadata = CastValueCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index aa72fcae5a..05661d0b59 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -22,7 +22,7 @@ class Empty(TypedDict, closed=True): """An empty mapping""" -class Crc32cCodecObject(TypedDict): +class Crc32cCodecObject(TypedDict, closed=True): """`crc32c` codec metadata in object form. Per spec the codec has no configuration fields. `configuration` is @@ -32,6 +32,7 @@ class Crc32cCodecObject(TypedDict): name: Crc32cCodecName configuration: NotRequired[Empty] + must_understand: NotRequired[bool] Crc32cCodecMetadata = Crc32cCodecObject | Crc32cCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 9a9647263c..d516088f99 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -4,7 +4,7 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/gzip/index.html """ -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import TypedDict @@ -15,7 +15,7 @@ """Literal type of the `name` field of the `gzip` codec.""" -class GzipCodecConfiguration(TypedDict): +class GzipCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `gzip` codec. @@ -32,11 +32,12 @@ class GzipCodecConfiguration(TypedDict): level: int -class GzipCodecObject(TypedDict): +class GzipCodecObject(TypedDict, closed=True): """`gzip` codec metadata in object form.""" name: GzipCodecName configuration: GzipCodecConfiguration + must_understand: NotRequired[bool] GzipCodecMetadata = GzipCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index abf11b4211..344a4c435c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -17,7 +17,7 @@ """Literal type of the `name` field of the `scale_offset` codec.""" -class ScaleOffsetCodecConfiguration(TypedDict): +class ScaleOffsetCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `scale_offset` codec. @@ -32,7 +32,7 @@ class ScaleOffsetCodecConfiguration(TypedDict): scale: NotRequired[JSONValue] -class ScaleOffsetCodecObject(TypedDict): +class ScaleOffsetCodecObject(TypedDict, closed=True): """`scale_offset` codec metadata in object form. `configuration` is itself optional per spec — when both `offset` and @@ -43,6 +43,7 @@ class ScaleOffsetCodecObject(TypedDict): name: ScaleOffsetCodecName configuration: NotRequired[ScaleOffsetCodecConfiguration] + must_understand: NotRequired[bool] ScaleOffsetCodecMetadata = ScaleOffsetCodecObject | ScaleOffsetCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index ac40e78b7b..0ce466b5cf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -23,7 +23,7 @@ """Tuple of permitted values for the `index_location` field of the `sharding_indexed` codec.""" -class ShardingIndexedCodecConfiguration(TypedDict): +class ShardingIndexedCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `sharding_indexed` codec. @@ -47,11 +47,12 @@ class ShardingIndexedCodecConfiguration(TypedDict): index_location: NotRequired[ShardingIndexLocation] -class ShardingIndexedCodecObject(TypedDict): +class ShardingIndexedCodecObject(TypedDict, closed=True): """`sharding_indexed` codec metadata in object form.""" name: ShardingIndexedCodecName configuration: ShardingIndexedCodecConfiguration + must_understand: NotRequired[bool] ShardingIndexedCodecMetadata = ShardingIndexedCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 41d9bcdc6d..8024174605 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -4,7 +4,7 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/index.html """ -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import TypedDict @@ -15,7 +15,7 @@ """Literal type of the `name` field of the `transpose` codec.""" -class TransposeCodecConfiguration(TypedDict): +class TransposeCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `transpose` codec. @@ -26,11 +26,12 @@ class TransposeCodecConfiguration(TypedDict): order: tuple[int, ...] -class TransposeCodecObject(TypedDict): +class TransposeCodecObject(TypedDict, closed=True): """`transpose` codec metadata in object form.""" name: TransposeCodecName configuration: TransposeCodecConfiguration + must_understand: NotRequired[bool] TransposeCodecMetadata = TransposeCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 9fdf177b4c..b7ee3f5685 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -17,7 +17,7 @@ """Literal type of the `name` field of the `zstd` codec.""" -class ZstdCodecConfiguration(TypedDict): +class ZstdCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `zstd` codec. @@ -30,11 +30,12 @@ class ZstdCodecConfiguration(TypedDict): checksum: NotRequired[bool] -class ZstdCodecObject(TypedDict): +class ZstdCodecObject(TypedDict, closed=True): """`zstd` codec metadata in object form.""" name: ZstdCodecName configuration: ZstdCodecConfiguration + must_understand: NotRequired[bool] ZstdCodecMetadata = ZstdCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index ba7c9aec0d..274af507b7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -20,7 +20,7 @@ from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -class ZarrV3ConsolidatedMetadataJSON(TypedDict): +class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): """ Inline consolidated metadata embedded in a v3 group. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index bed264e96e..4f9a6415c5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -4,7 +4,7 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md """ -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict @@ -20,7 +20,7 @@ """Time unit codes used by numpy.datetime64.""" -class NumpyDatetime64Configuration(TypedDict): +class NumpyDatetime64Configuration(TypedDict, closed=True): """ Configuration for the `numpy.datetime64` data type. @@ -36,11 +36,12 @@ class NumpyDatetime64Configuration(TypedDict): scale_factor: ReadOnly[int] -class NumpyDatetime64(TypedDict): +class NumpyDatetime64(TypedDict, closed=True): """`numpy.datetime64` data type metadata.""" name: NumpyDatetime64DataTypeName configuration: NumpyDatetime64Configuration + must_understand: NotRequired[bool] NumpyDatetime64FillValue = int | Literal["NaT"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index ceb31d8c40..a1c4fef772 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -4,7 +4,7 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.timedelta64/README.md """ -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict @@ -39,7 +39,7 @@ """Runtime tuple of the permitted `numpy.timedelta64`/`numpy.datetime64` unit strings.""" -class NumpyTimedelta64Configuration(TypedDict): +class NumpyTimedelta64Configuration(TypedDict, closed=True): """ Configuration for the `numpy.timedelta64` data type. @@ -55,11 +55,12 @@ class NumpyTimedelta64Configuration(TypedDict): scale_factor: ReadOnly[int] -class NumpyTimedelta64(TypedDict): +class NumpyTimedelta64(TypedDict, closed=True): """`numpy.timedelta64` data type metadata.""" name: NumpyTimedelta64DataTypeName configuration: NumpyTimedelta64Configuration + must_understand: NotRequired[bool] NumpyTimedelta64FillValue = int | Literal["NaT"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 5795c927f5..87a437951d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -5,7 +5,7 @@ """ from collections.abc import Mapping -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict @@ -19,7 +19,7 @@ """Literal type of the `name` field of the `struct` data type.""" -class StructField(TypedDict): +class StructField(TypedDict, closed=True): """ A single field entry inside a structured dtype. @@ -36,17 +36,18 @@ class StructField(TypedDict): data_type: ReadOnly[ZarrV3MetadataFieldJSON] -class StructConfiguration(TypedDict): +class StructConfiguration(TypedDict, closed=True): """Configuration for the `struct` data type.""" fields: ReadOnly[tuple[StructField, ...]] -class Struct(TypedDict): +class Struct(TypedDict, closed=True): """`struct` data type metadata.""" name: StructDataTypeName configuration: StructConfiguration + must_understand: NotRequired[bool] StructFillValue = Mapping[str, JSONValue] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index c30f457618..69a3823b6f 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -385,7 +385,7 @@ def test_v3_create_default_is_valid_empty_array() -> None: assert m.attributes == {} assert m.extra_fields == {} # the default document is structurally valid and round-trips - assert validate_array_metadata_v3(m.to_json()) == [] + assert validate_array_metadata_v3(m.to_json()) == () assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m @@ -407,7 +407,7 @@ def test_v2_create_default_is_valid_empty_array() -> None: assert m.compressor is None assert m.filters is None assert m.attributes is UNSET - assert validate_array_metadata_v2(m.to_json()) == [] + assert validate_array_metadata_v2(m.to_json()) == () assert ZarrV2ArrayMetadata.from_json(m.to_json()) == m @@ -760,7 +760,7 @@ def test_v3_parser_accepts_bare_string_data_type() -> None: @pytest.mark.parametrize("name", ["bytes", "ANY string", "urn:example:codec"]) def test_metadata_field_accepts_any_string_name(name: str) -> None: """The structural layer checks the name type, not syntax or registration.""" - assert validate_metadata_field_v3({"name": name}) == [] + assert validate_metadata_field_v3({"name": name}) == () @pytest.mark.parametrize("value", [0, 1, "false", None]) @@ -783,7 +783,7 @@ def test_optional_extension_points_allow_must_understand_false(field: str) -> No """Codecs and storage transformers may be explicitly ignorable.""" doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc[field] = ({"name": "optional", "must_understand": False},) - assert validate_array_metadata_v3(doc) == [] + assert validate_array_metadata_v3(doc) == () @pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) @@ -936,7 +936,7 @@ def test_is_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: """validate_json reports the problems (and their locs) for a value.""" problems = validate_json(case.input) - assert (problems == []) is (case.output == frozenset()) + assert (problems == ()) is (case.output == frozenset()) assert {p.loc for p in problems} >= case.output @@ -1027,7 +1027,7 @@ def test_validate_metadata_field_v3( ) -> None: """validate_metadata_field_v3 reports the problems for a metadata-field value.""" problems = validate_metadata_field_v3(case.input) - assert (problems == []) is (case.output == frozenset()) + assert (problems == ()) is (case.output == frozenset()) assert {p.loc for p in problems} >= case.output @@ -1178,7 +1178,7 @@ def test_array_metadata_guards( valid = case.output == frozenset() assert is_fn(doc) is valid problems = validate_fn(doc) - assert (problems == []) is valid + assert (problems == ()) is valid assert {p.loc for p in problems} >= case.output if valid: assert parse_fn(doc) is doc @@ -1258,7 +1258,7 @@ def test_metadata_validation_error_holds_problems() -> None: ), ] err = MetadataValidationError(problems) - assert err.problems == problems + assert err.problems == tuple(problems) assert "shape: missing required key" in str(err) assert "data_type: expected a metadata field" in str(err) @@ -1267,9 +1267,9 @@ def test_prefix_prepends_loc_head() -> None: """_prefix prepends a loc head to each problem's loc.""" problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] prefixed = _prefix(0, problems) - assert prefixed == [ - ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type") - ] + assert prefixed == ( + ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type"), + ) # --- Stricter v2/v3 field validation and error kinds ------------------------- @@ -1286,7 +1286,7 @@ def test_v2_structured_dtype_records_accepted() -> None: """A structured v2 dtype (field records, optionally nested/shaped) validates.""" dtype = (("a", " None: @@ -1345,7 +1345,7 @@ def test_v2_filters_may_be_empty() -> None: doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc["filters"] = () - assert validate_array_metadata_v2(doc) == [] + assert validate_array_metadata_v2(doc) == () assert ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}).filters == () @@ -1399,7 +1399,7 @@ def test_array_v2_ignores_unknown_document_member() -> None: """ doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} - assert validate_array_metadata_v2(doc) == [] + assert validate_array_metadata_v2(doc) == () assert "unexpected" not in ZarrV2ArrayMetadata.from_json(doc).to_json() @@ -1444,9 +1444,9 @@ def test_missing_key_kind_is_machine_readable() -> None: doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) del doc["chunk_key_encoding"] problems = validate_array_metadata_v3(doc) - assert problems == [ - ValidationProblem(("chunk_key_encoding",), "missing required key", "missing_key") - ] + assert problems == ( + ValidationProblem(("chunk_key_encoding",), "missing required key", "missing_key"), + ) # --- Unified error channels --------------------------------------------------- @@ -1479,9 +1479,9 @@ def test_from_key_value_missing_key_kind() -> None: """A missing store key surfaces as a missing_key problem at the store-key loc.""" with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ArrayMetadata.from_key_value({}) - assert exc_info.value.problems == [ - ValidationProblem((".zarray",), "missing store key", "missing_key") - ] + assert exc_info.value.problems == ( + ValidationProblem((".zarray",), "missing store key", "missing_key"), + ) def test_extra_fields_overlap_raises_metadata_error() -> None: @@ -1616,8 +1616,8 @@ def test_array_parsers_normalize_json_lists_before_narrowing() -> None: v3_raw = json.loads(json.dumps(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json())) v2_raw = json.loads(json.dumps(ZarrV2ArrayMetadata.create_default(shape=(2,)).to_json())) - assert validate_array_metadata_v3(v3_raw) == [] - assert validate_array_metadata_v2(v2_raw) == [] + assert validate_array_metadata_v3(v3_raw) == () + assert validate_array_metadata_v2(v2_raw) == () assert not is_array_metadata_v3(v3_raw) assert not is_array_metadata_v2(v2_raw) diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index d280091ba6..b0e451b0a0 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -474,7 +474,7 @@ def test_group_v3_valid_consolidated_passes_validator() -> None: "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, }, } - assert validate_group_metadata_v3(doc) == [] + assert validate_group_metadata_v3(doc) == () def test_v3_consolidated_rejects_unknown_envelope_member() -> None: @@ -522,7 +522,7 @@ def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: it is read as absence (UNSET) and never written back — the round-trip deliberately repairs the document rather than preserving the bug.""" null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} - assert validate_group_metadata_v3(null_doc) == [] + assert validate_group_metadata_v3(null_doc) == () model = ZarrV3GroupMetadata.from_json(null_doc) assert model.consolidated_metadata is UNSET assert "consolidated_metadata" not in model.to_json() diff --git a/packages/zarr-metadata/tests/test_partial_equivalence.py b/packages/zarr-metadata/tests/test_partial_equivalence.py index 33492b2356..b364fce091 100644 --- a/packages/zarr-metadata/tests/test_partial_equivalence.py +++ b/packages/zarr-metadata/tests/test_partial_equivalence.py @@ -10,8 +10,6 @@ from __future__ import annotations -from typing import Any - import pytest from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial @@ -29,7 +27,7 @@ @pytest.mark.parametrize(("full", "partial"), PAIRS, ids=lambda p: p.__name__) -def test_partial_matches_full(full: Any, partial: Any) -> None: +def test_partial_matches_full(full: type, partial: type) -> None: """Partial TypedDict has identical fields and extra_items, only total differs.""" assert full.__annotations__ == partial.__annotations__, ( f"{partial.__name__} fields drifted from {full.__name__}: "