From 8da13336dfa60c2b4412097bc5e58488aebbb66c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 31 May 2026 19:28:07 +0200 Subject: [PATCH 01/48] chore(deps): bump the actions group across 1 directory with 8 updates (#176) Bumps the actions group with 8 updates in the / directory: | Package | From | To | | --- | --- | --- | | [prefix-dev/setup-pixi](https://github.com/prefix-dev/setup-pixi) | `0.9.5` | `0.9.6` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `6.0.0` | `6.0.1` | | [github/issue-metrics](https://github.com/github/issue-metrics) | `4.2.2` | `4.2.7` | | [j178/prek-action](https://github.com/j178/prek-action) | `2.0.3` | `2.0.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.0` | `7.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.1` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.13.0` | `1.14.0` | | [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) | `0.5.3` | `0.5.6` | Updates `prefix-dev/setup-pixi` from 0.9.5 to 0.9.6 - [Release notes](https://github.com/prefix-dev/setup-pixi/releases) - [Commits](https://github.com/prefix-dev/setup-pixi/compare/1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0...5185adfbffb4bd703da3010310260805d89ebb11) Updates `codecov/codecov-action` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/57e3a136b779b570ffcdbf80b3bdc90e7fab3de2...e79a6962e0d4c0c17b229090214935d2e33f8354) Updates `github/issue-metrics` from 4.2.2 to 4.2.7 - [Release notes](https://github.com/github/issue-metrics/releases) - [Commits](https://github.com/github/issue-metrics/compare/c9e9838147fd355dace335ba787f01b6641a400a...1e38d5e62363e14db8019ed7d106b9855bdba6cc) Updates `j178/prek-action` from 2.0.3 to 2.0.4 - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/6ad80277337ad479fe43bd70701c3f7f8aa74db3...bdca6f102f98e2b4c7029491a53dfd366469e33d) Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) Updates `pypa/gh-action-pypi-publish` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.13.0...cef221092ed1bacb1cc03d23a2d87d1d172e277b) Updates `zizmorcore/zizmor-action` from 0.5.3 to 0.5.6 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...5f14fd08f7cf1cb1609c1e344975f152c7ee938d) --- updated-dependencies: - dependency-name: prefix-dev/setup-pixi dependency-version: 0.9.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: codecov/codecov-action dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/issue-metrics dependency-version: 4.2.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: j178/prek-action dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/downstream.yml | 2 +- .github/workflows/gpu_test.yml | 2 +- .github/workflows/hypothesis.yaml | 2 +- .github/workflows/issue-metrics.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/zarr-metadata-release.yml | 12 ++++++------ .github/workflows/zizmor.yml | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 74026233c4..3eb6898895 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -34,7 +34,7 @@ jobs: persist-credentials: false - name: Set up pixi - uses: prefix-dev/setup-pixi@1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0 # v0.9.5 + uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6 with: manifest-path: xarray/pixi.toml diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 403441b306..333769cb9e 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -76,7 +76,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: gpu diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 4f9467be7d..a456b2aa0a 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -93,7 +93,7 @@ jobs: key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/issue-metrics.yml b/.github/workflows/issue-metrics.yml index 14fba5b9ec..510849ef3e 100644 --- a/.github/workflows/issue-metrics.yml +++ b/.github/workflows/issue-metrics.yml @@ -33,7 +33,7 @@ jobs: echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" - name: Run issue-metrics tool - uses: github/issue-metrics@c9e9838147fd355dace335ba787f01b6641a400a # v4.2.2 + uses: github/issue-metrics@1e38d5e62363e14db8019ed7d106b9855bdba6cc # v4.2.7 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 768e660ec2..fec211b4dd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,4 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true - - uses: j178/prek-action@6ad80277337ad479fe43bd70701c3f7f8aa74db3 # v2.0.3 + - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03143d3e5b..62e571856b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests @@ -125,7 +125,7 @@ jobs: run: | hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index 809d502f16..9639fcfdd3 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -35,7 +35,7 @@ jobs: - name: Build run: hatch build - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: zarr-metadata-dist path: packages/zarr-metadata/dist @@ -45,7 +45,7 @@ jobs: needs: [build] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -76,7 +76,7 @@ jobs: id-token: write # required for OIDC trusted publishing attestations: write # required for artifact attestations steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -87,7 +87,7 @@ jobs: subject-path: dist/* - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 upload_testpypi: name: Upload to TestPyPI @@ -101,7 +101,7 @@ jobs: id-token: write attestations: write steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -112,6 +112,6 @@ jobs: subject-path: dist/* - name: Publish package to TestPyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index da19f22421..7ac4fe5d0e 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -32,4 +32,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 + uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6 From ce2cfd7ba44bc6f9b44e63ec372e026a4860c409 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 4 Jul 2026 22:41:46 +0200 Subject: [PATCH 02/48] =?UTF-8?q?feat(zarr-metadata):=20add=20model.=5Fval?= =?UTF-8?q?idation=20=E2=80=94=20structural=20validators?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/__init__.py | 1 + .../src/zarr_metadata/model/_validation.py | 271 ++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_validation.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py new file mode 100644 index 0000000000..e1382a6f0a --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -0,0 +1 @@ +"""In-memory models for Zarr metadata documents.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py new file mode 100644 index 0000000000..d0e30e1cda --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -0,0 +1,271 @@ +"""Structural validation for Zarr metadata documents. + +Validators check JSON structure (shapes, key presence, primitive kinds), +not domain validity. Each concept gets a `validate_*` function returning +every problem found, an `is_*` type guard, and a `parse_*` function that +narrows or raises `MetadataValidationError`. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Final, cast + +from typing_extensions import TypeIs + +from zarr_metadata import ArrayMetadataV2, ArrayMetadataV3, MetadataV3 +from zarr_metadata._common import JSONValue + + +@dataclass(frozen=True, slots=True) +class ValidationProblem: + """A single structural problem found while validating a metadata document. + + `loc` is the path from the document root to the offending value, e.g. + `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + """ + + loc: tuple[str | int, ...] + message: str + + def __str__(self) -> str: + location = ".".join(str(part) for part in self.loc) if self.loc else "" + return f"{location}: {self.message}" + + +class MetadataValidationError(ValueError): + """Raised when a value fails structural metadata validation. + + Carries every problem found (not just the first) in `.problems`. + """ + + def __init__(self, problems: list[ValidationProblem]) -> None: + self.problems = problems + super().__init__("\n".join(str(problem) for problem in problems)) + + +def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: + """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" + return [ValidationProblem((loc_head, *p.loc), p.message) for p in problems] + + +def validate_json(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not JSON-serializable (recursively).""" + if isinstance(value, (str, int, float, bool)) or value is None: + return [] + problems: list[ValidationProblem] = [] + if isinstance(value, Mapping): + for key, item in cast("Mapping[object, object]", value).items(): + if not isinstance(key, str): + problems.append(ValidationProblem((), f"non-string key {key!r} in JSON object")) + continue + problems.extend(_prefix(key, validate_json(item))) + return problems + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + for index, item in enumerate(cast("Sequence[object]", value)): + problems.extend(_prefix(index, validate_json(item))) + return problems + return [ValidationProblem((), f"not a JSON-serializable value: {value!r}")] + + +def is_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` is a JSON-serializable structure (recursively).""" + return not validate_json(value) + + +def parse_json(value: object) -> JSONValue: + """Return `value` narrowed to `JSONValue`, or raise `MetadataValidationError`.""" + problems = validate_json(value) + if problems: + raise MetadataValidationError(problems) + return cast(JSONValue, value) + + +# The standard top-level keys of a v3 array metadata document. Anything outside +# this set is an extension field. Built from the TypedDict's required/optional +# key sets (which resolve inherited keys, unlike `__annotations__`). +ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + ArrayMetadataV3.__required_keys__ +) +ARRAY_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + ArrayMetadataV3.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 +) + +ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + ArrayMetadataV2.__required_keys__ +) + + +def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a v3 metadata field. + + A metadata field is a bare name string or a `{name, configuration}` mapping. + """ + if isinstance(value, str): + return [] + if not isinstance(value, Mapping): + return [ + ValidationProblem((), "expected a metadata field (string or {name, configuration})") + ] + field = cast("Mapping[object, object]", value) + problems: list[ValidationProblem] = [] + if not isinstance(field.get("name"), str): + problems.append(ValidationProblem(("name",), "expected a string name")) + if "configuration" in field: + configuration = field["configuration"] + if not isinstance(configuration, Mapping): + problems.append(ValidationProblem(("configuration",), "expected a mapping")) + elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): + problems.append(ValidationProblem(("configuration",), "expected string keys")) + return problems + + +def is_metadata_field_v3(value: object) -> TypeIs[MetadataV3]: + """Whether `value` is a v3 metadata field: a bare name or a named config.""" + return not validate_metadata_field_v3(value) + + +def parse_metadata_field_v3(value: object) -> MetadataV3: + """Return `value` narrowed to `MetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_metadata_field_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(MetadataV3, value) + + +def _is_int_sequence(value: object) -> bool: + """Whether `value` is a non-string sequence of integers.""" + return ( + not isinstance(value, str) + and isinstance(value, Sequence) + and all(isinstance(item, int) for item in cast("Sequence[object]", value)) + ) + + +def _validate_attributes(value: object) -> list[ValidationProblem]: + """Validate an `attributes` value: a mapping with string keys. + + Returns a problem at `("attributes",)` if it is not, else `[]`. Shared by the + v2 and v3 validators. Unlike the other `validate_*` functions (which + return value-relative locs for the caller to `_prefix`), this emits the + already-parent-relative `("attributes",)` loc, since it is only ever called + with a document's `attributes` value. + """ + if not isinstance(value, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", value) + ): + return [ValidationProblem(("attributes",), "expected a mapping with string keys")] + return [] + + +def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 array doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V3 - doc.keys()) + ] + if "shape" in doc and not _is_int_sequence(doc["shape"]): + problems.append(ValidationProblem(("shape",), "expected a sequence of int")) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if key in doc: + problems.extend(_prefix(key, validate_metadata_field_v3(doc[key]))) + for key in ("codecs", "storage_transformers"): + if key in doc: + entries = doc[key] + if isinstance(entries, str) or not isinstance(entries, Sequence): + problems.append(ValidationProblem((key,), "expected a sequence")) + else: + for index, entry in enumerate(cast("Sequence[object]", entries)): + problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "dimension_names" in doc: + # Simple typed sequences (dimension_names, shape, chunks) report a single + # field-level loc, not per-bad-item locs; per-index locs are reserved for + # the metadata-field lists (codecs, storage_transformers). + names = doc["dimension_names"] + if isinstance(names, str) or not isinstance(names, Sequence): + problems.append(ValidationProblem(("dimension_names",), "expected a sequence")) + elif not all( + item is None or isinstance(item, str) for item in cast("Sequence[object]", names) + ): + problems.append( + ValidationProblem(("dimension_names",), "expected items of str or None") + ) + return problems + + +def is_array_metadata_v3(value: object) -> TypeIs[ArrayMetadataV3]: + """Whether `value` is a structurally-valid v3 array metadata document.""" + return not validate_array_metadata_v3(value) + + +def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: + """Return `value` narrowed to `ArrayMetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_array_metadata_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(ArrayMetadataV3, value) + + +def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 array doc. + + Checks structure, not domain validity. `compressor`/`filters` are required + keys but may be `None`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V2 - doc.keys()) + ] + problems.extend( + ValidationProblem((key,), "expected a sequence of int") + for key in ("shape", "chunks") + if key in doc and not _is_int_sequence(doc[key]) + ) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_array_metadata_v2(value: object) -> TypeIs[ArrayMetadataV2]: + """Whether `value` is a structurally-valid v2 array metadata document.""" + return not validate_array_metadata_v2(value) + + +def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: + """Return `value` narrowed to `ArrayMetadataV2`, or raise `MetadataValidationError`.""" + problems = validate_array_metadata_v2(value) + if problems: + raise MetadataValidationError(problems) + return cast(ArrayMetadataV2, value) + + +def _arrays_to_tuples(obj: object) -> object: + """Recursively convert every list in a JSON-decoded structure to a tuple.""" + if isinstance(obj, list): + return tuple(_arrays_to_tuples(item) for item in cast("list[object]", obj)) + if isinstance(obj, dict): + return { + key: _arrays_to_tuples(value) + for key, value in cast("dict[object, object]", obj).items() + } + return obj From eee768bdb419fe9c4f4f703da61956f54e176a77 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 10:19:35 +0200 Subject: [PATCH 03/48] =?UTF-8?q?feat(zarr-metadata):=20add=20model.=5Farr?= =?UTF-8?q?ay=20=E2=80=94=20array=20metadata=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_array.py | 348 ++++++++++++++++++ .../src/zarr_metadata/model/_validation.py | 7 +- 2 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_array.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py new file mode 100644 index 0000000000..ea2745284d --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -0,0 +1,348 @@ +"""In-memory models for Zarr array metadata documents.""" + +from __future__ import annotations + +import dataclasses +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, + arrays_to_tuples, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_metadata_field_v3, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata import ( + ArrayDimensionSeparatorV2, + ArrayMetadataV2, + ArrayMetadataV3, + ArrayOrderV2, + ExtensionFieldV3, + MetadataV3, + ) + from zarr_metadata._common import JSONValue + from zarr_metadata.v2 import CodecMetadataV2, DataTypeMetadataV2 + +ArrayMetadataStoreKeyV3 = Literal["zarr.json"] +ARRAY_METADATA_STORE_KEY_V3: Final[ArrayMetadataStoreKeyV3] = "zarr.json" + +ArrayMetadataStoreKeyV2 = Literal[".zarray"] +ARRAY_METADATA_STORE_KEY_V2: Final[ArrayMetadataStoreKeyV2] = ".zarray" + +AttributesStoreKeyV2 = Literal[".zattrs"] +ATTRIBUTES_STORE_KEY_V2: Final[AttributesStoreKeyV2] = ".zattrs" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrMetadataV3: + """A v3 metadata field in normalized form: a name plus a configuration. + + This is the in-memory model of `MetadataV3` (a bare name string or a + `{name, configuration}` mapping): the bare-name and missing-configuration + forms normalize to an empty configuration. + """ + + name: str + configuration: dict[str, JSONValue] + + def to_json(self) -> MetadataV3: + return {"name": self.name, "configuration": self.configuration} + + @classmethod + def from_json(cls, data: object) -> ZarrMetadataV3: + field = parse_metadata_field_v3(data) + if isinstance(field, str): + return cls(name=field, configuration={}) + configuration = arrays_to_tuples(dict(field.get("configuration", {}))) + return cls(name=field["name"], configuration=configuration) # type: ignore[arg-type] + + +class ArrayMetadataModelV3Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. + + Every key is optional and typed with the model's own (not serialized) + value types, so it describes valid keyword arguments to + `ArrayMetadataModelV3.update`. The `init=False` fields `zarr_format` and + `node_type` are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrMetadataV3 + chunk_grid: ZarrMetadataV3 + codecs: tuple[ZarrMetadataV3, ...] + chunk_key_encoding: ZarrMetadataV3 + dimension_names: tuple[str | None, ...] | None + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrMetadataV3, ...] + extra_fields: dict[str, ExtensionFieldV3] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ArrayMetadataModelV3: + """In-memory model of a v3 array metadata document. + + A canonical, lossless representation of the `zarr.json` content for an + array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, + `codecs`, `storage_transformers`) are held as `ZarrMetadataV3` name + + configuration pairs and are never interpreted; `fill_value` is held + verbatim in its JSON form. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["array"] = field(default="array", init=False) + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrMetadataV3 + chunk_grid: ZarrMetadataV3 + codecs: tuple[ZarrMetadataV3, ...] + chunk_key_encoding: ZarrMetadataV3 + dimension_names: tuple[str | None, ...] | None + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrMetadataV3, ...] + extra_fields: dict[str, ExtensionFieldV3] + + @classmethod + def create_default( + cls, **overrides: Unpack[ArrayMetadataModelV3Partial] + ) -> ArrayMetadataModelV3: + """ + Create a default (empty) v3 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` array — the array + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls( + shape=(), + fill_value=0, + data_type=ZarrMetadataV3(name="uint8", configuration={}), + chunk_grid=ZarrMetadataV3(name="regular", configuration={"chunk_shape": ()}), + codecs=(ZarrMetadataV3(name="bytes", configuration={}),), + chunk_key_encoding=ZarrMetadataV3(name="default", configuration={}), + dimension_names=None, + attributes={}, + storage_transformers=(), + extra_fields={}, + ) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ArrayMetadataModelV3Partial]) -> ArrayMetadataModelV3: + """ + Return a new `ArrayMetadataModelV3` with the given fields updated. + + Only the constructor-settable fields listed in + `ArrayMetadataModelV3Partial` can be updated; any attempt to update + other fields (including the fixed `zarr_format` / `node_type`) is + rejected at the type level. Each given field fully replaces its + previous value, including `extra_fields`. + + This is useful for test fixtures that want to override a few fields of a + base template without having to re-specify the entire document. + """ + return dataclasses.replace(self, **kwargs) + + def __post_init__(self) -> None: + if set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3): + raise ValueError("Extra fields cannot overlap with standard ArrayMetadataV3 fields") + + def to_json(self) -> ArrayMetadataV3: + out: ArrayMetadataV3 = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + "shape": self.shape, + "fill_value": self.fill_value, + "data_type": self.data_type.to_json(), + "chunk_grid": self.chunk_grid.to_json(), + "codecs": tuple(codec.to_json() for codec in self.codecs), + "chunk_key_encoding": self.chunk_key_encoding.to_json(), + } + if self.dimension_names is not None: + out["dimension_names"] = self.dimension_names + if len(self.attributes) > 0: + out["attributes"] = self.attributes + if len(self.storage_transformers) > 0: + out["storage_transformers"] = tuple( + transformer.to_json() for transformer in self.storage_transformers + ) + # Extra fields are the TypedDict's `extra_items` (PEP 728). Assign them + # by key rather than `out.update(**...)`: type checkers understand the + # indexed-write path against `extra_items`, but not the `update(**...)` + # overload. + for key, value in self.extra_fields.items(): + out[key] = value + return out + + @classmethod + def from_json(cls, data: object) -> ArrayMetadataModelV3: + parsed = parse_array_metadata_v3(arrays_to_tuples(data)) + extra_fields: dict[str, ExtensionFieldV3] = { + k: v # type: ignore[misc] + for k, v in parsed.items() + if k not in ARRAY_METADATA_STANDARD_KEYS_V3 + } + return cls( + shape=parsed["shape"], + fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + data_type=ZarrMetadataV3.from_json(parsed["data_type"]), + chunk_grid=ZarrMetadataV3.from_json(parsed["chunk_grid"]), + codecs=tuple(ZarrMetadataV3.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=ZarrMetadataV3.from_json(parsed["chunk_key_encoding"]), + dimension_names=parsed.get("dimension_names"), + attributes=dict(parsed.get("attributes", {})), + storage_transformers=tuple( + ZarrMetadataV3.from_json(t) for t in parsed.get("storage_transformers", ()) + ), + extra_fields=extra_fields, + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: + return cls.from_json(json.loads(mapping[ARRAY_METADATA_STORE_KEY_V3])) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + ARRAY_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") + } + + +class ArrayMetadataModelV2Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ArrayMetadataModelV2`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ArrayMetadataModelV2.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_v2_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + dtype: DataTypeMetadataV2 + chunks: tuple[int, ...] + fill_value: JSONValue + order: ArrayOrderV2 + compressor: CodecMetadataV2 | None + filters: tuple[CodecMetadataV2, ...] | None + dimension_separator: ArrayDimensionSeparatorV2 + attributes: dict[str, JSONValue] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ArrayMetadataModelV2: + """In-memory model of a v2 array metadata document. + + A canonical, lossless representation of the `.zarray` content plus the + sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are + held in their raw JSON forms and are never interpreted; `fill_value` is + held verbatim in its JSON form. + """ + + zarr_format: Literal[2] = field(default=2, init=False) + shape: tuple[int, ...] + dtype: DataTypeMetadataV2 + chunks: tuple[int, ...] + fill_value: JSONValue + order: ArrayOrderV2 + compressor: CodecMetadataV2 | None + filters: tuple[CodecMetadataV2, ...] | None + dimension_separator: ArrayDimensionSeparatorV2 = field(default="/") + attributes: dict[str, JSONValue] + + def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: + """ + Return a new `ArrayMetadataModelV2` with the given fields updated. + + Only the constructor-settable fields listed in + `ArrayMetadataModelV2Partial` can be updated; the fixed `zarr_format` is + rejected at the type level. Each given field fully replaces its previous + value. + """ + return dataclasses.replace(self, **kwargs) + + @classmethod + def create_default( + cls, **overrides: Unpack[ArrayMetadataModelV2Partial] + ) -> ArrayMetadataModelV2: + """ + Create a default (empty) v2 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` (`"|u1"`) array — the + array analog of `list()` returning `[]`. Any field can be overridden by + keyword (the same fields accepted by `update`). + """ + default = cls( + shape=(), + dtype="|u1", + chunks=(), + fill_value=0, + order="C", + compressor=None, + filters=None, + attributes={}, + ) + return default.update(**overrides) + + def to_json(self) -> ArrayMetadataV2: + out: ArrayMetadataV2 = { + "zarr_format": self.zarr_format, + "shape": self.shape, + "dtype": self.dtype, + "order": self.order, + "chunks": self.chunks, + "fill_value": self.fill_value, + "dimension_separator": self.dimension_separator, + "attributes": self.attributes, + "compressor": self.compressor, + "filters": self.filters, + } + return out + + @classmethod + def from_json(cls, data: object) -> ArrayMetadataModelV2: + parsed = parse_array_metadata_v2(arrays_to_tuples(data)) + return cls( + shape=parsed["shape"], + dtype=parsed["dtype"], + chunks=parsed["chunks"], + fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + order=parsed["order"], + compressor=parsed["compressor"], + filters=parsed["filters"], + dimension_separator=parsed.get("dimension_separator", "/"), + attributes=dict(parsed.get("attributes", {})), + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: + zarray = json.loads(mapping[ARRAY_METADATA_STORE_KEY_V2]) + zattrs: dict[str, JSONValue] = ( + json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + if ATTRIBUTES_STORE_KEY_V2 in mapping + else {} + ) + return cls.from_json({**zarray, "attributes": zattrs}) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zarray` + # document must exclude them. + zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} + return { + ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8"), + ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), + } diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index d0e30e1cda..25f3422c50 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -259,13 +259,12 @@ def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: return cast(ArrayMetadataV2, value) -def _arrays_to_tuples(obj: object) -> object: +def arrays_to_tuples(obj: object) -> object: """Recursively convert every list in a JSON-decoded structure to a tuple.""" if isinstance(obj, list): - return tuple(_arrays_to_tuples(item) for item in cast("list[object]", obj)) + return tuple(arrays_to_tuples(item) for item in cast("list[object]", obj)) if isinstance(obj, dict): return { - key: _arrays_to_tuples(value) - for key, value in cast("dict[object, object]", obj).items() + key: arrays_to_tuples(value) for key, value in cast("dict[object, object]", obj).items() } return obj From 6b391474ea5be5c9ef41ee791bcfa502b8c75640 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 10:23:53 +0200 Subject: [PATCH 04/48] test(zarr-metadata): port array model test suite Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/__init__.py | 60 + .../zarr-metadata/tests/model/__init__.py | 0 packages/zarr-metadata/tests/model/_cases.py | 45 + .../zarr-metadata/tests/model/test_array.py | 1019 +++++++++++++++++ 4 files changed, 1124 insertions(+) create mode 100644 packages/zarr-metadata/tests/model/__init__.py create mode 100644 packages/zarr-metadata/tests/model/_cases.py create mode 100644 packages/zarr-metadata/tests/model/test_array.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index e1382a6f0a..8776b4fab3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -1 +1,61 @@ """In-memory models for Zarr metadata documents.""" + +from zarr_metadata.model._array import ( + ARRAY_METADATA_STORE_KEY_V2, + ARRAY_METADATA_STORE_KEY_V3, + ATTRIBUTES_STORE_KEY_V2, + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + ZarrMetadataV3, +) +from zarr_metadata.model._validation import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V2, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + is_array_metadata_v2, + is_array_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_json, + validate_metadata_field_v3, +) + +__all__ = [ + "ARRAY_METADATA_OPTIONAL_KEYS_V3", + "ARRAY_METADATA_REQUIRED_KEYS_V2", + "ARRAY_METADATA_REQUIRED_KEYS_V3", + "ARRAY_METADATA_STANDARD_KEYS_V3", + "ARRAY_METADATA_STORE_KEY_V2", + "ARRAY_METADATA_STORE_KEY_V3", + "ATTRIBUTES_STORE_KEY_V2", + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", + "MetadataValidationError", + "ValidationProblem", + "ZarrMetadataV3", + "is_array_metadata_v2", + "is_array_metadata_v3", + "is_json", + "is_metadata_field_v3", + "parse_array_metadata_v2", + "parse_array_metadata_v3", + "parse_json", + "parse_metadata_field_v3", + "validate_array_metadata_v2", + "validate_array_metadata_v3", + "validate_json", + "validate_metadata_field_v3", +] diff --git a/packages/zarr-metadata/tests/model/__init__.py b/packages/zarr-metadata/tests/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/model/_cases.py b/packages/zarr-metadata/tests/model/_cases.py new file mode 100644 index 0000000000..f925c69dd6 --- /dev/null +++ b/packages/zarr-metadata/tests/model/_cases.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, TypeVar + +import pytest + +if TYPE_CHECKING: + from contextlib import AbstractContextManager + +TIn = TypeVar("TIn") +TOut = TypeVar("TOut") + + +@dataclass(frozen=True) +class Expect(Generic[TIn, TOut]): + """A test case with explicit input, expected output, and a human-readable id.""" + + input: TIn + output: TOut + id: str + + +@dataclass(frozen=True) +class ExpectFail(Generic[TIn]): + """A test case that should raise an exception. + + `msg` is a regex matched against the exception text (pytest's native + `match=` semantics). Leave it `None` to assert only the exception type. Set + `escape=True` when `msg` is a literal that contains regex metacharacters + such as `(`, `[`, or `.`; `escape` has no effect when `msg` is `None`. + """ + + input: TIn + exception: type[Exception] + id: str + msg: str | None = None + escape: bool = False + + def raises(self) -> AbstractContextManager[pytest.ExceptionInfo[Exception]]: + if self.msg is None: + return pytest.raises(self.exception) + pattern = re.escape(self.msg) if self.escape else self.msg + return pytest.raises(self.exception, match=pattern) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py new file mode 100644 index 0000000000..80dac709de --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -0,0 +1,1019 @@ +"""Tests for the metadata models in ``zarr_metadata.model``.""" + +import dataclasses +import json +from collections.abc import Callable +from typing import TYPE_CHECKING + +import pytest + +from tests.model._cases import Expect, ExpectFail +from zarr_metadata.model import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + MetadataValidationError, + ValidationProblem, + ZarrMetadataV3, + is_array_metadata_v2, + is_array_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_json, + validate_metadata_field_v3, +) +from zarr_metadata.model._validation import _prefix, arrays_to_tuples + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.v2 import CodecMetadataV2 + +# --- public exports -------------------------------------------------------- + + +def test_guards_exported_from_package() -> None: + """The wire-type guard/parser functions are exported from the package.""" + import zarr_metadata.model + + for name in ( + "is_json", + "parse_json", + "is_metadata_field_v3", + "parse_metadata_field_v3", + "is_array_metadata_v3", + "parse_array_metadata_v3", + "is_array_metadata_v2", + "parse_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_validation_diagnostics_exported_from_package() -> None: + """The validation-diagnostic types and validators are exported from the package.""" + import zarr_metadata.model + + for name in ( + "ValidationProblem", + "MetadataValidationError", + "validate_json", + "validate_metadata_field_v3", + "validate_array_metadata_v3", + "validate_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_expect_expectfail_smoke() -> None: + """The Expect/ExpectFail test-case dataclasses behave as expected.""" + e = Expect(input=1, output=2, id="x") + assert (e.input, e.output, e.id) == (1, 2, "x") + f = ExpectFail(input=1, exception=ValueError, id="y", msg="boom") + with f.raises(): + raise ValueError("boom") + + +def test_v3_from_json_error_lists_all_problems() -> None: + """A malformed v3 document surfaces every problem via MetadataValidationError.problems.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + del doc["shape"] + doc["data_type"] = 5 + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_json(doc) + locs = {p.loc for p in exc_info.value.problems} + assert ("shape",) in locs + assert ("data_type",) in locs + + +# --- JSON type / fill_value contract --------------------------------------- + + +def test_json_value_type_accepts_json_shapes() -> None: + # JSONValue is the package's public JSON type alias; assigning JSON-shaped + # values to it is valid. + """The JSONValue type alias accepts JSON-shaped values.""" + value: JSONValue = {"a": [1, 2.0, "x", True, None]} + assert value == {"a": [1, 2.0, "x", True, None]} + + +def test_string_nan_fill_value_roundtrips() -> None: + # Non-finite floats are represented as the spec strings ("NaN", "Infinity", + # "-Infinity") by the caller — the metadata layer does not interpret dtypes. + # The string form round-trips cleanly under default dataclass equality, + # unlike a raw float('nan') (which is an invalid fill_value the caller must + # not pass). + """A string 'NaN' fill_value round-trips cleanly (non-finite floats are the caller's responsibility).""" + m = ArrayMetadataModelV3.create_default(fill_value="NaN") + assert ArrayMetadataModelV3.from_json(m.to_json()) == m + assert ArrayMetadataModelV3.from_json(m.to_json()).fill_value == "NaN" + + +# --- ZarrMetadataV3.to_json ------------------------------------------------ + +ZARR_TO_JSON_CASES = [ + Expect( + ZarrMetadataV3(name="regular", configuration={"chunk_shape": [1]}), + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + id="with-configuration", + ), + Expect( + ZarrMetadataV3(name="bytes", configuration={}), + {"name": "bytes", "configuration": {}}, + id="without-configuration", + ), +] + + +@pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_to_json(case: Expect[ZarrMetadataV3, dict[str, object]]) -> None: + """ZarrMetadataV3.to_json emits the canonical object form.""" + assert case.input.to_json() == case.output + + +# --- ZarrMetadataV3.from_json ----------------------------------------------- + +ZARR_FROM_JSON_CASES = [ + Expect("bytes", ZarrMetadataV3(name="bytes", configuration={}), id="bare-string"), + Expect( + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + ZarrMetadataV3(name="regular", configuration={"chunk_shape": (1,)}), + id="object-with-config", + ), + Expect( + {"name": "bytes"}, + ZarrMetadataV3(name="bytes", configuration={}), + id="object-without-config", + ), +] + + +@pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrMetadataV3]) -> None: + """ZarrMetadataV3.from_json parses both the bare-string and object forms.""" + assert ZarrMetadataV3.from_json(case.input) == case.output + + +# --- V3 baseline ----------------------------------------------------------- + + +def test_v3_to_json_includes_required_fields() -> None: + """V3 to_json emits all required fields with the expected values.""" + out = ArrayMetadataModelV3.create_default( + shape=(10,), data_type=ZarrMetadataV3(name="int32", configuration={}) + ).to_json() + assert out["zarr_format"] == 3 + assert out["node_type"] == "array" + assert out["shape"] == (10,) + assert out["fill_value"] == 0 + assert out["data_type"] == {"name": "int32", "configuration": {}} + assert out["codecs"] == ({"name": "bytes", "configuration": {}},) + + +def test_v3_dimension_names_included_when_present() -> None: + """V3 to_json includes dimension_names when they are set.""" + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default(dimension_names=("x",)).to_json() + ) + assert out["dimension_names"] == ("x",) + + +def test_v3_dimension_names_omitted_when_none() -> None: + """V3 to_json omits dimension_names when they are None.""" + out = ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + assert "dimension_names" not in out + + +# --- BUG 1: attributes gated on dimension_names ---------------------------- + + +def test_v3_attributes_included_when_dimension_names_is_none() -> None: + """Attributes must be emitted regardless of dimension_names. + + Regression: attributes were gated on ``dimension_names is not None``, + so non-empty attributes were silently dropped when there were no + dimension names. + """ + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default( + dimension_names=None, attributes={"foo": "bar"} + ).to_json() + ) + assert out["attributes"] == {"foo": "bar"} + + +# --- BUG 2: single storage transformer dropped ----------------------------- + + +def test_v3_single_storage_transformer_included() -> None: + """A single storage transformer must be emitted. + + Regression: the guard used ``> 1`` instead of ``> 0``, dropping a + lone storage transformer. + """ + st = ZarrMetadataV3(name="some_transformer", configuration={}) + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() + ) + assert out["storage_transformers"] == ({"name": "some_transformer", "configuration": {}},) + + +def test_v3_no_storage_transformers_omitted() -> None: + """V3 to_json omits storage_transformers when empty.""" + out = ArrayMetadataModelV3.create_default(storage_transformers=()).to_json() + assert "storage_transformers" not in out + + +# --- V3 extra fields ------------------------------------------------------- + + +def test_v3_extra_fields_merged() -> None: + """V3 to_json merges extra_fields into the top-level document.""" + out = ArrayMetadataModelV3.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + assert out["my_ext"] == {"must_understand": False} + + +def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: + """Constructing a V3 model with an extra field that collides with a standard key is rejected.""" + with pytest.raises(ValueError): + ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + + +# --- V3 key/value ---------------------------------------------------------- + + +def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: + """V3 to_key_value produces valid JSON bytes under the zarr.json key.""" + kv = ArrayMetadataModelV3.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {"zarr.json"} + parsed = json.loads(kv["zarr.json"].decode("utf-8")) + assert parsed["zarr_format"] == 3 + assert parsed["attributes"] == {"a": 1} + + +# --- V3 standard-key sets -------------------------------------------------- + + +def test_standard_keys_is_union_of_required_and_optional() -> None: + """The standard-key set is the union of the required and optional key sets.""" + assert ( + ARRAY_METADATA_STANDARD_KEYS_V3 + == ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 + ) + + +def test_standard_keys_contains_known_fields_and_excludes_extensions() -> None: + """The standard-key set contains known fields and excludes extension keys.""" + assert { + "zarr_format", + "node_type", + "shape", + "codecs", + } <= ARRAY_METADATA_STANDARD_KEYS_V3 + assert "my_ext" not in ARRAY_METADATA_STANDARD_KEYS_V3 + + +# --- create_default -------------------------------------------------------- + + +def test_v3_create_default_is_valid_empty_array() -> None: + """V3 create_default builds a structurally valid empty array that round-trips.""" + m = ArrayMetadataModelV3.create_default() + assert m.shape == () + assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + assert m.fill_value == 0 + assert m.attributes == {} + assert m.extra_fields == {} + # the default document is structurally valid and round-trips + assert validate_array_metadata_v3(m.to_json()) == [] + assert ArrayMetadataModelV3.from_json(m.to_json()) == m + + +def test_v3_create_default_applies_overrides() -> None: + """V3 create_default applies keyword overrides over the defaults.""" + m = ArrayMetadataModelV3.create_default(shape=(4, 4), attributes={"a": 1}) + assert m.shape == (4, 4) + assert m.attributes == {"a": 1} + # un-overridden fields keep their defaults + assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + + +def test_v2_create_default_is_valid_empty_array() -> None: + """V2 create_default builds a structurally valid empty array that round-trips.""" + m = ArrayMetadataModelV2.create_default() + assert m.shape == () + assert m.chunks == () + assert m.fill_value == 0 + assert m.compressor is None + assert m.filters is None + assert m.attributes == {} + assert validate_array_metadata_v2(m.to_json()) == [] + assert ArrayMetadataModelV2.from_json(m.to_json()) == m + + +def test_v2_create_default_applies_overrides() -> None: + """V2 create_default applies keyword overrides over the defaults.""" + m = ArrayMetadataModelV2.create_default(shape=(8,), attributes={"k": "v"}) + assert m.shape == (8,) + assert m.attributes == {"k": "v"} + assert m.dtype == "|u1" # default dtype unchanged + + +# --- V3 update ------------------------------------------------------------- + +# Cluster 3: update same-shape pairs across versions — parametrized + +UPDATE_NEW_INSTANCE_PARAMS = [ + pytest.param(ArrayMetadataModelV3, id="v3"), + pytest.param(ArrayMetadataModelV2, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NEW_INSTANCE_PARAMS) +def test_update_returns_new_instance( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], +) -> None: + """update returns a new instance with the field replaced, leaving the original unchanged.""" + base = model_cls.create_default(shape=(10,)) + updated = base.update(shape=(20,)) + assert updated.shape == (20,) + assert base.shape == (10,) # original unchanged + assert isinstance(updated, model_cls) + + +UPDATE_NO_ARGS_PARAMS = [ + pytest.param(ArrayMetadataModelV3, id="v3"), + pytest.param(ArrayMetadataModelV2, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NO_ARGS_PARAMS) +def test_update_no_args_returns_equal_model( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], +) -> None: + """update with no arguments returns a model equal to the original.""" + base = model_cls.create_default() + assert base.update() == base + + +# V3-only update tests — kept direct (extra_fields is v3-specific) + + +def test_update_can_replace_extra_fields() -> None: + """update can replace the extra_fields mapping.""" + base = ArrayMetadataModelV3.create_default(extra_fields={}) + updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) + assert updated.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_update_replaces_extra_fields_rather_than_merging() -> None: + """update replaces extra_fields wholesale rather than merging.""" + base = ArrayMetadataModelV3.create_default(extra_fields={"a": {"must_understand": False}}) + updated = base.update(extra_fields={"b": {"must_understand": True}}) + assert updated.extra_fields == {"b": {"must_understand": True}} + + +def test_partial_keys_match_settable_model_fields() -> None: + """The partial TypedDict must list exactly the constructor-settable fields. + + Guards against drift: adding/removing a settable field on the model + without updating ``ArrayMetadataModelV3Partial`` fails here. + """ + settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV3) if f.init} + assert set(ArrayMetadataModelV3Partial.__annotations__) == settable + + +# --- V2 model -------------------------------------------------------------- + + +def test_v2_partial_keys_match_settable_model_fields() -> None: + """The v2 partial TypedDict must list exactly the settable fields.""" + settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV2) if f.init} + assert set(ArrayMetadataModelV2Partial.__annotations__) == settable + + +def test_v2_to_key_value_splits_zarray_and_zattrs() -> None: + """V2 to_key_value splits the document into .zarray and .zattrs.""" + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {".zarray", ".zattrs"} + zarray = json.loads(kv[".zarray"].decode("utf-8")) + zattrs = json.loads(kv[".zattrs"].decode("utf-8")) + assert zarray["zarr_format"] == 2 + assert zattrs == {"a": 1} + + +def test_v2_zarray_excludes_attributes() -> None: + """The on-disk ``.zarray`` document must not contain user attributes. + + In v2, attributes live only in the sibling ``.zattrs`` file. The bundled + ``ArrayMetadataV2`` / ``to_json()`` carry attributes for convenience, but + ``to_key_value()`` must split them out. + """ + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + zarray = json.loads(kv[".zarray"].decode("utf-8")) + assert "attributes" not in zarray + + +def test_v2_to_json_still_includes_attributes() -> None: + """``to_json()`` is the bundled in-memory form and keeps attributes.""" + out: dict[str, object] = dict( + ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() + ) + assert out["attributes"] == {"a": 1} + + +# --- arrays_to_tuples helper ---------------------------------------------- + +ARRAYS_TO_TUPLES_CASES = [ + Expect([1, 2, 3], (1, 2, 3), id="top-level-list"), + Expect({"a": [1, [2, 3]], "b": "x"}, {"a": (1, (2, 3)), "b": "x"}, id="nested-in-dict"), + Expect(5, 5, id="scalar-int"), + Expect("s", "s", id="scalar-str"), + Expect(None, None, id="scalar-none"), + Expect( + {"name": "bytes", "configuration": {"nums": [1, 2]}}, + {"name": "bytes", "configuration": {"nums": (1, 2)}}, + id="dict-keys-preserved", + ), +] + + +@pytest.mark.parametrize("case", ARRAYS_TO_TUPLES_CASES, ids=lambda c: c.id) +def test_arrays_to_tuples(case: Expect[object, object]) -> None: + """arrays_to_tuples recursively converts JSON arrays to tuples.""" + assert arrays_to_tuples(case.input) == case.output + + +# --- ArrayMetadataModelV3.from_json ---------------------------------------- + + +def test_v3_from_json_reconstructs_required_fields() -> None: + """V3 from_json reconstructs the required fields from a document.""" + doc = ArrayMetadataModelV3.create_default( + shape=(7,), + attributes={"a": 1}, + data_type=ZarrMetadataV3(name="int32", configuration={}), + ).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.shape == (7,) + assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.attributes == {"a": 1} + + +def test_v3_from_json_defaults_for_omitted_optionals() -> None: + """V3 from_json supplies defaults for omitted optional fields.""" + doc = ArrayMetadataModelV3.create_default( + attributes={}, storage_transformers=(), dimension_names=None + ).to_json() + # to_json omits these entirely; from_json must restore defaults + model = ArrayMetadataModelV3.from_json(doc) + assert model.attributes == {} + assert model.storage_transformers == () + assert model.dimension_names is None + + +def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: + """V3 from_json routes unknown top-level keys into extra_fields.""" + doc = ArrayMetadataModelV3.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: + """V3 from_json keeps standard keys out of extra_fields.""" + doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields == {} + + +def test_v3_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V3 from_json converts nested arrays in attributes into tuples.""" + doc = ArrayMetadataModelV3.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.attributes == {"scale": ((1, 2), (3, 4))} + + +# --- ArrayMetadataModelV3.from_key_value ---------------------------------- + + +def test_v3_from_key_value_parses_zarr_json() -> None: + """V3 from_key_value parses the zarr.json entry into a model.""" + kv = ArrayMetadataModelV3.create_default(shape=(3,)).to_key_value() + model = ArrayMetadataModelV3.from_key_value(kv) + assert model.shape == (3,) + + +# --- Cluster 2: from_key_value missing-key raises (parametrized) ----------- + +FROM_KEY_VALUE_MISSING_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ExpectFail({}, KeyError, id="v3-missing-zarr-json"), + id="v3-missing-zarr-json", + ), + pytest.param( + ArrayMetadataModelV2, + ExpectFail({}, KeyError, id="v2-missing-zarray"), + id="v2-missing-zarray", + ), +] + + +@pytest.mark.parametrize(("model_cls", "case"), FROM_KEY_VALUE_MISSING_PARAMS) +def test_from_key_value_missing_key_raises( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + case: ExpectFail[dict[str, bytes]], +) -> None: + """from_key_value raises KeyError when the required store key is absent.""" + with case.raises(): + model_cls.from_key_value(case.input) + + +# --- Cluster 1: round-trips (model → json → model, parametrized) ----------- + +ROUNDTRIP_MODEL_JSON_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default( + attributes={"a": 1}, + dimension_names=("x",), + storage_transformers=(ZarrMetadataV3(name="t", configuration={}),), + extra_fields={"ext": {"must_understand": False}}, + ), + id="v3-full", + ), + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default( + attributes={}, + dimension_names=None, + storage_transformers=(), + extra_fields={}, + ), + id="v3-empty-optionals", + ), + pytest.param( + ArrayMetadataModelV2, + ArrayMetadataModelV2.create_default(attributes={"a": 1}, filters=None, compressor=None), + id="v2-basic", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_MODEL_JSON_PARAMS) +def test_roundtrip_model_json_model( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model: ArrayMetadataModelV3 | ArrayMetadataModelV2, +) -> None: + """A model round-trips through to_json/from_json back to an equal model.""" + assert model_cls.from_json(model.to_json()) == model + + +# --- Round-trips (model → key_value → model, parametrized) ----------------- + +ROUNDTRIP_KEY_VALUE_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default(attributes={"a": 1}), + id="v3", + ), + pytest.param( + ArrayMetadataModelV2, + ArrayMetadataModelV2.create_default(attributes={"a": 1}), + id="v2", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_KEY_VALUE_PARAMS) +def test_roundtrip_via_key_value( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model: ArrayMetadataModelV3 | ArrayMetadataModelV2, +) -> None: + """A model round-trips through to_key_value/from_key_value back to an equal model.""" + assert model_cls.from_key_value(model.to_key_value()) == model + + +# --- Round-trips (json → model → json, direction distinct — kept direct) --- + + +def test_v3_roundtrip_json_model_json() -> None: + """A v3 document round-trips through from_json/to_json back to an equal document.""" + doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + assert ArrayMetadataModelV3.from_json(doc).to_json() == doc + + +def test_v2_roundtrip_json_model_json() -> None: + """A v2 document round-trips through from_json/to_json back to an equal document.""" + doc = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() + assert ArrayMetadataModelV2.from_json(doc).to_json() == doc + + +def test_v3_parser_accepts_bare_string_data_type() -> None: + """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" + doc = ArrayMetadataModelV3.create_default().to_json() + doc["data_type"] = "int32" # bare-string form, not canonical object form + model = ArrayMetadataModelV3.from_json(doc) + # parses correctly, re-serializes to canonical object form + assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} + + +def test_v2_roundtrip_with_compressor_and_filters() -> None: + # Non-None compressor/filters must round-trip; extra assertion on .compressor. + """A v2 model with non-None compressor and filters round-trips.""" + compressor: CodecMetadataV2 = {"id": "blosc", "clevel": 5} + filters: tuple[CodecMetadataV2, ...] = ({"id": "delta"},) + m = ArrayMetadataModelV2.create_default(compressor=compressor, filters=filters) + restored = ArrayMetadataModelV2.from_json(m.to_json()) + assert restored == m + assert restored.compressor == {"id": "blosc", "clevel": 5} + + +# --- ArrayMetadataModelV2.from_json ---------------------------------------- + + +def test_v2_from_json_reconstructs_fields() -> None: + """V2 from_json reconstructs the fields from a document.""" + doc = ArrayMetadataModelV2.create_default( + shape=(4,), attributes={"a": 1}, dtype=" None: + """V2 from_json defaults attributes to empty when the key is absent.""" + doc = ArrayMetadataModelV2.create_default().to_json() + del doc["attributes"] + model = ArrayMetadataModelV2.from_json(doc) + assert model.attributes == {} + + +# --- ArrayMetadataModelV2.from_key_value -------------------------------- + + +def test_v2_from_key_value_remerges_zattrs() -> None: + """V2 from_key_value re-merges .zattrs back into attributes.""" + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() + model = ArrayMetadataModelV2.from_key_value(kv) + assert model.attributes == {"a": 1} + assert model.shape == (10,) + + +def test_v2_from_key_value_absent_zattrs_gives_empty_attributes() -> None: + """V2 from_key_value yields empty attributes when .zattrs is absent.""" + kv: dict[str, bytes] = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) + del kv[".zattrs"] + model = ArrayMetadataModelV2.from_key_value(kv) + assert model.attributes == {} + + +def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V2 from_json converts nested arrays in attributes into tuples.""" + doc = ArrayMetadataModelV2.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() + model = ArrayMetadataModelV2.from_json(doc) + assert model.attributes == {"axes": ((0, 1), (2, 3))} + + +# --- scalar wire-type guards (is_/validate_/parse_) ------------------------ +# +# Each value is modelled once as Expect[object, frozenset[tuple[str | int, ...]]] +# where `output` is the set of expected problem locs validate_* must report — +# frozenset() means VALID. Valid iff output == frozenset(). + +JSON_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("s", frozenset(), id="str"), + Expect(1, frozenset(), id="int"), + Expect(1.5, frozenset(), id="float"), + Expect(True, frozenset(), id="bool"), + Expect(None, frozenset(), id="none"), + Expect({"a": [1, {"b": None}], "c": "x"}, frozenset(), id="nested-containers"), + Expect((1, 2, 3), frozenset(), id="tuple-array"), + Expect(float("nan"), frozenset(), id="nan"), + Expect(float("inf"), frozenset(), id="inf"), + Expect(object(), frozenset({()}), id="object"), + Expect(b"abc", frozenset({()}), id="bytes"), + Expect(bytearray(b"abc"), frozenset({()}), id="bytearray"), + Expect({1: "x"}, frozenset({()}), id="non-str-key"), + Expect([1, object()], frozenset({(1,)}), id="non-json-list-item"), + Expect({"ok": object()}, frozenset({("ok",)}), id="non-json-value"), +] + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_json reports whether a value is JSON-serializable.""" + assert is_json(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """validate_json reports the problems (and their locs) for a value.""" + problems = validate_json(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """parse_json returns valid JSON values and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_json(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_json(case.input) + + +def test_validate_json_reports_json_in_message() -> None: + """validate_json's message for a non-JSON value mentions JSON.""" + problems = validate_json(object()) + assert problems[0].loc == () + assert "JSON" in problems[0].message + + +METADATA_FIELD_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("bytes", frozenset(), id="bare-string"), + Expect({"name": "x", "configuration": {"a": 1}}, frozenset(), id="named-config"), + Expect({"name": "bytes"}, frozenset(), id="name-only"), + Expect(5, frozenset({()}), id="not-str-or-mapping"), + Expect({"configuration": {}}, frozenset({("name",)}), id="missing-name"), + Expect({"name": 3}, frozenset({("name",)}), id="non-str-name"), + Expect( + {"name": "x", "configuration": [1]}, + frozenset({("configuration",)}), + id="config-not-mapping", + ), + Expect( + {"name": "x", "configuration": {1: "y"}}, + frozenset({("configuration",)}), + id="config-non-str-key", + ), +] + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_metadata_field_v3(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_metadata_field_v3 reports whether a value is a v3 metadata field.""" + assert is_metadata_field_v3(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """validate_metadata_field_v3 reports the problems for a metadata-field value.""" + problems = validate_metadata_field_v3(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """parse_metadata_field_v3 returns valid fields and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_metadata_field_v3(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_metadata_field_v3(case.input) + + +# --- array-document wire-type guards (is_/validate_/parse_) ---------------- +# +# Each case starts from a valid document (built by `make`) and applies a +# mutation. `expected_locs` are loc paths `validate_*` must report for the +# invalid cases (a subset check, so accumulation of OTHER problems is allowed). + + +def _build_v3(**overrides: object) -> dict[str, object]: + return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) # type: ignore[arg-type] + + +def _build_v2(**overrides: object) -> dict[str, object]: + return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) # type: ignore[arg-type] + + +def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: + def _factory() -> dict: + doc = build() + mutate(doc) + return doc + + return _factory + + +def _del(key: str) -> Callable[[dict], object]: + return lambda doc: doc.pop(key) + + +def _set(key: str, value: object) -> Callable[[dict], object]: + return lambda doc: doc.__setitem__(key, value) + + +V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v3, frozenset(), id="valid"), + Expect( + lambda: _build_v3(attributes={"a": 1}, dimension_names=("x",)), + frozenset(), + id="valid-with-attributes-and-dim-names", + ), + Expect( + lambda: _build_v3(extra_fields={"my_ext": {"must_understand": False}}), + frozenset(), + id="valid-with-extra-fields", + ), + Expect(_mutate(_build_v3, _del("shape")), frozenset({("shape",)}), id="missing-shape"), + Expect( + _mutate(_build_v3, _set("data_type", 5)), + frozenset({("data_type",)}), + id="bad-data-type", + ), + Expect( + _mutate(_build_v3, _set("shape", "not-a-shape")), + frozenset({("shape",)}), + id="shape-not-sequence", + ), + Expect( + _mutate(_build_v3, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="shape-non-int-item", + ), + Expect( + _mutate(_build_v3, _set("codecs", (5,))), + frozenset({("codecs", 0)}), + id="bad-codec-entry", + ), + Expect(lambda: [1, 2, 3], frozenset({()}), id="non-mapping-list"), + Expect(lambda: "nope", frozenset({()}), id="non-mapping-str"), + Expect( + _mutate(_mutate(_build_v3, _del("shape")), _set("data_type", 5)), + frozenset({("shape",), ("data_type",)}), + id="missing-shape-and-bad-data-type", + ), +] + +V2_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v2, frozenset(), id="valid"), + Expect(lambda: _build_v2(attributes={"a": 1}), frozenset(), id="valid-with-attributes"), + Expect( + lambda: _build_v2(compressor=None, filters=None), + frozenset(), + id="valid-none-compressor-filters", + ), + Expect( + _mutate(_build_v2, _del("chunks")), + frozenset({("chunks",)}), + id="missing-chunks", + ), + Expect( + _mutate(_build_v2, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="bad-shape", + ), + Expect( + _mutate(_mutate(_build_v2, _del("chunks")), _set("shape", [1, "x"])), + frozenset({("chunks",), ("shape",)}), + id="missing-chunks-and-bad-shape", + ), +] + +ALL_DOC_CASES = [ + *( + pytest.param( + is_array_metadata_v3, + validate_array_metadata_v3, + parse_array_metadata_v3, + c, + id=f"v3-{c.id}", + ) + for c in V3_DOC_CASES + ), + *( + pytest.param( + is_array_metadata_v2, + validate_array_metadata_v2, + parse_array_metadata_v2, + c, + id=f"v2-{c.id}", + ) + for c in V2_DOC_CASES + ), +] + + +@pytest.mark.parametrize(("is_fn", "validate_fn", "parse_fn", "case"), ALL_DOC_CASES) +def test_array_metadata_guards( + is_fn: Callable[[object], bool], + validate_fn: Callable[[object], list[ValidationProblem]], + parse_fn: Callable[[object], object], + case: Expect[Callable[[], object], frozenset[tuple[str | int, ...]]], +) -> None: + """is_/validate_/parse_ array-metadata guards agree on validity and locs for each case.""" + doc = case.input() + valid = case.output == frozenset() + assert is_fn(doc) is valid + problems = validate_fn(doc) + assert (problems == []) is valid + assert {p.loc for p in problems} >= case.output + if valid: + assert parse_fn(doc) is doc + else: + with pytest.raises(MetadataValidationError): + parse_fn(doc) + + +# --- strict from_json validation ------------------------------------------- + + +FROM_JSON_REJECT_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ExpectFail(lambda: {"zarr_format": 3}, MetadataValidationError, id="x"), + id="v3-missing-required", + ), + pytest.param( + ArrayMetadataModelV3, + ExpectFail(_mutate(_build_v3, _set("data_type", 5)), MetadataValidationError, id="x"), + id="v3-bad-field-type", + ), + pytest.param( + ArrayMetadataModelV2, + ExpectFail(lambda: {"zarr_format": 2}, MetadataValidationError, id="x"), + id="v2-missing-required", + ), + pytest.param( + ZarrMetadataV3, + ExpectFail(lambda: 5, MetadataValidationError, id="x"), + id="zarr-metadata-bad-input", + ), +] + + +@pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) +def test_from_json_rejects_malformed( + model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | ZarrMetadataV3], + case: ExpectFail[Callable[[], object]], +) -> None: + """from_json raises MetadataValidationError on a malformed document.""" + with case.raises(): + model.from_json(case.input()) + + +# --- ValidationProblem / MetadataValidationError / _prefix ----------------- +# Small structural tests — not "parametrize over inputs" shaped, kept direct. + + +def test_validation_problem_str_with_loc() -> None: + """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" + p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str") + assert str(p) == "codecs.0.name: expected str" + + +def test_validation_problem_str_empty_loc() -> None: + """ValidationProblem.__str__ renders an empty loc as .""" + p = ValidationProblem(loc=(), message="not a mapping") + assert str(p) == ": not a mapping" + + +def test_validation_problem_is_frozen() -> None: + """ValidationProblem is immutable (frozen dataclass).""" + p = ValidationProblem(loc=("shape",), message="x") + with pytest.raises(dataclasses.FrozenInstanceError): + p.message = "y" # type: ignore[misc] + + +def test_metadata_validation_error_holds_problems() -> None: + """MetadataValidationError carries its problem list and renders them in its message.""" + problems = [ + ValidationProblem(loc=("shape",), message="missing required key"), + ValidationProblem(loc=("data_type",), message="expected a metadata field"), + ] + err = MetadataValidationError(problems) + assert err.problems == problems + assert "shape: missing required key" in str(err) + assert "data_type: expected a metadata field" in str(err) + + +def test_prefix_prepends_loc_head() -> None: + """_prefix prepends a loc head to each problem's loc.""" + problems = [ValidationProblem(loc=("name",), message="expected str")] + prefixed = _prefix(0, problems) + assert prefixed == [ValidationProblem(loc=(0, "name"), message="expected str")] From 9f32f14f75f9e97204c01e8ad28b34f6515e2a44 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:24:31 +0200 Subject: [PATCH 05/48] feat(zarr-metadata): add group metadata models Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_group.py | 370 ++++++++++++++++++ .../src/zarr_metadata/model/_validation.py | 89 ++++- .../zarr-metadata/tests/model/test_group.py | 137 +++++++ 3 files changed, 595 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_group.py create mode 100644 packages/zarr-metadata/tests/model/test_group.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py new file mode 100644 index 0000000000..8eeaed1a89 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -0,0 +1,370 @@ +"""In-memory models for Zarr group and consolidated metadata documents.""" + +from __future__ import annotations + +import dataclasses +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._array import ( + ATTRIBUTES_STORE_KEY_V2, + ArrayMetadataModelV3, +) +from zarr_metadata.model._validation import ( + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + parse_group_metadata_v2, + parse_group_metadata_v3, +) + +if TYPE_CHECKING: + from zarr_metadata import ( + ConsolidatedMetadataV3, + ExtensionFieldV3, + GroupMetadataV2, + GroupMetadataV3, + ) + from zarr_metadata._common import JSONValue + +GroupMetadataStoreKeyV3 = Literal["zarr.json"] +GROUP_METADATA_STORE_KEY_V3: Final[GroupMetadataStoreKeyV3] = "zarr.json" + +GroupMetadataStoreKeyV2 = Literal[".zgroup"] +GROUP_METADATA_STORE_KEY_V2: Final[GroupMetadataStoreKeyV2] = ".zgroup" + +ConsolidatedMetadataStoreKeyV2 = Literal[".zmetadata"] +CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ConsolidatedMetadataStoreKeyV2] = ".zmetadata" + +# The key under which consolidated metadata is embedded in a v3 group document. +# This is a reference-implementation convention (not a spec artifact), stored +# as an extension field on the group's `zarr.json`. +CONSOLIDATED_METADATA_KEY_V3: Final = "consolidated_metadata" + + +class GroupMetadataModelV3Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `GroupMetadataModelV3`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `GroupMetadataModelV3.update` and + `create_default`. The `init=False` fields `zarr_format` and `node_type` + are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] + consolidated_metadata: ConsolidatedMetadataModelV3 | None + extra_fields: dict[str, ExtensionFieldV3] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class GroupMetadataModelV3: + """In-memory model of a v3 group metadata document. + + A canonical, lossless representation of the `zarr.json` content for a + group. The `consolidated_metadata` reference-implementation convention is + modeled as a typed field holding thin child models; every other unknown + top-level key lands in `extra_fields` verbatim. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["group"] = field(default="group", init=False) + attributes: dict[str, JSONValue] + consolidated_metadata: ConsolidatedMetadataModelV3 | None + extra_fields: dict[str, ExtensionFieldV3] + + def __post_init__(self) -> None: + reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} + if set(self.extra_fields.keys()).intersection(reserved): + raise ValueError("Extra fields cannot overlap with standard GroupMetadataV3 fields") + + @classmethod + def create_default( + cls, **overrides: Unpack[GroupMetadataModelV3Partial] + ) -> GroupMetadataModelV3: + """ + Create a default (empty) v3 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes={}, consolidated_metadata=None, extra_fields={}) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[GroupMetadataModelV3Partial]) -> GroupMetadataModelV3: + """ + Return a new `GroupMetadataModelV3` with the given fields updated. + + Only the constructor-settable fields listed in + `GroupMetadataModelV3Partial` can be updated; the fixed `zarr_format` / + `node_type` are rejected at the type level. Each given field fully + replaces its previous value, including `extra_fields`. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> GroupMetadataV3: + out: GroupMetadataV3 = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + } + if len(self.attributes) > 0: + out["attributes"] = self.attributes + if self.consolidated_metadata is not None: + # The consolidated-metadata shape ({kind, must_understand, metadata}, + # no `name`) predates the strict v3.1 extension-field rules, so it is + # not assignable to `ExtensionFieldV3`; see the discussion on + # `zarr_metadata.v3.consolidated`. + out[CONSOLIDATED_METADATA_KEY_V3] = cast( + "ExtensionFieldV3", self.consolidated_metadata.to_json() + ) + for key, value in self.extra_fields.items(): + out[key] = value + return out + + @classmethod + def from_json(cls, data: object) -> GroupMetadataModelV3: + parsed = parse_group_metadata_v3(arrays_to_tuples(data)) + consolidated_raw = parsed.get(CONSOLIDATED_METADATA_KEY_V3) + consolidated = ( + None + if consolidated_raw is None + else ConsolidatedMetadataModelV3.from_json(consolidated_raw) + ) + extra_fields: dict[str, ExtensionFieldV3] = { + k: v # type: ignore[misc] + for k, v in parsed.items() + if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 + } + return cls( + attributes=dict(parsed.get("attributes", {})), + consolidated_metadata=consolidated, + extra_fields=extra_fields, + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: + return cls.from_json(json.loads(mapping[GROUP_METADATA_STORE_KEY_V3])) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + GROUP_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ConsolidatedMetadataModelV3: + """In-memory model of v3 inline consolidated metadata. + + Models the reference-implementation convention where consolidated metadata + is embedded as an extension field on a group's `zarr.json`. Each entry in + `metadata` is a complete child document, held as a thin array or group + model. `must_understand` is typed permissively as `bool` to mirror the + document shape, but only `False` is valid; this is enforced at runtime. + """ + + kind: Literal["inline"] = field(default="inline", init=False) + must_understand: bool = False + metadata: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] + + def __post_init__(self) -> None: + if self.must_understand is not False: + raise ValueError( + f"Invalid value for 'must_understand'. Expected False. " + f"Got {self.must_understand!r}." + ) + + def to_json(self) -> ConsolidatedMetadataV3: + # `must_understand` is emitted as the literal False: the field is typed + # permissively as `bool`, but `__post_init__` guarantees the value. + return { + "kind": self.kind, + "must_understand": False, + "metadata": {key: node.to_json() for key, node in self.metadata.items()}, + } + + @classmethod + def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: + if not isinstance(data, Mapping): + raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + doc = cast("Mapping[str, object]", data) + entries_raw = doc.get("metadata") + if not isinstance(entries_raw, Mapping): + raise MetadataValidationError([ValidationProblem(("metadata",), "expected a mapping")]) + entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} + problems: list[ValidationProblem] = [] + for key, entry in cast("Mapping[object, object]", entries_raw).items(): + if not isinstance(key, str): + problems.append(ValidationProblem(("metadata",), f"non-string key {key!r}")) + continue + entry_obj: object = entry + node_type: object = None + if isinstance(entry, Mapping): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + entries[key] = ArrayMetadataModelV3.from_json(entry_obj) + elif node_type == "group": + entries[key] = GroupMetadataModelV3.from_json(entry_obj) + else: + problems.append( + ValidationProblem(("metadata", key, "node_type"), "expected 'array' or 'group'") + ) + if problems: + raise MetadataValidationError(problems) + must_understand = doc.get("must_understand", False) + if must_understand is not False: + raise MetadataValidationError( + [ValidationProblem(("must_understand",), "expected False")] + ) + return cls(must_understand=must_understand, metadata=entries) + + +class GroupMetadataModelV2Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `GroupMetadataModelV2`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `GroupMetadataModelV2.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class GroupMetadataModelV2: + """In-memory model of a v2 group metadata document. + + A canonical, lossless representation of the `.zgroup` content plus the + sibling `.zattrs` attributes, folded into a single in-memory value + (mirroring the merged `GroupMetadataV2` document form). + """ + + zarr_format: Literal[2] = field(default=2, init=False) + attributes: dict[str, JSONValue] + + @classmethod + def create_default( + cls, **overrides: Unpack[GroupMetadataModelV2Partial] + ) -> GroupMetadataModelV2: + """ + Create a default (empty) v2 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes={}) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadataModelV2: + """ + Return a new `GroupMetadataModelV2` with the given fields updated. + + Only the constructor-settable fields listed in + `GroupMetadataModelV2Partial` can be updated; the fixed `zarr_format` + is rejected at the type level. Each given field fully replaces its + previous value. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> GroupMetadataV2: + out: GroupMetadataV2 = {"zarr_format": self.zarr_format} + if len(self.attributes) > 0: + out["attributes"] = self.attributes + return out + + @classmethod + def from_json(cls, data: object) -> GroupMetadataModelV2: + parsed = parse_group_metadata_v2(arrays_to_tuples(data)) + return cls(attributes=dict(parsed.get("attributes", {}))) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: + zgroup = json.loads(mapping[GROUP_METADATA_STORE_KEY_V2]) + zattrs: dict[str, JSONValue] = ( + json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + if ATTRIBUTES_STORE_KEY_V2 in mapping + else {} + ) + return cls.from_json({**zgroup, "attributes": zattrs}) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zgroup` + # document must exclude them. + zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} + return { + GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8"), + ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ConsolidatedMetadataModelV2: + """In-memory model of a v2 `.zmetadata` document. + + The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, + `"path/.zattrs"`, ...) verbatim, preserving byte-faithful round-tripping. + Entries are deliberately NOT merged into per-node models: which nodes had + a `.zattrs` file at all is information the canonical representation must + keep. Interpreting entries into node models is consumer work. + """ + + zarr_consolidated_format: Literal[1] = field(default=1, init=False) + metadata: dict[str, JSONValue] + + def to_json(self) -> dict[str, JSONValue]: + return { + "zarr_consolidated_format": self.zarr_consolidated_format, + "metadata": self.metadata, + } + + @classmethod + def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: + if not isinstance(data, Mapping): + raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + doc = cast("Mapping[str, object]", data) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in ("zarr_consolidated_format", "metadata") + if key not in doc + ] + if "metadata" in doc: + entries = doc["metadata"] + if not isinstance(entries, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", entries) + ): + problems.append( + ValidationProblem(("metadata",), "expected a mapping with string keys") + ) + if problems: + raise MetadataValidationError(problems) + entries_tupled = cast( + "dict[str, JSONValue]", + arrays_to_tuples(dict(cast("Mapping[str, object]", doc["metadata"]))), + ) + return cls(metadata=entries_tupled) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ConsolidatedMetadataModelV2: + return cls.from_json(json.loads(mapping[CONSOLIDATED_METADATA_STORE_KEY_V2])) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + CONSOLIDATED_METADATA_STORE_KEY_V2: json.dumps(self.to_json(), indent=indent).encode( + "utf-8" + ) + } diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 25f3422c50..c69a80c144 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -14,7 +14,13 @@ from typing_extensions import TypeIs -from zarr_metadata import ArrayMetadataV2, ArrayMetadataV3, MetadataV3 +from zarr_metadata import ( + ArrayMetadataV2, + ArrayMetadataV3, + GroupMetadataV2, + GroupMetadataV3, + MetadataV3, +) from zarr_metadata._common import JSONValue @@ -99,6 +105,22 @@ def parse_json(value: object) -> JSONValue: ArrayMetadataV2.__required_keys__ ) +# The standard top-level keys of a v3 group metadata document. Anything outside +# this set is an extension field. +GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + GroupMetadataV3.__required_keys__ +) +GROUP_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + GroupMetadataV3.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V3 | GROUP_METADATA_OPTIONAL_KEYS_V3 +) + +GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + GroupMetadataV2.__required_keys__ +) + def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. @@ -259,6 +281,71 @@ def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: return cast(ArrayMetadataV2, value) +def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 group doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`); a `consolidated_metadata` key, if present, + must be a mapping (its entries are validated by the consolidated model). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V3 - doc.keys()) + ] + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "consolidated_metadata" in doc and not isinstance(doc["consolidated_metadata"], Mapping): + problems.append(ValidationProblem(("consolidated_metadata",), "expected a mapping")) + return problems + + +def is_group_metadata_v3(value: object) -> TypeIs[GroupMetadataV3]: + """Whether `value` is a structurally-valid v3 group metadata document.""" + return not validate_group_metadata_v3(value) + + +def parse_group_metadata_v3(value: object) -> GroupMetadataV3: + """Return `value` narrowed to `GroupMetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_group_metadata_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(GroupMetadataV3, value) + + +def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 group doc. + + Validates the in-memory merged form: the `.zgroup` fields plus an + optional `attributes` mapping folded in from `.zattrs`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V2 - doc.keys()) + ] + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_group_metadata_v2(value: object) -> TypeIs[GroupMetadataV2]: + """Whether `value` is a structurally-valid v2 group metadata document.""" + return not validate_group_metadata_v2(value) + + +def parse_group_metadata_v2(value: object) -> GroupMetadataV2: + """Return `value` narrowed to `GroupMetadataV2`, or raise `MetadataValidationError`.""" + problems = validate_group_metadata_v2(value) + if problems: + raise MetadataValidationError(problems) + return cast(GroupMetadataV2, value) + + def arrays_to_tuples(obj: object) -> object: """Recursively convert every list in a JSON-decoded structure to a tuple.""" if isinstance(obj, list): diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py new file mode 100644 index 0000000000..1e98cf3e47 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -0,0 +1,137 @@ +"""Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" + +import json + +import pytest + +from zarr_metadata.model._group import ( + GroupMetadataModelV2, + GroupMetadataModelV3, +) +from zarr_metadata.model._validation import ( + MetadataValidationError, + parse_group_metadata_v2, + parse_group_metadata_v3, +) + +# --- GroupMetadataModelV3 --------------------------------------------------- + + +def test_group_v3_roundtrip() -> None: + """A v3 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} + model = GroupMetadataModelV3.from_json(doc) + assert model.to_json() == doc + + +def test_group_v3_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + model = GroupMetadataModelV3.create_default() + assert "attributes" not in model.to_json() + + +def test_group_v3_lists_become_tuples() -> None: + """from_json converts JSON arrays in attributes to tuples.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2]}} + model = GroupMetadataModelV3.from_json(doc) + assert model.attributes == {"a": (1, 2)} + + +def test_group_v3_extra_fields_roundtrip() -> None: + """Unknown top-level keys land in extra_fields and reappear in to_json.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "my_extension": {"name": "thing", "must_understand": False}, + } + model = GroupMetadataModelV3.from_json(doc) + assert model.extra_fields == {"my_extension": {"name": "thing", "must_understand": False}} + assert model.to_json() == doc + + +def test_group_v3_extra_fields_overlap_rejected() -> None: + """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" + with pytest.raises(ValueError, match="Extra fields"): + GroupMetadataModelV3( + attributes={}, + consolidated_metadata=None, + extra_fields={"node_type": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_consolidated_extra_field_rejected() -> None: + """extra_fields may not shadow the consolidated_metadata convention key.""" + with pytest.raises(ValueError, match="Extra fields"): + GroupMetadataModelV3( + attributes={}, + consolidated_metadata=None, + extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_missing_required_key() -> None: + """parse_group_metadata_v3 reports each missing required key.""" + with pytest.raises(MetadataValidationError, match="node_type"): + parse_group_metadata_v3({"zarr_format": 3}) + + +def test_group_v3_bad_attributes() -> None: + """parse_group_metadata_v3 rejects a non-mapping attributes value.""" + with pytest.raises(MetadataValidationError, match="attributes"): + parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) + + +def test_group_v3_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for v3 groups.""" + model = GroupMetadataModelV3.create_default(attributes={"a": 1}) + assert GroupMetadataModelV3.from_key_value(model.to_key_value()) == model + + +def test_group_v3_update() -> None: + """update replaces the given fields and returns a new instance.""" + base = GroupMetadataModelV3.create_default() + updated = base.update(attributes={"a": 1}) + assert updated.attributes == {"a": 1} + assert base.attributes == {} + + +# --- GroupMetadataModelV2 --------------------------------------------------- + + +def test_group_v2_key_value_split() -> None: + """v2 to_key_value writes .zgroup and .zattrs; from_key_value merges them.""" + model = GroupMetadataModelV2.create_default(attributes={"a": 1}) + kv = model.to_key_value() + assert set(kv) == {".zgroup", ".zattrs"} + assert json.loads(kv[".zgroup"]) == {"zarr_format": 2} + assert GroupMetadataModelV2.from_key_value(kv) == model + + +def test_group_v2_from_key_value_without_zattrs() -> None: + """A v2 group with no .zattrs file parses with empty attributes.""" + model = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + assert model.attributes == {} + + +def test_group_v2_json_roundtrip() -> None: + """A merged-form v2 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 2, "attributes": {"a": 1}} + model = GroupMetadataModelV2.from_json(doc) + assert model.to_json() == doc + + +def test_group_v2_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + assert "attributes" not in GroupMetadataModelV2.create_default().to_json() + + +def test_group_v2_not_a_mapping() -> None: + """parse_group_metadata_v2 rejects a non-mapping document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + parse_group_metadata_v2([1, 2, 3]) + + +def test_group_v2_missing_required_key() -> None: + """parse_group_metadata_v2 reports a missing zarr_format key.""" + with pytest.raises(MetadataValidationError, match="zarr_format"): + parse_group_metadata_v2({}) From a85319f69d381d648fe28e9411e093228789a022 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:27:15 +0200 Subject: [PATCH 06/48] test(zarr-metadata): add consolidated metadata model tests Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/tests/model/test_group.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 1e98cf3e47..909346586b 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -1,12 +1,18 @@ """Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" +import dataclasses import json import pytest +from zarr_metadata.model._array import ArrayMetadataModelV3 from zarr_metadata.model._group import ( + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, GroupMetadataModelV2, + GroupMetadataModelV2Partial, GroupMetadataModelV3, + GroupMetadataModelV3Partial, ) from zarr_metadata.model._validation import ( MetadataValidationError, @@ -135,3 +141,127 @@ def test_group_v2_missing_required_key() -> None: """parse_group_metadata_v2 reports a missing zarr_format key.""" with pytest.raises(MetadataValidationError, match="zarr_format"): parse_group_metadata_v2({}) + + +# --- Partial TypedDict drift guards ----------------------------------------- + + +def test_group_partial_keys_match_settable_model_fields() -> None: + """Each group partial TypedDict must list exactly the settable model fields. + + Guards against drift: adding/removing a settable field on a group model + without updating its `*Partial` TypedDict fails here. + """ + for model_cls, partial_cls in ( + (GroupMetadataModelV3, GroupMetadataModelV3Partial), + (GroupMetadataModelV2, GroupMetadataModelV2Partial), + ): + settable = {f.name for f in dataclasses.fields(model_cls) if f.init} + assert set(partial_cls.__annotations__) == settable + + +# --- ConsolidatedMetadataModelV3 -------------------------------------------- + + +def test_consolidated_v3_roundtrip() -> None: + """A v3 group with inline consolidated metadata round-trips, with child + entries parsed into array/group models.""" + child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + model = GroupMetadataModelV3.from_json(doc) + assert isinstance(model.consolidated_metadata, ConsolidatedMetadataModelV3) + assert isinstance(model.consolidated_metadata.metadata["a"], ArrayMetadataModelV3) + assert isinstance(model.consolidated_metadata.metadata["g"], GroupMetadataModelV3) + assert model.to_json() == doc + + +def test_consolidated_v3_must_understand_true_rejected() -> None: + """ConsolidatedMetadataModelV3 enforces must_understand=False at runtime.""" + with pytest.raises(ValueError, match="must_understand"): + ConsolidatedMetadataModelV3(must_understand=True, metadata={}) + + +def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: + """from_json rejects a consolidated document carrying must_understand=true.""" + with pytest.raises(MetadataValidationError, match="must_understand"): + ConsolidatedMetadataModelV3.from_json( + {"kind": "inline", "must_understand": True, "metadata": {}} + ) + + +def test_consolidated_v3_entry_without_node_type_rejected() -> None: + """from_json rejects a consolidated entry lacking a recognizable node_type.""" + with pytest.raises(MetadataValidationError, match="node_type"): + ConsolidatedMetadataModelV3.from_json( + {"kind": "inline", "must_understand": False, "metadata": {"a": {"zarr_format": 3}}} + ) + + +def test_consolidated_v3_not_a_mapping() -> None: + """from_json rejects a non-mapping consolidated document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ConsolidatedMetadataModelV3.from_json(5) + + +# --- ConsolidatedMetadataModelV2 -------------------------------------------- + + +def test_consolidated_v2_verbatim_roundtrip() -> None: + """The v2 .zmetadata model holds the flat file-keyed map verbatim, + including nodes that have no .zattrs entry.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": {"zarr_format": 2}, + "a/.zarray": { + "zarr_format": 2, + "shape": (2,), + "chunks": (2,), + "dtype": "|u1", + "fill_value": 0, + "order": "C", + "compressor": None, + "filters": None, + }, + }, + } + model = ConsolidatedMetadataModelV2.from_json(doc) + assert model.to_json() == doc + + +def test_consolidated_v2_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for .zmetadata documents.""" + model = ConsolidatedMetadataModelV2.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}} + ) + assert ConsolidatedMetadataModelV2.from_key_value(model.to_key_value()) == model + + +def test_consolidated_v2_lists_become_tuples() -> None: + """from_json converts JSON arrays inside entries to tuples.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": {"a/.zarray": {"shape": [2, 3]}}, + } + model = ConsolidatedMetadataModelV2.from_json(doc) + assert model.metadata == {"a/.zarray": {"shape": (2, 3)}} + + +def test_consolidated_v2_envelope_validation() -> None: + """from_json rejects a .zmetadata document missing the metadata key.""" + with pytest.raises(MetadataValidationError, match="metadata"): + ConsolidatedMetadataModelV2.from_json({"zarr_consolidated_format": 1}) + + +def test_consolidated_v2_not_a_mapping() -> None: + """from_json rejects a non-mapping .zmetadata document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ConsolidatedMetadataModelV2.from_json([1]) From fbfb425ad913cdc2424f51db029c57ec7377207d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:30:18 +0200 Subject: [PATCH 07/48] feat(zarr-metadata): export model layer from package front door Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/__init__.py | 28 +++++++++++++ .../src/zarr_metadata/model/__init__.py | 42 +++++++++++++++++++ .../src/zarr_metadata/model/_array.py | 12 +++--- .../src/zarr_metadata/model/_group.py | 10 ++--- .../src/zarr_metadata/model/_validation.py | 12 +++--- .../zarr-metadata/tests/test_public_api.py | 14 +++++++ 6 files changed, 99 insertions(+), 19 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 46949570a2..d522c89cd2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,6 +1,21 @@ from importlib.metadata import version from zarr_metadata._common import JSONValue, NamedConfigV3 +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV2Partial, + GroupMetadataModelV3, + GroupMetadataModelV3Partial, + MetadataValidationError, + ValidationProblem, + ZarrMetadataV3, +) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, ARRAY_ORDER_V2, @@ -235,6 +250,10 @@ "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", "ArrayDimensionSeparatorV2", + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", "ArrayMetadataV2", "ArrayMetadataV2Partial", "ArrayMetadataV3", @@ -259,6 +278,8 @@ "Complex64FillValue", "Complex128DataTypeName", "Complex128FillValue", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", "ConsolidatedMetadataV2", "ConsolidatedMetadataV3", "Crc32cCodecMetadata", @@ -275,6 +296,10 @@ "Float32FillValue", "Float64DataTypeName", "Float64FillValue", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", "GroupMetadataV2", "GroupMetadataV2Partial", "GroupMetadataV3", @@ -291,6 +316,7 @@ "Int64FillValue", "JSONValue", "MetadataV3", + "MetadataValidationError", "NamedConfigV3", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", @@ -325,9 +351,11 @@ "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", + "ValidationProblem", "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", + "ZarrMetadataV3", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 8776b4fab3..68e36be0a7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -10,23 +10,45 @@ ArrayMetadataModelV3Partial, ZarrMetadataV3, ) +from zarr_metadata.model._group import ( + CONSOLIDATED_METADATA_KEY_V3, + CONSOLIDATED_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV2Partial, + GroupMetadataModelV3, + GroupMetadataModelV3Partial, +) from zarr_metadata.model._validation import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V2, ARRAY_METADATA_REQUIRED_KEYS_V3, ARRAY_METADATA_STANDARD_KEYS_V3, + GROUP_METADATA_OPTIONAL_KEYS_V3, + GROUP_METADATA_REQUIRED_KEYS_V2, + GROUP_METADATA_REQUIRED_KEYS_V3, + GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, ValidationProblem, is_array_metadata_v2, is_array_metadata_v3, + is_group_metadata_v2, + is_group_metadata_v3, is_json, is_metadata_field_v3, parse_array_metadata_v2, parse_array_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, parse_json, parse_metadata_field_v3, validate_array_metadata_v2, validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, validate_json, validate_metadata_field_v3, ) @@ -39,23 +61,43 @@ "ARRAY_METADATA_STORE_KEY_V2", "ARRAY_METADATA_STORE_KEY_V3", "ATTRIBUTES_STORE_KEY_V2", + "CONSOLIDATED_METADATA_KEY_V3", + "CONSOLIDATED_METADATA_STORE_KEY_V2", + "GROUP_METADATA_OPTIONAL_KEYS_V3", + "GROUP_METADATA_REQUIRED_KEYS_V2", + "GROUP_METADATA_REQUIRED_KEYS_V3", + "GROUP_METADATA_STANDARD_KEYS_V3", + "GROUP_METADATA_STORE_KEY_V2", + "GROUP_METADATA_STORE_KEY_V3", "ArrayMetadataModelV2", "ArrayMetadataModelV2Partial", "ArrayMetadataModelV3", "ArrayMetadataModelV3Partial", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", "MetadataValidationError", "ValidationProblem", "ZarrMetadataV3", "is_array_metadata_v2", "is_array_metadata_v3", + "is_group_metadata_v2", + "is_group_metadata_v3", "is_json", "is_metadata_field_v3", "parse_array_metadata_v2", "parse_array_metadata_v3", + "parse_group_metadata_v2", + "parse_group_metadata_v3", "parse_json", "parse_metadata_field_v3", "validate_array_metadata_v2", "validate_array_metadata_v3", + "validate_group_metadata_v2", + "validate_group_metadata_v3", "validate_json", "validate_metadata_field_v3", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index ea2745284d..e5b23918d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -20,16 +20,16 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata import ( + from zarr_metadata._common import JSONValue + from zarr_metadata.v2.array import ( ArrayDimensionSeparatorV2, ArrayMetadataV2, - ArrayMetadataV3, ArrayOrderV2, - ExtensionFieldV3, - MetadataV3, + DataTypeMetadataV2, ) - from zarr_metadata._common import JSONValue - from zarr_metadata.v2 import CodecMetadataV2, DataTypeMetadataV2 + from zarr_metadata.v2.codec import CodecMetadataV2 + from zarr_metadata.v3._common import MetadataV3 + from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 ArrayMetadataStoreKeyV3 = Literal["zarr.json"] ARRAY_METADATA_STORE_KEY_V3: Final[ArrayMetadataStoreKeyV3] = "zarr.json" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 8eeaed1a89..a061ed5d67 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -24,13 +24,11 @@ ) if TYPE_CHECKING: - from zarr_metadata import ( - ConsolidatedMetadataV3, - ExtensionFieldV3, - GroupMetadataV2, - GroupMetadataV3, - ) from zarr_metadata._common import JSONValue + from zarr_metadata.v2.group import GroupMetadataV2 + from zarr_metadata.v3.array import ExtensionFieldV3 + from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 + from zarr_metadata.v3.group import GroupMetadataV3 GroupMetadataStoreKeyV3 = Literal["zarr.json"] GROUP_METADATA_STORE_KEY_V3: Final[GroupMetadataStoreKeyV3] = "zarr.json" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index c69a80c144..a38258316b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -14,14 +14,12 @@ from typing_extensions import TypeIs -from zarr_metadata import ( - ArrayMetadataV2, - ArrayMetadataV3, - GroupMetadataV2, - GroupMetadataV3, - MetadataV3, -) from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ArrayMetadataV2 +from zarr_metadata.v2.group import GroupMetadataV2 +from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.group import GroupMetadataV3 @dataclass(frozen=True, slots=True) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index d3270579c3..7a6db7ab30 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -39,6 +39,20 @@ def _group_rank(s: str) -> int: "NamedConfigV3", "MetadataV3", "JSONValue", + # Category A' — metadata models (in-memory dataclasses over the documents) + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", + "ZarrMetadataV3", + "ValidationProblem", + "MetadataValidationError", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From 99608ce38789fcd9c677d34cdcfd9fa6cce17fb0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:31:40 +0200 Subject: [PATCH 08/48] docs(zarr-metadata): changelog entry for the model layer Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/zarr-metadata/changes/210.feature.md diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md new file mode 100644 index 0000000000..35835c9ebe --- /dev/null +++ b/packages/zarr-metadata/changes/210.feature.md @@ -0,0 +1,7 @@ +Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, +`ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`, +`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `ZarrMetadataV3`) +that are canonical, lossless representations of Zarr metadata documents, plus +structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension +point (data type, chunk grid, chunk key encoding, codecs, storage transformers) +is held as a name + configuration pair; nothing is interpreted. From c98a1220493813935a2e451725117952dbe01eed Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 14:58:41 +0200 Subject: [PATCH 09/48] feat(zarr-metadata): harden model validation and error reporting Findings from an API-ergonomics exercise (a fresh agent consuming defective metadata documents): - ValidationProblem gains a machine-readable kind (missing_key / invalid_type / invalid_value / invalid_json), ending message string-matching in consumers. - The v2 array validator now enforces what its types declare (dtype, order, compressor, filters, dimension_separator), and all four document validators check the fixed zarr_format / node_type literals. - All ingestion failures surface as MetadataValidationError: missing store keys and undecodable bytes in from_key_value (previously KeyError / JSONDecodeError) and constructor invariants (previously bare ValueError). - ZarrMetadataV3 is renamed NamedConfigModelV3: it models a name + configuration pair, and the old name read as a whole-document type. - Discoverability: the validate_*/is_*/parse_* contract is documented on zarr_metadata.model itself; update() documents that it does not re-validate; the v2 to_json/to_key_value attributes split is documented on both. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 11 +- .../src/zarr_metadata/__init__.py | 6 +- .../src/zarr_metadata/model/__init__.py | 19 +- .../src/zarr_metadata/model/_array.py | 76 ++++-- .../src/zarr_metadata/model/_group.py | 67 +++-- .../src/zarr_metadata/model/_validation.py | 237 +++++++++++++++--- .../zarr-metadata/tests/model/test_array.py | 187 +++++++++++--- .../zarr-metadata/tests/model/test_group.py | 19 ++ .../zarr-metadata/tests/test_public_api.py | 3 +- 9 files changed, 501 insertions(+), 124 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 35835c9ebe..13517ce38a 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -1,7 +1,16 @@ Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, `ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`, -`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `ZarrMetadataV3`) +`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `NamedConfigModelV3`) that are canonical, lossless representations of Zarr metadata documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, storage transformers) is held as a name + configuration pair; nothing is interpreted. + +Validation is strict about what the types declare: v2 `dtype` / `order` / +`compressor` / `filters` / `dimension_separator` shapes and the fixed +`zarr_format` / `node_type` literals are all enforced. Every +`ValidationProblem` carries a machine-readable `kind` +(`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so +consumers can dispatch on the failure mode without matching message strings, +and every ingestion failure — including missing store keys and undecodable +bytes in `from_key_value` — surfaces as `MetadataValidationError`. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index d522c89cd2..ecd7e342cf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -13,8 +13,9 @@ GroupMetadataModelV3, GroupMetadataModelV3Partial, MetadataValidationError, + NamedConfigModelV3, + ProblemKind, ValidationProblem, - ZarrMetadataV3, ) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, @@ -317,12 +318,14 @@ "JSONValue", "MetadataV3", "MetadataValidationError", + "NamedConfigModelV3", "NamedConfigV3", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", + "ProblemKind", "RawBytesDataTypeName", "RawBytesFillValue", "RectilinearChunkGridMetadata", @@ -355,7 +358,6 @@ "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", - "ZarrMetadataV3", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 68e36be0a7..aa8555d623 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -1,4 +1,15 @@ -"""In-memory models for Zarr metadata documents.""" +"""In-memory models for Zarr metadata documents. + +Models are frozen dataclasses that hold a canonical, lossless representation +of the JSON documents; they never interpret extension points (codecs, chunk +grids, data types). Validators check JSON structure, not domain validity. +Each document concept gets a `validate_*` function returning every problem +found (a `list[ValidationProblem]`, each with a machine-readable `kind`), an +`is_*` type guard, and a `parse_*` function that narrows or raises +`MetadataValidationError`. Model `from_json` / `from_key_value` constructors +raise `MetadataValidationError` for every ingestion failure, including +missing store keys and undecodable bytes. +""" from zarr_metadata.model._array import ( ARRAY_METADATA_STORE_KEY_V2, @@ -8,7 +19,7 @@ ArrayMetadataModelV2Partial, ArrayMetadataModelV3, ArrayMetadataModelV3Partial, - ZarrMetadataV3, + NamedConfigModelV3, ) from zarr_metadata.model._group import ( CONSOLIDATED_METADATA_KEY_V3, @@ -32,6 +43,7 @@ GROUP_METADATA_REQUIRED_KEYS_V3, GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, + ProblemKind, ValidationProblem, is_array_metadata_v2, is_array_metadata_v3, @@ -80,8 +92,9 @@ "GroupMetadataModelV3", "GroupMetadataModelV3Partial", "MetadataValidationError", + "NamedConfigModelV3", + "ProblemKind", "ValidationProblem", - "ZarrMetadataV3", "is_array_metadata_v2", "is_array_metadata_v3", "is_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index e5b23918d4..4ec02137ec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -11,7 +11,10 @@ from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, arrays_to_tuples, + load_store_json, parse_array_metadata_v2, parse_array_metadata_v3, parse_metadata_field_v3, @@ -42,7 +45,7 @@ @dataclass(frozen=True, slots=True, kw_only=True) -class ZarrMetadataV3: +class NamedConfigModelV3: """A v3 metadata field in normalized form: a name plus a configuration. This is the in-memory model of `MetadataV3` (a bare name string or a @@ -57,7 +60,7 @@ def to_json(self) -> MetadataV3: return {"name": self.name, "configuration": self.configuration} @classmethod - def from_json(cls, data: object) -> ZarrMetadataV3: + def from_json(cls, data: object) -> NamedConfigModelV3: field = parse_metadata_field_v3(data) if isinstance(field, str): return cls(name=field, configuration={}) @@ -81,13 +84,13 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): shape: tuple[int, ...] fill_value: JSONValue - data_type: ZarrMetadataV3 - chunk_grid: ZarrMetadataV3 - codecs: tuple[ZarrMetadataV3, ...] - chunk_key_encoding: ZarrMetadataV3 + data_type: NamedConfigModelV3 + chunk_grid: NamedConfigModelV3 + codecs: tuple[NamedConfigModelV3, ...] + chunk_key_encoding: NamedConfigModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[ZarrMetadataV3, ...] + storage_transformers: tuple[NamedConfigModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -97,7 +100,7 @@ class ArrayMetadataModelV3: A canonical, lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, - `codecs`, `storage_transformers`) are held as `ZarrMetadataV3` name + + `codecs`, `storage_transformers`) are held as `NamedConfigModelV3` name + configuration pairs and are never interpreted; `fill_value` is held verbatim in its JSON form. """ @@ -106,13 +109,13 @@ class ArrayMetadataModelV3: node_type: Literal["array"] = field(default="array", init=False) shape: tuple[int, ...] fill_value: JSONValue - data_type: ZarrMetadataV3 - chunk_grid: ZarrMetadataV3 - codecs: tuple[ZarrMetadataV3, ...] - chunk_key_encoding: ZarrMetadataV3 + data_type: NamedConfigModelV3 + chunk_grid: NamedConfigModelV3 + codecs: tuple[NamedConfigModelV3, ...] + chunk_key_encoding: NamedConfigModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[ZarrMetadataV3, ...] + storage_transformers: tuple[NamedConfigModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @classmethod @@ -129,10 +132,10 @@ def create_default( default = cls( shape=(), fill_value=0, - data_type=ZarrMetadataV3(name="uint8", configuration={}), - chunk_grid=ZarrMetadataV3(name="regular", configuration={"chunk_shape": ()}), - codecs=(ZarrMetadataV3(name="bytes", configuration={}),), - chunk_key_encoding=ZarrMetadataV3(name="default", configuration={}), + data_type=NamedConfigModelV3(name="uint8", configuration={}), + chunk_grid=NamedConfigModelV3(name="regular", configuration={"chunk_shape": ()}), + codecs=(NamedConfigModelV3(name="bytes", configuration={}),), + chunk_key_encoding=NamedConfigModelV3(name="default", configuration={}), dimension_names=None, attributes={}, storage_transformers=(), @@ -152,12 +155,25 @@ def update(self, **kwargs: Unpack[ArrayMetadataModelV3Partial]) -> ArrayMetadata This is useful for test fixtures that want to override a few fields of a base template without having to re-specify the entire document. + + No re-validation is performed (`update` is `dataclasses.replace`), so + a repair or edit can produce an invalid document; validity is checked + on `from_json`, not on field replacement. """ return dataclasses.replace(self, **kwargs) def __post_init__(self) -> None: - if set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3): - raise ValueError("Extra fields cannot overlap with standard ArrayMetadataV3 fields") + overlap = set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3) + if overlap: + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard ArrayMetadataV3 fields", + "invalid_value", + ) + ] + ) def to_json(self) -> ArrayMetadataV3: out: ArrayMetadataV3 = { @@ -197,21 +213,21 @@ def from_json(cls, data: object) -> ArrayMetadataModelV3: return cls( shape=parsed["shape"], fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict - data_type=ZarrMetadataV3.from_json(parsed["data_type"]), - chunk_grid=ZarrMetadataV3.from_json(parsed["chunk_grid"]), - codecs=tuple(ZarrMetadataV3.from_json(c) for c in parsed["codecs"]), - chunk_key_encoding=ZarrMetadataV3.from_json(parsed["chunk_key_encoding"]), + data_type=NamedConfigModelV3.from_json(parsed["data_type"]), + chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), + codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=NamedConfigModelV3.from_json(parsed["chunk_key_encoding"]), dimension_names=parsed.get("dimension_names"), attributes=dict(parsed.get("attributes", {})), storage_transformers=tuple( - ZarrMetadataV3.from_json(t) for t in parsed.get("storage_transformers", ()) + NamedConfigModelV3.from_json(t) for t in parsed.get("storage_transformers", ()) ), extra_fields=extra_fields, ) @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: - return cls.from_json(json.loads(mapping[ARRAY_METADATA_STORE_KEY_V3])) + return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { @@ -299,6 +315,12 @@ def create_default( return default.update(**overrides) def to_json(self) -> ArrayMetadataV2: + """Return the merged in-memory document form, INCLUDING `attributes`. + + This is not the on-disk `.zarray` content: a conforming `.zarray` must + exclude `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ out: ArrayMetadataV2 = { "zarr_format": self.zarr_format, "shape": self.shape, @@ -330,9 +352,9 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: - zarray = json.loads(mapping[ARRAY_METADATA_STORE_KEY_V2]) + zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) zattrs: dict[str, JSONValue] = ( - json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping else {} ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index a061ed5d67..6128b0306b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -19,6 +19,7 @@ MetadataValidationError, ValidationProblem, arrays_to_tuples, + load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, ) @@ -83,7 +84,15 @@ class GroupMetadataModelV3: def __post_init__(self) -> None: reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} if set(self.extra_fields.keys()).intersection(reserved): - raise ValueError("Extra fields cannot overlap with standard GroupMetadataV3 fields") + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard GroupMetadataV3 fields", + "invalid_value", + ) + ] + ) @classmethod def create_default( @@ -151,7 +160,7 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: - return cls.from_json(json.loads(mapping[GROUP_METADATA_STORE_KEY_V3])) + return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { @@ -176,9 +185,15 @@ class ConsolidatedMetadataModelV3: def __post_init__(self) -> None: if self.must_understand is not False: - raise ValueError( - f"Invalid value for 'must_understand'. Expected False. " - f"Got {self.must_understand!r}." + raise MetadataValidationError( + [ + ValidationProblem( + ("must_understand",), + f"Invalid value for 'must_understand'. Expected False. " + f"Got {self.must_understand!r}.", + "invalid_value", + ) + ] ) def to_json(self) -> ConsolidatedMetadataV3: @@ -193,16 +208,22 @@ def to_json(self) -> ConsolidatedMetadataV3: @classmethod def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: if not isinstance(data, Mapping): - raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + raise MetadataValidationError( + [ValidationProblem((), "expected a mapping", "invalid_type")] + ) doc = cast("Mapping[str, object]", data) entries_raw = doc.get("metadata") if not isinstance(entries_raw, Mapping): - raise MetadataValidationError([ValidationProblem(("metadata",), "expected a mapping")]) + raise MetadataValidationError( + [ValidationProblem(("metadata",), "expected a mapping", "invalid_type")] + ) entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} problems: list[ValidationProblem] = [] for key, entry in cast("Mapping[object, object]", entries_raw).items(): if not isinstance(key, str): - problems.append(ValidationProblem(("metadata",), f"non-string key {key!r}")) + problems.append( + ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") + ) continue entry_obj: object = entry node_type: object = None @@ -214,14 +235,18 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: entries[key] = GroupMetadataModelV3.from_json(entry_obj) else: problems.append( - ValidationProblem(("metadata", key, "node_type"), "expected 'array' or 'group'") + ValidationProblem( + ("metadata", key, "node_type"), + "expected 'array' or 'group'", + "invalid_value", + ) ) if problems: raise MetadataValidationError(problems) must_understand = doc.get("must_understand", False) if must_understand is not False: raise MetadataValidationError( - [ValidationProblem(("must_understand",), "expected False")] + [ValidationProblem(("must_understand",), "expected False", "invalid_value")] ) return cls(must_understand=must_understand, metadata=entries) @@ -280,6 +305,12 @@ def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadata return dataclasses.replace(self, **kwargs) def to_json(self) -> GroupMetadataV2: + """Return the merged in-memory document form, INCLUDING `attributes`. + + This is not the on-disk `.zgroup` content: a conforming `.zgroup` must + exclude `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ out: GroupMetadataV2 = {"zarr_format": self.zarr_format} if len(self.attributes) > 0: out["attributes"] = self.attributes @@ -292,9 +323,9 @@ def from_json(cls, data: object) -> GroupMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: - zgroup = json.loads(mapping[GROUP_METADATA_STORE_KEY_V2]) + zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) zattrs: dict[str, JSONValue] = ( - json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping else {} ) @@ -333,10 +364,12 @@ def to_json(self) -> dict[str, JSONValue]: @classmethod def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: if not isinstance(data, Mapping): - raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + raise MetadataValidationError( + [ValidationProblem((), "expected a mapping", "invalid_type")] + ) doc = cast("Mapping[str, object]", data) problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") + ValidationProblem((key,), "missing required key", "missing_key") for key in ("zarr_consolidated_format", "metadata") if key not in doc ] @@ -346,7 +379,9 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: isinstance(k, str) for k in cast("Mapping[object, object]", entries) ): problems.append( - ValidationProblem(("metadata",), "expected a mapping with string keys") + ValidationProblem( + ("metadata",), "expected a mapping with string keys", "invalid_type" + ) ) if problems: raise MetadataValidationError(problems) @@ -358,7 +393,7 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ConsolidatedMetadataModelV2: - return cls.from_json(json.loads(mapping[CONSOLIDATED_METADATA_STORE_KEY_V2])) + return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index a38258316b..304a80d4ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -1,16 +1,22 @@ """Structural validation for Zarr metadata documents. -Validators check JSON structure (shapes, key presence, primitive kinds), -not domain validity. Each concept gets a `validate_*` function returning -every problem found, an `is_*` type guard, and a `parse_*` function that -narrows or raises `MetadataValidationError`. +Validators check JSON structure (key presence, value shapes, and fixed +literals like `zarr_format`), not domain validity. Each concept gets a +`validate_*` function returning every problem found, an `is_*` type guard, +and a `parse_*` function that narrows or raises `MetadataValidationError`. + +Every `ValidationProblem` carries a machine-readable `kind` alongside its +human-readable `message`, so consumers can dispatch on the failure mode +(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`) without +string-matching messages. """ from __future__ import annotations +import json from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Final, cast +from typing import Any, Final, Literal, cast from typing_extensions import TypeIs @@ -21,6 +27,17 @@ from zarr_metadata.v3.array import ArrayMetadataV3 from zarr_metadata.v3.group import GroupMetadataV3 +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] +"""Machine-readable classification of a `ValidationProblem`. + +- `missing_key`: a required key (document key or store key) is absent. +- `invalid_type`: a value has the wrong structural type (e.g. a string where + a mapping is required, a non-JSON-serializable object). +- `invalid_value`: a value has an acceptable type but an invalid content + (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). +- `invalid_json`: bytes that do not decode as JSON. +""" + @dataclass(frozen=True, slots=True) class ValidationProblem: @@ -28,10 +45,13 @@ class ValidationProblem: `loc` is the path from the document root to the offending value, e.g. `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + `kind` classifies the failure mode for programmatic dispatch; `message` + is the human-readable description. """ loc: tuple[str | int, ...] message: str + kind: ProblemKind def __str__(self) -> str: location = ".".join(str(part) for part in self.loc) if self.loc else "" @@ -51,7 +71,7 @@ def __init__(self, problems: list[ValidationProblem]) -> None: def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" - return [ValidationProblem((loc_head, *p.loc), p.message) for p in problems] + return [ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems] def validate_json(value: object) -> list[ValidationProblem]: @@ -62,7 +82,9 @@ def validate_json(value: object) -> list[ValidationProblem]: if isinstance(value, Mapping): for key, item in cast("Mapping[object, object]", value).items(): if not isinstance(key, str): - problems.append(ValidationProblem((), f"non-string key {key!r} in JSON object")) + problems.append( + ValidationProblem((), f"non-string key {key!r} in JSON object", "invalid_type") + ) continue problems.extend(_prefix(key, validate_json(item))) return problems @@ -70,7 +92,7 @@ def validate_json(value: object) -> list[ValidationProblem]: for index, item in enumerate(cast("Sequence[object]", value)): problems.extend(_prefix(index, validate_json(item))) return problems - return [ValidationProblem((), f"not a JSON-serializable value: {value!r}")] + return [ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type")] def is_json(value: object) -> TypeIs[JSONValue]: @@ -120,6 +142,25 @@ def parse_json(value: object) -> JSONValue: ) +def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[ValidationProblem]: + """One `missing_key` problem per required key absent from `doc`.""" + return [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in sorted(required - doc.keys()) + ] + + +def _check_literal( + doc: Mapping[str, object], key: str, expected: object +) -> list[ValidationProblem]: + """One `invalid_value` problem if `doc[key]` is present but not `expected`.""" + if key in doc and doc[key] != expected: + return [ + ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value") + ] + return [] + + def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. @@ -129,18 +170,26 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: return [] if not isinstance(value, Mapping): return [ - ValidationProblem((), "expected a metadata field (string or {name, configuration})") + ValidationProblem( + (), + "expected a metadata field (string or {name, configuration})", + "invalid_type", + ) ] field = cast("Mapping[object, object]", value) problems: list[ValidationProblem] = [] if not isinstance(field.get("name"), str): - problems.append(ValidationProblem(("name",), "expected a string name")) + problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) if "configuration" in field: configuration = field["configuration"] if not isinstance(configuration, Mapping): - problems.append(ValidationProblem(("configuration",), "expected a mapping")) + problems.append( + ValidationProblem(("configuration",), "expected a mapping", "invalid_type") + ) elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): - problems.append(ValidationProblem(("configuration",), "expected string keys")) + problems.append( + ValidationProblem(("configuration",), "expected string keys", "invalid_type") + ) return problems @@ -166,6 +215,40 @@ def _is_int_sequence(value: object) -> bool: ) +def _is_dtype_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 dtype: a string or field records. + + A field record is a `(name, dtype)` or `(name, dtype, shape)` sequence, + where `dtype` is itself a string or nested field records and `shape` is a + sequence of int. The string content is NOT interpreted — whether the + string names a real dtype is domain validity, not structure. + """ + if isinstance(value, str): + return True + if not isinstance(value, Sequence): + return False + for record in cast("Sequence[object]", value): + if isinstance(record, str) or not isinstance(record, Sequence): + return False + fields = cast("Sequence[object]", record) + if len(fields) not in (2, 3): + return False + if not isinstance(fields[0], str): + return False + if not _is_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not _is_int_sequence(fields[2]): + return False + return True + + +def _is_codec_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 codec config: a mapping with a string `id`.""" + return isinstance(value, Mapping) and isinstance( + cast("Mapping[object, object]", value).get("id"), str + ) + + def _validate_attributes(value: object) -> list[ValidationProblem]: """Validate an `attributes` value: a mapping with string keys. @@ -178,7 +261,11 @@ def _validate_attributes(value: object) -> list[ValidationProblem]: if not isinstance(value, Mapping) or not all( isinstance(k, str) for k in cast("Mapping[object, object]", value) ): - return [ValidationProblem(("attributes",), "expected a mapping with string keys")] + return [ + ValidationProblem( + ("attributes",), "expected a mapping with string keys", "invalid_type" + ) + ] return [] @@ -189,14 +276,13 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: (they map to `extra_fields`). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V3 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "array")) if "shape" in doc and not _is_int_sequence(doc["shape"]): - problems.append(ValidationProblem(("shape",), "expected a sequence of int")) + problems.append(ValidationProblem(("shape",), "expected a sequence of int", "invalid_type")) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): @@ -206,7 +292,7 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: if key in doc: entries = doc[key] if isinstance(entries, str) or not isinstance(entries, Sequence): - problems.append(ValidationProblem((key,), "expected a sequence")) + problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) else: for index, entry in enumerate(cast("Sequence[object]", entries)): problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) @@ -218,12 +304,16 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: # the metadata-field lists (codecs, storage_transformers). names = doc["dimension_names"] if isinstance(names, str) or not isinstance(names, Sequence): - problems.append(ValidationProblem(("dimension_names",), "expected a sequence")) + problems.append( + ValidationProblem(("dimension_names",), "expected a sequence", "invalid_type") + ) elif not all( item is None or isinstance(item, str) for item in cast("Sequence[object]", names) ): problems.append( - ValidationProblem(("dimension_names",), "expected items of str or None") + ValidationProblem( + ("dimension_names",), "expected items of str or None", "invalid_type" + ) ) return problems @@ -244,21 +334,67 @@ def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a structurally-valid v2 array doc. - Checks structure, not domain validity. `compressor`/`filters` are required - keys but may be `None`. + Checks structure, not domain validity: `dtype` must be a string or field + records, but the string content is not interpreted; `compressor` and + `filters` are required keys that may be `None`, and otherwise must be + codec configurations (mappings with a string `id`). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V2 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend(_check_literal(doc, "zarr_format", 2)) problems.extend( - ValidationProblem((key,), "expected a sequence of int") + ValidationProblem((key,), "expected a sequence of int", "invalid_type") for key in ("shape", "chunks") if key in doc and not _is_int_sequence(doc[key]) ) + if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): + problems.append( + ValidationProblem( + ("dtype",), + "expected a v2 dtype string or a sequence of field records", + "invalid_type", + ) + ) + if "order" in doc and doc["order"] not in ("C", "F"): + problems.append( + ValidationProblem( + ("order",), f"expected 'C' or 'F', got {doc['order']!r}", "invalid_value" + ) + ) + if "compressor" in doc: + compressor = doc["compressor"] + if compressor is not None and not _is_codec_v2(compressor): + problems.append( + ValidationProblem( + ("compressor",), + "expected null or a codec configuration with a string 'id'", + "invalid_type", + ) + ) + if "filters" in doc: + filters = doc["filters"] + if filters is not None and ( + isinstance(filters, str) + or not isinstance(filters, Sequence) + or not all(_is_codec_v2(item) for item in cast("Sequence[object]", filters)) + ): + problems.append( + ValidationProblem( + ("filters",), + "expected null or a sequence of codec configurations with string 'id's", + "invalid_type", + ) + ) + if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): + problems.append( + ValidationProblem( + ("dimension_separator",), + f"expected '.' or '/', got {doc['dimension_separator']!r}", + "invalid_value", + ) + ) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) if "attributes" in doc: @@ -287,16 +423,17 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: must be a mapping (its entries are validated by the consolidated model). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V3 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) if "consolidated_metadata" in doc and not isinstance(doc["consolidated_metadata"], Mapping): - problems.append(ValidationProblem(("consolidated_metadata",), "expected a mapping")) + problems.append( + ValidationProblem(("consolidated_metadata",), "expected a mapping", "invalid_type") + ) return problems @@ -320,12 +457,10 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: optional `attributes` mapping folded in from `.zattrs`. """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V2 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend(_check_literal(doc, "zarr_format", 2)) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) return problems @@ -344,6 +479,26 @@ def parse_group_metadata_v2(value: object) -> GroupMetadataV2: return cast(GroupMetadataV2, value) +def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: + """Decode the JSON document stored at `key` in `mapping`. + + Every ingestion failure surfaces as `MetadataValidationError`: a missing + store key is a `missing_key` problem and undecodable bytes are an + `invalid_json` problem, rather than leaking `KeyError` / + `json.JSONDecodeError` to callers. + """ + if key not in mapping: + raise MetadataValidationError( + [ValidationProblem((key,), "missing store key", "missing_key")] + ) + try: + return json.loads(mapping[key]) + except json.JSONDecodeError as exc: + raise MetadataValidationError( + [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] + ) from exc + + def arrays_to_tuples(obj: object) -> object: """Recursively convert every list in a JSON-decoded structure to a tuple.""" if isinstance(obj, list): diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 80dac709de..904e0b12fa 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -17,8 +17,8 @@ ArrayMetadataModelV3, ArrayMetadataModelV3Partial, MetadataValidationError, + NamedConfigModelV3, ValidationProblem, - ZarrMetadataV3, is_array_metadata_v2, is_array_metadata_v3, is_json, @@ -119,16 +119,16 @@ def test_string_nan_fill_value_roundtrips() -> None: assert ArrayMetadataModelV3.from_json(m.to_json()).fill_value == "NaN" -# --- ZarrMetadataV3.to_json ------------------------------------------------ +# --- NamedConfigModelV3.to_json ------------------------------------------------ ZARR_TO_JSON_CASES = [ Expect( - ZarrMetadataV3(name="regular", configuration={"chunk_shape": [1]}), + NamedConfigModelV3(name="regular", configuration={"chunk_shape": [1]}), {"name": "regular", "configuration": {"chunk_shape": [1]}}, id="with-configuration", ), Expect( - ZarrMetadataV3(name="bytes", configuration={}), + NamedConfigModelV3(name="bytes", configuration={}), {"name": "bytes", "configuration": {}}, id="without-configuration", ), @@ -136,32 +136,32 @@ def test_string_nan_fill_value_roundtrips() -> None: @pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_to_json(case: Expect[ZarrMetadataV3, dict[str, object]]) -> None: - """ZarrMetadataV3.to_json emits the canonical object form.""" +def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, dict[str, object]]) -> None: + """NamedConfigModelV3.to_json emits the canonical object form.""" assert case.input.to_json() == case.output -# --- ZarrMetadataV3.from_json ----------------------------------------------- +# --- NamedConfigModelV3.from_json ----------------------------------------------- ZARR_FROM_JSON_CASES = [ - Expect("bytes", ZarrMetadataV3(name="bytes", configuration={}), id="bare-string"), + Expect("bytes", NamedConfigModelV3(name="bytes", configuration={}), id="bare-string"), Expect( {"name": "regular", "configuration": {"chunk_shape": [1]}}, - ZarrMetadataV3(name="regular", configuration={"chunk_shape": (1,)}), + NamedConfigModelV3(name="regular", configuration={"chunk_shape": (1,)}), id="object-with-config", ), Expect( {"name": "bytes"}, - ZarrMetadataV3(name="bytes", configuration={}), + NamedConfigModelV3(name="bytes", configuration={}), id="object-without-config", ), ] @pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrMetadataV3]) -> None: - """ZarrMetadataV3.from_json parses both the bare-string and object forms.""" - assert ZarrMetadataV3.from_json(case.input) == case.output +def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> None: + """NamedConfigModelV3.from_json parses both the bare-string and object forms.""" + assert NamedConfigModelV3.from_json(case.input) == case.output # --- V3 baseline ----------------------------------------------------------- @@ -170,7 +170,7 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrMetadataV3]) -> Non def test_v3_to_json_includes_required_fields() -> None: """V3 to_json emits all required fields with the expected values.""" out = ArrayMetadataModelV3.create_default( - shape=(10,), data_type=ZarrMetadataV3(name="int32", configuration={}) + shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) ).to_json() assert out["zarr_format"] == 3 assert out["node_type"] == "array" @@ -221,7 +221,7 @@ def test_v3_single_storage_transformer_included() -> None: Regression: the guard used ``> 1`` instead of ``> 0``, dropping a lone storage transformer. """ - st = ZarrMetadataV3(name="some_transformer", configuration={}) + st = NamedConfigModelV3(name="some_transformer", configuration={}) out: dict[str, object] = dict( ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() ) @@ -292,7 +292,7 @@ def test_v3_create_default_is_valid_empty_array() -> None: """V3 create_default builds a structurally valid empty array that round-trips.""" m = ArrayMetadataModelV3.create_default() assert m.shape == () - assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) assert m.fill_value == 0 assert m.attributes == {} assert m.extra_fields == {} @@ -307,7 +307,7 @@ def test_v3_create_default_applies_overrides() -> None: assert m.shape == (4, 4) assert m.attributes == {"a": 1} # un-overridden fields keep their defaults - assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) def test_v2_create_default_is_valid_empty_array() -> None: @@ -464,11 +464,11 @@ def test_v3_from_json_reconstructs_required_fields() -> None: doc = ArrayMetadataModelV3.create_default( shape=(7,), attributes={"a": 1}, - data_type=ZarrMetadataV3(name="int32", configuration={}), + data_type=NamedConfigModelV3(name="int32", configuration={}), ).to_json() model = ArrayMetadataModelV3.from_json(doc) assert model.shape == (7,) - assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) assert model.attributes == {"a": 1} @@ -522,12 +522,12 @@ def test_v3_from_key_value_parses_zarr_json() -> None: FROM_KEY_VALUE_MISSING_PARAMS = [ pytest.param( ArrayMetadataModelV3, - ExpectFail({}, KeyError, id="v3-missing-zarr-json"), + ExpectFail({}, MetadataValidationError, id="v3-missing-zarr-json", msg="missing store key"), id="v3-missing-zarr-json", ), pytest.param( ArrayMetadataModelV2, - ExpectFail({}, KeyError, id="v2-missing-zarray"), + ExpectFail({}, MetadataValidationError, id="v2-missing-zarray", msg="missing store key"), id="v2-missing-zarray", ), ] @@ -538,7 +538,7 @@ def test_from_key_value_missing_key_raises( model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], case: ExpectFail[dict[str, bytes]], ) -> None: - """from_key_value raises KeyError when the required store key is absent.""" + """from_key_value raises MetadataValidationError when the required store key is absent.""" with case.raises(): model_cls.from_key_value(case.input) @@ -551,7 +551,7 @@ def test_from_key_value_missing_key_raises( ArrayMetadataModelV3.create_default( attributes={"a": 1}, dimension_names=("x",), - storage_transformers=(ZarrMetadataV3(name="t", configuration={}),), + storage_transformers=(NamedConfigModelV3(name="t", configuration={}),), extra_fields={"ext": {"must_understand": False}}, ), id="v3-full", @@ -629,7 +629,7 @@ def test_v3_parser_accepts_bare_string_data_type() -> None: doc["data_type"] = "int32" # bare-string form, not canonical object form model = ArrayMetadataModelV3.from_json(doc) # parses correctly, re-serializes to canonical object form - assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} @@ -960,7 +960,7 @@ def test_array_metadata_guards( id="v2-missing-required", ), pytest.param( - ZarrMetadataV3, + NamedConfigModelV3, ExpectFail(lambda: 5, MetadataValidationError, id="x"), id="zarr-metadata-bad-input", ), @@ -969,7 +969,7 @@ def test_array_metadata_guards( @pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) def test_from_json_rejects_malformed( - model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | ZarrMetadataV3], + model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | NamedConfigModelV3], case: ExpectFail[Callable[[], object]], ) -> None: """from_json raises MetadataValidationError on a malformed document.""" @@ -983,19 +983,19 @@ def test_from_json_rejects_malformed( def test_validation_problem_str_with_loc() -> None: """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" - p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str") + p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str", kind="invalid_type") assert str(p) == "codecs.0.name: expected str" def test_validation_problem_str_empty_loc() -> None: """ValidationProblem.__str__ renders an empty loc as .""" - p = ValidationProblem(loc=(), message="not a mapping") + p = ValidationProblem(loc=(), message="not a mapping", kind="invalid_type") assert str(p) == ": not a mapping" def test_validation_problem_is_frozen() -> None: """ValidationProblem is immutable (frozen dataclass).""" - p = ValidationProblem(loc=("shape",), message="x") + p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") with pytest.raises(dataclasses.FrozenInstanceError): p.message = "y" # type: ignore[misc] @@ -1003,8 +1003,10 @@ def test_validation_problem_is_frozen() -> None: def test_metadata_validation_error_holds_problems() -> None: """MetadataValidationError carries its problem list and renders them in its message.""" problems = [ - ValidationProblem(loc=("shape",), message="missing required key"), - ValidationProblem(loc=("data_type",), message="expected a metadata field"), + ValidationProblem(loc=("shape",), message="missing required key", kind="missing_key"), + ValidationProblem( + loc=("data_type",), message="expected a metadata field", kind="invalid_type" + ), ] err = MetadataValidationError(problems) assert err.problems == problems @@ -1014,6 +1016,125 @@ def test_metadata_validation_error_holds_problems() -> None: def test_prefix_prepends_loc_head() -> None: """_prefix prepends a loc head to each problem's loc.""" - problems = [ValidationProblem(loc=("name",), message="expected str")] + problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] prefixed = _prefix(0, problems) - assert prefixed == [ValidationProblem(loc=(0, "name"), message="expected str")] + assert prefixed == [ + ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type") + ] + + +# --- Stricter v2/v3 field validation and error kinds ------------------------- + + +def test_v2_dtype_must_be_string_or_records() -> None: + """A non-string, non-records v2 dtype is rejected with an invalid_type problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": 42} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")] + + +def test_v2_structured_dtype_records_accepted() -> None: + """A structured v2 dtype (field records, optionally nested/shaped) validates.""" + dtype = (("a", " None: + """A field record with the wrong arity is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": (("a",),)} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("dtype",)] + + +def test_v2_order_literal_enforced() -> None: + """An order other than 'C' or 'F' is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"order": "Q"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("order",), "invalid_value")] + + +def test_v2_compressor_must_be_codec_or_none() -> None: + """A compressor that is not null or a codec config mapping is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": "zlib"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("compressor",), "invalid_type")] + + +def test_v2_compressor_requires_string_id() -> None: + """A compressor mapping without a string id is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": {"level": 3}} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("compressor",)] + + +def test_v2_filters_must_be_codec_sequence_or_none() -> None: + """Filters that are not null or a sequence of codec configs are rejected.""" + for bad in (7, (5,), "gzip"): + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"filters": bad} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad + + +def test_v2_dimension_separator_literal_enforced() -> None: + """A dimension_separator other than '.' or '/' is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": "-"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_v2_zarr_format_literal_enforced() -> None: + """A v2 document claiming zarr_format 3 is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"zarr_format": 3} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_zarr_format_literal_enforced() -> None: + """A v3 document claiming zarr_format 2 is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"zarr_format": 2} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_node_type_literal_enforced() -> None: + """A v3 array document claiming node_type 'group' is rejected.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"node_type": "group"} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("node_type",), "invalid_value")] + + +def test_missing_key_kind_is_machine_readable() -> None: + """A missing required key is distinguishable by kind, without message matching.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + del doc["chunk_key_encoding"] + problems = validate_array_metadata_v3(doc) + assert problems == [ + ValidationProblem(("chunk_key_encoding",), "missing required key", "missing_key") + ] + + +# --- Unified error channels --------------------------------------------------- + + +def test_from_key_value_invalid_json_raises_metadata_error() -> None: + """Undecodable store bytes raise MetadataValidationError (kind invalid_json), not JSONDecodeError.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_key_value({"zarr.json": b"{not json"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_from_key_value_missing_key_kind() -> None: + """A missing store key surfaces as a missing_key problem at the store-key loc.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV2.from_key_value({}) + assert exc_info.value.problems == [ + ValidationProblem((".zarray",), "missing store key", "missing_key") + ] + + +def test_extra_fields_overlap_raises_metadata_error() -> None: + """The extra-fields overlap invariant raises MetadataValidationError (a ValueError).""" + with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: + ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 909346586b..829d09b11a 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -18,6 +18,8 @@ MetadataValidationError, parse_group_metadata_v2, parse_group_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, ) # --- GroupMetadataModelV3 --------------------------------------------------- @@ -265,3 +267,20 @@ def test_consolidated_v2_not_a_mapping() -> None: """from_json rejects a non-mapping .zmetadata document.""" with pytest.raises(MetadataValidationError, match="expected a mapping"): ConsolidatedMetadataModelV2.from_json([1]) + + +# --- Literal-value enforcement ----------------------------------------------- + + +def test_group_v3_literals_enforced() -> None: + """A v3 group doc with wrong zarr_format or node_type is rejected with invalid_value.""" + base = GroupMetadataModelV3.create_default().to_json() + for key, bad in (("zarr_format", 2), ("node_type", "array")): + problems = validate_group_metadata_v3(dict(base) | {key: bad}) + assert [(p.loc, p.kind) for p in problems] == [((key,), "invalid_value")], key + + +def test_group_v2_zarr_format_literal_enforced() -> None: + """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" + problems = validate_group_metadata_v2({"zarr_format": 3}) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 7a6db7ab30..cd0196d47b 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -50,9 +50,10 @@ def _group_rank(s: str) -> int: "GroupMetadataModelV3Partial", "ConsolidatedMetadataModelV2", "ConsolidatedMetadataModelV3", - "ZarrMetadataV3", + "NamedConfigModelV3", "ValidationProblem", "MetadataValidationError", + "ProblemKind", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From be736dabd317fda8da16075c697450393aa4c906 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 15:10:26 +0200 Subject: [PATCH 10/48] feat(zarr-metadata): annotate metadata fields with role alias MetadataFieldModelV3 Model fields and consumer signatures should convey the logical meaning of the type (a metadata-document field), not the form it takes when JSON-serialized (a named configuration). MetadataFieldModelV3 is today exactly NamedConfigModelV3; if a future spec revision adds a field form that cannot normalize to name + configuration, the alias widens to a union and annotation sites do not move. Mirrors the raw-layer split between NamedConfigV3 (shape) and MetadataV3 (field union). Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 ++- .../src/zarr_metadata/__init__.py | 2 + .../src/zarr_metadata/model/__init__.py | 2 + .../src/zarr_metadata/model/_array.py | 41 +++++++++++++------ .../zarr-metadata/tests/model/test_array.py | 13 ++++++ .../zarr-metadata/tests/test_public_api.py | 1 + 6 files changed, 50 insertions(+), 14 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 13517ce38a..326aee1b42 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -4,7 +4,10 @@ Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, that are canonical, lossless representations of Zarr metadata documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, storage transformers) -is held as a name + configuration pair; nothing is interpreted. +is held as a name + configuration pair; nothing is interpreted. Model fields +are annotated with the role alias `MetadataFieldModelV3` (today exactly +`NamedConfigModelV3`), so the annotations convey the logical meaning of the +field and stay put if the spec ever adds a new field form. Validation is strict about what the types declare: v2 `dtype` / `order` / `compressor` / `filters` / `dimension_separator` shapes and the fixed diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index ecd7e342cf..354f2a97f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -12,6 +12,7 @@ GroupMetadataModelV2Partial, GroupMetadataModelV3, GroupMetadataModelV3Partial, + MetadataFieldModelV3, MetadataValidationError, NamedConfigModelV3, ProblemKind, @@ -316,6 +317,7 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", + "MetadataFieldModelV3", "MetadataV3", "MetadataValidationError", "NamedConfigModelV3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index aa8555d623..42fbc233d8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -19,6 +19,7 @@ ArrayMetadataModelV2Partial, ArrayMetadataModelV3, ArrayMetadataModelV3Partial, + MetadataFieldModelV3, NamedConfigModelV3, ) from zarr_metadata.model._group import ( @@ -91,6 +92,7 @@ "GroupMetadataModelV2Partial", "GroupMetadataModelV3", "GroupMetadataModelV3Partial", + "MetadataFieldModelV3", "MetadataValidationError", "NamedConfigModelV3", "ProblemKind", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 4ec02137ec..33682c2008 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -5,7 +5,7 @@ import dataclasses import json from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from typing_extensions import TypedDict, Unpack @@ -68,6 +68,20 @@ def from_json(cls, data: object) -> NamedConfigModelV3: return cls(name=field["name"], configuration=configuration) # type: ignore[arg-type] +MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 +"""The in-memory model of one field of a v3 metadata document. + +This is the role-named alias for annotation positions: model fields and +consumer signatures should say `MetadataFieldModelV3` (the logical meaning) +rather than `NamedConfigModelV3` (the serialized form the field currently +takes). Today every metadata field normalizes to a named configuration, so +the alias is exactly `NamedConfigModelV3`; if a future spec revision adds a +field form that cannot be normalized to name + configuration, this alias +widens to a union and annotation sites do not change. Mirrors the raw-layer +split between `NamedConfigV3` (shape) and `MetadataV3` (field union). +""" + + class ArrayMetadataModelV3Partial(TypedDict, total=False): """ Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. @@ -84,13 +98,13 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): shape: tuple[int, ...] fill_value: JSONValue - data_type: NamedConfigModelV3 - chunk_grid: NamedConfigModelV3 - codecs: tuple[NamedConfigModelV3, ...] - chunk_key_encoding: NamedConfigModelV3 + data_type: MetadataFieldModelV3 + chunk_grid: MetadataFieldModelV3 + codecs: tuple[MetadataFieldModelV3, ...] + chunk_key_encoding: MetadataFieldModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[NamedConfigModelV3, ...] + storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -100,8 +114,9 @@ class ArrayMetadataModelV3: A canonical, lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, - `codecs`, `storage_transformers`) are held as `NamedConfigModelV3` name + - configuration pairs and are never interpreted; `fill_value` is held + `codecs`, `storage_transformers`) are held as `MetadataFieldModelV3` + values (currently always `NamedConfigModelV3` name + configuration pairs) + and are never interpreted; `fill_value` is held verbatim in its JSON form. """ @@ -109,13 +124,13 @@ class ArrayMetadataModelV3: node_type: Literal["array"] = field(default="array", init=False) shape: tuple[int, ...] fill_value: JSONValue - data_type: NamedConfigModelV3 - chunk_grid: NamedConfigModelV3 - codecs: tuple[NamedConfigModelV3, ...] - chunk_key_encoding: NamedConfigModelV3 + data_type: MetadataFieldModelV3 + chunk_grid: MetadataFieldModelV3 + codecs: tuple[MetadataFieldModelV3, ...] + chunk_key_encoding: MetadataFieldModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[NamedConfigModelV3, ...] + storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @classmethod diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 904e0b12fa..e825f0211f 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -16,6 +16,7 @@ ArrayMetadataModelV2Partial, ArrayMetadataModelV3, ArrayMetadataModelV3Partial, + MetadataFieldModelV3, MetadataValidationError, NamedConfigModelV3, ValidationProblem, @@ -1138,3 +1139,15 @@ def test_extra_fields_overlap_raises_metadata_error() -> None: with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] + + +def test_extension_point_fields_annotated_with_role_alias() -> None: + """Extension-point fields are annotated with MetadataFieldModelV3 (the + logical role), not NamedConfigModelV3 (the current serialized form), so a + future widening of the field union does not move annotation sites.""" + assert MetadataFieldModelV3 is NamedConfigModelV3 + annotations = ArrayMetadataModelV3.__annotations__ + for field_name in ("data_type", "chunk_grid", "chunk_key_encoding"): + assert annotations[field_name] == "MetadataFieldModelV3" + for field_name in ("codecs", "storage_transformers"): + assert annotations[field_name] == "tuple[MetadataFieldModelV3, ...]" diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index cd0196d47b..fb2f574a4f 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -51,6 +51,7 @@ def _group_rank(s: str) -> int: "ConsolidatedMetadataModelV2", "ConsolidatedMetadataModelV3", "NamedConfigModelV3", + "MetadataFieldModelV3", "ValidationProblem", "MetadataValidationError", "ProblemKind", From 7fc0211e4ca329ddbab3b4b3ee718fc173719072 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 15:15:18 +0200 Subject: [PATCH 11/48] test(zarr-metadata): assert required-key coverage via the typed constant test_v3_to_json_includes_required_fields hand-enumerated keys with chained asserts, restating what ARRAY_METADATA_REQUIRED_KEYS_V3 already defines. Now: one coverage assert driven by the constant (tracks the TypedDict automatically) and one whole-document equality for the values. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/tests/model/test_array.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index e825f0211f..8806eb8490 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -169,16 +169,22 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> def test_v3_to_json_includes_required_fields() -> None: - """V3 to_json emits all required fields with the expected values.""" + """V3 to_json emits every spec-required key (per the typed constant), + and the whole document matches the model's values.""" out = ArrayMetadataModelV3.create_default( shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) ).to_json() - assert out["zarr_format"] == 3 - assert out["node_type"] == "array" - assert out["shape"] == (10,) - assert out["fill_value"] == 0 - assert out["data_type"] == {"name": "int32", "configuration": {}} - assert out["codecs"] == ({"name": "bytes", "configuration": {}},) + assert out.keys() >= ARRAY_METADATA_REQUIRED_KEYS_V3 + assert out == { + "zarr_format": 3, + "node_type": "array", + "shape": (10,), + "fill_value": 0, + "data_type": {"name": "int32", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": ()}}, + "codecs": ({"name": "bytes", "configuration": {}},), + "chunk_key_encoding": {"name": "default", "configuration": {}}, + } def test_v3_dimension_names_included_when_present() -> None: From 15e02189def66f60923209aac44368c60535349b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 15:16:12 +0200 Subject: [PATCH 12/48] test(zarr-metadata): single whole-document comparison for v3 to_json The subset assert against ARRAY_METADATA_REQUIRED_KEYS_V3 was redundant: equality with a literal that spells out the full document already covers every required key. One dict, one assert. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/tests/model/test_array.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 8806eb8490..0df935144b 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -168,13 +168,12 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> # --- V3 baseline ----------------------------------------------------------- -def test_v3_to_json_includes_required_fields() -> None: - """V3 to_json emits every spec-required key (per the typed constant), - and the whole document matches the model's values.""" +def test_v3_to_json_emits_canonical_document() -> None: + """V3 to_json emits exactly the expected document (which covers every + spec-required key by construction).""" out = ArrayMetadataModelV3.create_default( shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) ).to_json() - assert out.keys() >= ARRAY_METADATA_REQUIRED_KEYS_V3 assert out == { "zarr_format": 3, "node_type": "array", From c0a40cf6bd1c3e9d2b355a86279ddb25c0db557b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 16:52:10 +0200 Subject: [PATCH 13/48] fix(zarr-metadata): close validation holes found by adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Invalid documents that previously passed validation: - shape/chunks containing JSON booleans (bool is an int subclass in Python but not an integer in a metadata document) or negative values - dimension_names whose length does not match shape - attributes and configuration values that are not JSON-serializable — now checked recursively like fill_value, so an int-keyed dict cannot be silently rewritten by json.dumps on round-trip and a set() cannot escape as a TypeError from to_key_value - consolidated_metadata envelopes: the group validator now deep-validates the envelope and its entries via the shared validate_consolidated_metadata_v3, which ConsolidatedMetadataModelV3 .from_json also uses, so is_group_metadata_v3 never vouches for a document the model constructor would reject Three pre-existing test fixtures paired dimension_names=('x',) with the default scalar shape () and were themselves spec-invalid; they now use a matching 1-d shape. Deliberately unchanged, pending a design decision: unknown extension fields with must_understand: true still pass (which layer owns the spec's refusal duty), and empty v2 dtype records / empty codec names still pass (domain territory). Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 8 +- .../src/zarr_metadata/model/_group.py | 49 ++------ .../src/zarr_metadata/model/_validation.py | 119 +++++++++++++++--- .../zarr-metadata/tests/model/test_array.py | 65 +++++++++- .../zarr-metadata/tests/model/test_group.py | 51 ++++++++ 5 files changed, 235 insertions(+), 57 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 326aee1b42..fbecb7b4a4 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -16,4 +16,10 @@ Validation is strict about what the types declare: v2 `dtype` / `order` / (`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so consumers can dispatch on the failure mode without matching message strings, and every ingestion failure — including missing store keys and undecodable -bytes in `from_key_value` — surfaces as `MetadataValidationError`. +bytes in `from_key_value` — surfaces as `MetadataValidationError`. An +adversarial review added further structural checks: JSON booleans are not +accepted as dimension lengths, dimensions are non-negative, +`dimension_names` must have one entry per dimension of `shape`, `attributes` +and `configuration` values are JSON-checked recursively (like `fill_value`), +and the inline consolidated-metadata envelope and entries are deep-validated +so the group validator's verdict always agrees with the model constructor. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 6128b0306b..b138a7d0f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -22,6 +22,7 @@ load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, + validate_consolidated_metadata_v3, ) if TYPE_CHECKING: @@ -207,48 +208,18 @@ def to_json(self) -> ConsolidatedMetadataV3: @classmethod def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: - if not isinstance(data, Mapping): - raise MetadataValidationError( - [ValidationProblem((), "expected a mapping", "invalid_type")] - ) - doc = cast("Mapping[str, object]", data) - entries_raw = doc.get("metadata") - if not isinstance(entries_raw, Mapping): - raise MetadataValidationError( - [ValidationProblem(("metadata",), "expected a mapping", "invalid_type")] - ) + problems = validate_consolidated_metadata_v3(data) + if problems: + raise MetadataValidationError(problems) + env = cast("Mapping[str, object]", data) entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} - problems: list[ValidationProblem] = [] - for key, entry in cast("Mapping[object, object]", entries_raw).items(): - if not isinstance(key, str): - problems.append( - ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") - ) - continue - entry_obj: object = entry - node_type: object = None - if isinstance(entry, Mapping): - node_type = cast("Mapping[str, object]", entry).get("node_type") + for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): + node_type = cast("Mapping[str, object]", entry).get("node_type") if node_type == "array": - entries[key] = ArrayMetadataModelV3.from_json(entry_obj) - elif node_type == "group": - entries[key] = GroupMetadataModelV3.from_json(entry_obj) + entries[key] = ArrayMetadataModelV3.from_json(entry) else: - problems.append( - ValidationProblem( - ("metadata", key, "node_type"), - "expected 'array' or 'group'", - "invalid_value", - ) - ) - if problems: - raise MetadataValidationError(problems) - must_understand = doc.get("must_understand", False) - if must_understand is not False: - raise MetadataValidationError( - [ValidationProblem(("must_understand",), "expected False", "invalid_value")] - ) - return cls(must_understand=must_understand, metadata=entries) + entries[key] = GroupMetadataModelV3.from_json(entry) + return cls(metadata=entries) class GroupMetadataModelV2Partial(TypedDict, total=False): diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 304a80d4ab..fc781c7899 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -190,6 +190,9 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: problems.append( ValidationProblem(("configuration",), "expected string keys", "invalid_type") ) + else: + for key, item in cast("Mapping[str, object]", configuration).items(): + problems.extend(_prefix("configuration", _prefix(key, validate_json(item)))) return problems @@ -207,14 +210,36 @@ def parse_metadata_field_v3(value: object) -> MetadataV3: def _is_int_sequence(value: object) -> bool: - """Whether `value` is a non-string sequence of integers.""" + """Whether `value` is a non-string sequence of integers. + + JSON booleans decode to `bool`, which is an `int` subclass in Python but + is not an integer in a metadata document, so booleans are excluded. + """ return ( not isinstance(value, str) and isinstance(value, Sequence) - and all(isinstance(item, int) for item in cast("Sequence[object]", value)) + and all( + isinstance(item, int) and not isinstance(item, bool) + for item in cast("Sequence[object]", value) + ) ) +def _validate_dim_sequence(doc: Mapping[str, object], key: str) -> list[ValidationProblem]: + """Validate a dimension sequence (`shape` / `chunks`) if present in `doc`. + + Dimension lengths are non-negative integers. + """ + if key not in doc: + return [] + value = doc[key] + if not _is_int_sequence(value): + return [ValidationProblem((key,), "expected a sequence of int", "invalid_type")] + if any(item < 0 for item in cast("Sequence[int]", value)): + return [ValidationProblem((key,), "expected non-negative integers", "invalid_value")] + return [] + + def _is_dtype_v2(value: object) -> bool: """Whether `value` is shaped like a v2 dtype: a string or field records. @@ -266,7 +291,10 @@ def _validate_attributes(value: object) -> list[ValidationProblem]: ("attributes",), "expected a mapping with string keys", "invalid_type" ) ] - return [] + problems: list[ValidationProblem] = [] + for key, item in cast("Mapping[str, object]", value).items(): + problems.extend(_prefix("attributes", _prefix(key, validate_json(item)))) + return problems def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: @@ -281,8 +309,7 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) problems.extend(_check_literal(doc, "zarr_format", 3)) problems.extend(_check_literal(doc, "node_type", "array")) - if "shape" in doc and not _is_int_sequence(doc["shape"]): - problems.append(ValidationProblem(("shape",), "expected a sequence of int", "invalid_type")) + problems.extend(_validate_dim_sequence(doc, "shape")) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): @@ -315,6 +342,16 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: ("dimension_names",), "expected items of str or None", "invalid_type" ) ) + elif _is_int_sequence(doc.get("shape")) and len(cast("Sequence[object]", names)) != len( + cast("Sequence[int]", doc["shape"]) + ): + problems.append( + ValidationProblem( + ("dimension_names",), + "expected one name per dimension of shape", + "invalid_value", + ) + ) return problems @@ -344,11 +381,8 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) problems.extend(_check_literal(doc, "zarr_format", 2)) - problems.extend( - ValidationProblem((key,), "expected a sequence of int", "invalid_type") - for key in ("shape", "chunks") - if key in doc and not _is_int_sequence(doc[key]) - ) + problems.extend(_validate_dim_sequence(doc, "shape")) + problems.extend(_validate_dim_sequence(doc, "chunks")) if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): problems.append( ValidationProblem( @@ -415,12 +449,66 @@ def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: return cast(ArrayMetadataV2, value) +def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a valid inline consolidated envelope. + + Locs are value-relative (the caller prefixes with `consolidated_metadata` + where appropriate). Entries recurse into the array and group document + validators, so a validator verdict always agrees with what + `ConsolidatedMetadataModelV3.from_json` accepts. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + env = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in ("kind", "must_understand", "metadata") + if key not in env + ] + problems.extend(_check_literal(env, "kind", "inline")) + if "must_understand" in env and env["must_understand"] is not False: + problems.append(ValidationProblem(("must_understand",), "expected False", "invalid_value")) + if "metadata" in env: + entries = env["metadata"] + if not isinstance(entries, Mapping): + problems.append(ValidationProblem(("metadata",), "expected a mapping", "invalid_type")) + else: + for key, entry in cast("Mapping[object, object]", entries).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") + ) + continue + entry_obj: object = entry + node_type: object = None + if isinstance(entry, Mapping): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + problems.extend( + _prefix("metadata", _prefix(key, validate_array_metadata_v3(entry_obj))) + ) + elif node_type == "group": + problems.extend( + _prefix("metadata", _prefix(key, validate_group_metadata_v3(entry_obj))) + ) + else: + problems.append( + ValidationProblem( + ("metadata", key, "node_type"), + "expected 'array' or 'group'", + "invalid_value", + ) + ) + return problems + + def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a structurally-valid v3 group doc. Checks structure, not domain validity. Unknown top-level keys are allowed (they map to `extra_fields`); a `consolidated_metadata` key, if present, - must be a mapping (its entries are validated by the consolidated model). + is deep-validated (envelope and entries) via + `validate_consolidated_metadata_v3`. """ if not isinstance(value, Mapping): return [ValidationProblem((), "expected a mapping", "invalid_type")] @@ -430,9 +518,12 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) - if "consolidated_metadata" in doc and not isinstance(doc["consolidated_metadata"], Mapping): - problems.append( - ValidationProblem(("consolidated_metadata",), "expected a mapping", "invalid_type") + if "consolidated_metadata" in doc: + problems.extend( + _prefix( + "consolidated_metadata", + validate_consolidated_metadata_v3(doc["consolidated_metadata"]), + ) ) return problems diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 0df935144b..9032f18638 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -501,7 +501,9 @@ def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: """V3 from_json keeps standard keys out of extra_fields.""" - doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + doc = ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() model = ArrayMetadataModelV3.from_json(doc) assert model.extra_fields == {} @@ -555,6 +557,7 @@ def test_from_key_value_missing_key_raises( pytest.param( ArrayMetadataModelV3, ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",), storage_transformers=(NamedConfigModelV3(name="t", configuration={}),), @@ -619,7 +622,9 @@ def test_roundtrip_via_key_value( def test_v3_roundtrip_json_model_json() -> None: """A v3 document round-trips through from_json/to_json back to an equal document.""" - doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + doc = ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() assert ArrayMetadataModelV3.from_json(doc).to_json() == doc @@ -837,7 +842,7 @@ def _set(key: str, value: object) -> Callable[[dict], object]: V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ Expect(_build_v3, frozenset(), id="valid"), Expect( - lambda: _build_v3(attributes={"a": 1}, dimension_names=("x",)), + lambda: _build_v3(shape=(10,), attributes={"a": 1}, dimension_names=("x",)), frozenset(), id="valid-with-attributes-and-dim-names", ), @@ -1156,3 +1161,57 @@ def test_extension_point_fields_annotated_with_role_alias() -> None: assert annotations[field_name] == "MetadataFieldModelV3" for field_name in ("codecs", "storage_transformers"): assert annotations[field_name] == "tuple[MetadataFieldModelV3, ...]" + + +# --- Adversarial-probe fixes: documents that used to pass validation --------- + + +def test_shape_rejects_json_booleans() -> None: + """JSON booleans are not integers: shape/chunks containing true/false are + rejected (bool is an int subclass in Python, so isinstance alone passes).""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (True, True)} + assert [p.loc for p in validate_array_metadata_v3(v3)] == [("shape",)] + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (True,)} + assert [p.loc for p in validate_array_metadata_v2(v2)] == [("chunks",)] + + +def test_shape_rejects_negative_dimensions() -> None: + """Dimension lengths must be non-negative; a negative entry is invalid_value.""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (-1,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_value") + ] + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (-5,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_value") + ] + + +def test_dimension_names_length_must_match_shape() -> None: + """dimension_names must have one entry per dimension of shape.""" + doc = dict(ArrayMetadataModelV3.create_default(shape=(10,)).to_json()) | { + "dimension_names": ("x", "y", "z") + } + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(doc)] == [ + (("dimension_names",), "invalid_value") + ] + + +def test_attributes_values_must_be_json() -> None: + """Attribute values are JSON-checked recursively (like fill_value), so a + non-serializable value is a validation problem, not a later TypeError.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"attributes": {"a": {1, 2}}} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("attributes", "a"), "invalid_type")] + + +def test_configuration_values_must_be_json() -> None: + """Configuration values are JSON-checked recursively, so an int-keyed dict + cannot pass validation and be silently rewritten by json.dumps.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | { + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": {1: 2}}} + } + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [ + (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") + ] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 829d09b11a..329471cd94 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -284,3 +284,54 @@ def test_group_v2_zarr_format_literal_enforced() -> None: """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" problems = validate_group_metadata_v2({"zarr_format": 3}) assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +# --- Consolidated envelope validated by the group validator ------------------ + + +def test_group_v3_validator_agrees_with_from_json_on_consolidated() -> None: + """The group validator validates the consolidated envelope and entries, so + is_group_metadata_v3 never vouches for a document from_json would reject.""" + bad_docs = ( + # empty envelope: missing kind/must_understand/metadata + {"zarr_format": 3, "node_type": "group", "consolidated_metadata": {}}, + # entry without a recognizable node_type + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": {"zarr_format": 3}}, + }, + }, + # must_understand: true + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": True, + "metadata": {}, + }, + }, + ) + for doc in bad_docs: + assert validate_group_metadata_v3(doc) != [], doc + with pytest.raises(MetadataValidationError): + GroupMetadataModelV3.from_json(doc) + + +def test_group_v3_valid_consolidated_passes_validator() -> None: + """A well-formed consolidated group validates cleanly (control case).""" + child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + assert validate_group_metadata_v3(doc) == [] From b6e53ce02b8db5e95d48683ee4e3974081abffce Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:09:31 +0200 Subject: [PATCH 14/48] feat(zarr-metadata): expose must_understand_fields on the v3 models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v3 core spec: 'An implementation MUST fail to open Zarr groups or arrays if any metadata fields are present which (a) the implementation does not recognize and (b) are not explicitly set to "must_understand": false' — and fields are implicitly must-understand unless waived. The model layer cannot discharge this itself: recognition is reader-specific (consolidated_metadata is itself an extension field one reader understands and another does not), and a document carrying a must-understand extension is still a valid document. So the models partition by obligation: must_understand_fields is the subset of extra_fields not explicitly waived, and a compliant reader fails to open when must_understand_fields.keys() - recognized is non-empty. The design spec pins that duty on the part-2 resolve layer, matching what zarr-python's parse_extra_fields enforces today. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 6 +++ .../src/zarr_metadata/model/_array.py | 44 +++++++++++++++++-- .../src/zarr_metadata/model/_group.py | 13 ++++++ .../zarr-metadata/tests/model/test_array.py | 29 ++++++++++++ .../zarr-metadata/tests/model/test_group.py | 15 +++++++ 5 files changed, 104 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index fbecb7b4a4..f35625558b 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -23,3 +23,9 @@ accepted as dimension lengths, dimensions are non-negative, and `configuration` values are JSON-checked recursively (like `fill_value`), and the inline consolidated-metadata envelope and entries are deep-validated so the group validator's verdict always agrees with the model constructor. + +The v3 models expose `must_understand_fields`: the subset of `extra_fields` +not explicitly waived with `must_understand: false` (fields are implicitly +must-understand per the spec). Readers discharge the spec's fail-to-open +duty by subtracting the extension names they recognize; the model only +partitions by obligation, since recognition is reader-specific. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 33682c2008..7a4487b7f4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -4,8 +4,9 @@ import dataclasses import json +from collections.abc import Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Final, Literal, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast from typing_extensions import TypedDict, Unpack @@ -21,8 +22,6 @@ ) if TYPE_CHECKING: - from collections.abc import Mapping - from zarr_metadata._common import JSONValue from zarr_metadata.v2.array import ( ArrayDimensionSeparatorV2, @@ -82,6 +81,33 @@ def from_json(cls, data: object) -> NamedConfigModelV3: """ +def must_understand_subset( + extra_fields: Mapping[str, ExtensionFieldV3], +) -> dict[str, ExtensionFieldV3]: + """The subset of `extra_fields` the reader is obligated to understand. + + Per the v3 spec, an extension field is implicitly `must_understand: True` + unless it explicitly says otherwise, and an implementation MUST fail to + open a group or array carrying fields it does not recognize that are not + explicitly `must_understand: false`. A non-mapping field value cannot + carry the explicit waiver, so it always requires understanding (the + runtime isinstance check defends against values looser than the declared + `ExtensionFieldV3`). + """ + fields = cast("Mapping[str, object]", extra_fields) + return cast( + "dict[str, ExtensionFieldV3]", + { + name: value + for name, value in fields.items() + if not ( + isinstance(value, Mapping) + and cast("Mapping[str, object]", value).get("must_understand") is False + ) + }, + ) + + class ArrayMetadataModelV3Partial(TypedDict, total=False): """ Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. @@ -240,6 +266,18 @@ def from_json(cls, data: object) -> ArrayMetadataModelV3: extra_fields=extra_fields, ) + @property + def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the array if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index b138a7d0f5..b8cca3849d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -13,6 +13,7 @@ from zarr_metadata.model._array import ( ATTRIBUTES_STORE_KEY_V2, ArrayMetadataModelV3, + must_understand_subset, ) from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, @@ -159,6 +160,18 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: extra_fields=extra_fields, ) + @property + def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the group if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 9032f18638..e73739929a 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1215,3 +1215,32 @@ def test_configuration_values_must_be_json() -> None: assert [(p.loc, p.kind) for p in problems] == [ (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") ] + + +# --- must_understand partition (spec: MUST fail to open unrecognized fields) -- + + +def test_must_understand_fields_partition() -> None: + """must_understand_fields contains every extra field not explicitly waived + with must_understand: false, including implicitly-true and non-mapping + fields, so a reader can discharge the spec's fail-to-open duty by + subtracting the extensions it recognizes.""" + model = ArrayMetadataModelV3.create_default( + extra_fields={ + "ext_a": {"name": "a", "must_understand": False}, + "ext_b": {"name": "b"}, + "ext_c": {"name": "c", "must_understand": True}, + "ext_d": 123, + } + ) + assert set(model.must_understand_fields) == {"ext_b", "ext_c", "ext_d"} + recognized = {"ext_b"} + assert model.must_understand_fields.keys() - recognized == {"ext_c", "ext_d"} + + +def test_must_understand_fields_empty_when_all_waived() -> None: + """must_understand_fields is empty when every extra field is explicitly waived.""" + model = ArrayMetadataModelV3.create_default( + extra_fields={"ext_a": {"name": "a", "must_understand": False}} + ) + assert model.must_understand_fields == {} diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 329471cd94..d2399220be 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -335,3 +335,18 @@ def test_group_v3_valid_consolidated_passes_validator() -> None: }, } assert validate_group_metadata_v3(doc) == [] + + +# --- must_understand partition ------------------------------------------------ + + +def test_group_must_understand_fields_partition() -> None: + """The group model partitions extra fields by the spec's implicit-true rule, + like the array model.""" + model = GroupMetadataModelV3.create_default( + extra_fields={ + "waived": {"name": "w", "must_understand": False}, + "implicit": {"name": "i"}, + } + ) + assert set(model.must_understand_fields) == {"implicit"} From 22188c9fddb703450049ae335c039de600fc5c11 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:18:27 +0200 Subject: [PATCH 15/48] test(zarr-metadata): executable example of pydantic integration Delegate wholesale rather than letting pydantic introspect the dataclass: InstanceOf (is-instance core schema) + BeforeValidator(from_json) + PlainSerializer(to_json, return_type=dict). Field-by-field validation is impossible anyway (the models' annotation-only imports live behind TYPE_CHECKING, so pydantic raises class-not-fully-defined) and would diverge from the library's structural validation via coercion if it weren't. MetadataValidationError subclasses ValueError, so failed parses surface as pydantic ValidationError with the loc-annotated messages. pydantic is already in the package's test dependency group. Assisted-by: ClaudeCode:claude-fable-5 --- .../tests/model/test_pydantic.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 packages/zarr-metadata/tests/model/test_pydantic.py diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py new file mode 100644 index 0000000000..23f69aaf52 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -0,0 +1,130 @@ +"""Executable example: integrating the metadata models with pydantic (v2). + +Do NOT hand the dataclass to pydantic for field-by-field validation: the +models keep their annotation-only imports behind `TYPE_CHECKING` (so the +string annotations are unresolvable at runtime, and pydantic raises +`class-not-fully-defined`), and even if they resolved, pydantic's coercion +rules would diverge from the library's structural validation. Delegate +wholesale instead — treat the model as an opaque value: + +- `InstanceOf` makes pydantic's core schema an is-instance check (no field + introspection), +- validation goes through `from_json` (the single source of truth for what + a well-formed document is, including normalization: bare-string metadata + fields, arrays-to-tuples), +- serialization goes through `to_json` (the canonical document form). + +`MetadataValidationError` subclasses `ValueError`, so pydantic converts a +failed parse into its own `ValidationError` with the loc-annotated problem +messages intact. +""" + +from typing import Annotated + +import pytest +from pydantic import ( + BaseModel, + BeforeValidator, + InstanceOf, + PlainSerializer, + TypeAdapter, + ValidationError, +) + +from zarr_metadata.model import ArrayMetadataModelV3 + +# --- the integration (this is the example) ----------------------------------- + + +def _as_array_metadata_v3(value: object) -> ArrayMetadataModelV3: + """Accept an existing model instance or a raw metadata document.""" + if isinstance(value, ArrayMetadataModelV3): + return value + return ArrayMetadataModelV3.from_json(value) + + +# return_type is explicit because to_json's own annotation (`ArrayMetadataV3`) +# is a TYPE_CHECKING-only name pydantic cannot resolve at runtime. +ArrayMetadataV3Field = Annotated[ + InstanceOf[ArrayMetadataModelV3], + BeforeValidator(_as_array_metadata_v3), + PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), +] +"""A pydantic-ready field type for v3 array metadata. + +Validates raw documents via `from_json`, passes model instances through, +and serializes to the canonical document form via `to_json`. +""" + + +class ArrayManifest(BaseModel): + """Example consumer model: a named array with its metadata document.""" + + path: str + metadata: ArrayMetadataV3Field + + +# --- tests pinning the example ------------------------------------------------ + +VALID_DOC = { + "zarr_format": 3, + "node_type": "array", + "shape": [10], + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, + "chunk_key_encoding": {"name": "default"}, + "codecs": [{"name": "bytes"}], +} + + +def test_raw_document_is_validated_into_a_model() -> None: + """A raw metadata document on a pydantic field is parsed by from_json, + with the library's normalization applied (tuples, canonical field form).""" + manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + assert isinstance(manifest.metadata, ArrayMetadataModelV3) + assert manifest.metadata.shape == (10,) + assert manifest.metadata.data_type.name == "uint8" + + +def test_model_instance_passes_through() -> None: + """An already-constructed model instance is accepted unchanged.""" + model = ArrayMetadataModelV3.from_json(VALID_DOC) + manifest = ArrayManifest(path="a/b", metadata=model) + assert manifest.metadata is model + + +def test_invalid_document_surfaces_problems_in_validation_error() -> None: + """A structurally-invalid document fails pydantic validation, carrying the + loc-annotated problem messages from MetadataValidationError.""" + doc = dict(VALID_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError) as exc_info: + ArrayManifest(path="a/b", metadata=doc) # type: ignore[arg-type] + assert "chunk_key_encoding: missing required key" in str(exc_info.value) + + +def test_dump_emits_canonical_document() -> None: + """model_dump serializes the field via to_json — the canonical document, + not pydantic's field-by-field view of the dataclass.""" + manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + dumped = manifest.model_dump() + assert dumped["metadata"] == manifest.metadata.to_json() + # the bare-string data_type was normalized to the canonical object form + assert dumped["metadata"]["data_type"] == {"name": "uint8", "configuration": {}} + + +def test_json_roundtrip_through_pydantic() -> None: + """model_dump_json output re-validates to an equal manifest (JSON emits + tuples as arrays; from_json converts them back).""" + manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) + assert revived == manifest + + +def test_type_adapter_standalone() -> None: + """The annotated alias also works without a BaseModel, via TypeAdapter.""" + adapter = TypeAdapter(ArrayMetadataV3Field) + model = adapter.validate_python(VALID_DOC) + assert isinstance(model, ArrayMetadataModelV3) + assert adapter.dump_python(model) == model.to_json() From 6ea15335d2ce732ba49efd614308c568ad1b212e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:24:03 +0200 Subject: [PATCH 16/48] test(zarr-metadata): document pydantic's native dataclass path and why not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correcting the previous commit's too-strong claim: pydantic CAN introspect the model dataclass — TypeAdapter(...).rebuild() with the TYPE_CHECKING-only names supplied as _types_namespace resolves the schema, and __post_init__ invariants still run. A new test exercises that path and pins why it is not the recommended integration: it validates the model shape, not the document (bare-string data_type rejected — no from_json normalization), and pydantic's lax coercion silently re-opens holes the library validators close (shape=[True, -5] coerces to (1, -5); a wrong dimension_names count passes). Assisted-by: ClaudeCode:claude-fable-5 --- .../tests/model/test_pydantic.py | 74 +++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 23f69aaf52..cfd077651a 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -1,11 +1,16 @@ """Executable example: integrating the metadata models with pydantic (v2). -Do NOT hand the dataclass to pydantic for field-by-field validation: the -models keep their annotation-only imports behind `TYPE_CHECKING` (so the -string annotations are unresolvable at runtime, and pydantic raises -`class-not-fully-defined`), and even if they resolved, pydantic's coercion -rules would diverge from the library's structural validation. Delegate -wholesale instead — treat the model as an opaque value: +Pydantic's native dataclass introspection CAN be made to work (see +`test_native_dataclass_introspection_is_possible_but_diverges`): the models +keep their annotation-only imports behind `TYPE_CHECKING`, so a bare +`TypeAdapter(ArrayMetadataModelV3)` raises `class-not-fully-defined`, but +`rebuild(_types_namespace=...)` with the names supplied resolves the schema. +It is still the wrong tool: it validates the MODEL SHAPE, not the DOCUMENT — +no `from_json` normalization (a bare-string `data_type` is rejected), and +pydantic's lax coercion silently re-opens holes the library's validators +close (`shape=[True, -5]` coerces to `(1, -5)`; a wrong `dimension_names` +count passes). The recommended integration delegates wholesale — treat the +model as an opaque value: - `InstanceOf` makes pydantic's core schema an is-instance check (no field introspection), @@ -128,3 +133,60 @@ def test_type_adapter_standalone() -> None: model = adapter.validate_python(VALID_DOC) assert isinstance(model, ArrayMetadataModelV3) assert adapter.dump_python(model) == model.to_json() + + +# --- the road not taken: native dataclass introspection ---------------------- + + +def test_native_dataclass_introspection_is_possible_but_diverges() -> None: + """Pydantic CAN introspect the model dataclass after a namespace rebuild, + but that path validates the model shape, not the document: it rejects the + document form, coerces booleans into dimensions, and skips the library's + cross-field checks. This test documents why the delegation pattern above + is the recommended integration.""" + from zarr_metadata._common import JSONValue + from zarr_metadata.model import NamedConfigModelV3 + from zarr_metadata.v3._common import MetadataV3 + from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 + + adapter = TypeAdapter(ArrayMetadataModelV3) + adapter.rebuild( + force=True, + _types_namespace={ + "JSONValue": JSONValue, + "ExtensionFieldV3": ExtensionFieldV3, + "MetadataV3": MetadataV3, + "ArrayMetadataV3": ArrayMetadataV3, + "NamedConfigModelV3": NamedConfigModelV3, + "MetadataFieldModelV3": NamedConfigModelV3, + }, + ) + model_shaped = { + "shape": [10], + "fill_value": 0, + "data_type": {"name": "uint8", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, + "codecs": [{"name": "bytes", "configuration": {}}], + "chunk_key_encoding": {"name": "default", "configuration": {}}, + "dimension_names": None, + "attributes": {}, + "storage_transformers": [], + "extra_fields": {}, + } + # model-shaped data validates, nested named configs and all + model = adapter.validate_python(model_shaped) + assert isinstance(model.data_type, NamedConfigModelV3) + + # divergence 1: the DOCUMENT form is rejected — no from_json normalization + with pytest.raises(ValidationError): + adapter.validate_python(model_shaped | {"data_type": "uint8"}) + + # divergence 2: lax coercion re-opens holes the library validators close + coerced = adapter.validate_python(model_shaped | {"shape": [True, -5]}) + assert coerced.shape == (1, -5) # from_json would reject both entries + + # __post_init__ invariants DO still run under pydantic construction + with pytest.raises(ValidationError, match="Extra fields"): + adapter.validate_python( + model_shaped | {"extra_fields": {"shape": {"must_understand": False}}} + ) From 03ee0b15fa8b821f2f326e003add95c009b67483 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:31:15 +0200 Subject: [PATCH 17/48] test(zarr-metadata): engine-backed pydantic BaseModel example (pydantic-zarr pattern) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For consumers that want a first-class BaseModel — JSON schema generation and generics for typed attributes, as in pydantic-zarr's ArraySpec — the example adds a third pattern: pydantic-native fields as the user-facing surface, with the library as the engine. A mode='before' validator canonicalizes every input via from_json(...).to_json(), so structural validation and normalization run before pydantic parses fields (the [True, -5] coercion divergence cannot occur), and to_metadata_model / to_document bridge both ways through the document form. One translation noted at the bridge: the document spells 'no dimension names' as key absence, the pydantic side as None. Assisted-by: ClaudeCode:claude-fable-5 --- .../tests/model/test_pydantic.py | 123 +++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index cfd077651a..fa4fb5f590 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -24,19 +24,23 @@ messages intact. """ -from typing import Annotated +from collections.abc import Mapping +from typing import Annotated, Generic, TypeVar import pytest from pydantic import ( BaseModel, BeforeValidator, + ConfigDict, InstanceOf, PlainSerializer, TypeAdapter, ValidationError, + model_validator, ) -from zarr_metadata.model import ArrayMetadataModelV3 +from zarr_metadata import JSONValue +from zarr_metadata.model import ArrayMetadataModelV3, NamedConfigModelV3 # --- the integration (this is the example) ----------------------------------- @@ -145,7 +149,6 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: cross-field checks. This test documents why the delegation pattern above is the recommended integration.""" from zarr_metadata._common import JSONValue - from zarr_metadata.model import NamedConfigModelV3 from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 @@ -190,3 +193,117 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: adapter.validate_python( model_shaped | {"extra_fields": {"shape": {"must_understand": False}}} ) + + +# --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- +# +# When a consumer wants a real BaseModel — JSON schema generation, and +# generics for typed attributes, as in pydantic-zarr's ArraySpec — the model +# fields are pydantic-native, but validation and serialization still route +# through the library: a mode="before" validator canonicalizes every input +# document with from_json(...).to_json(), so the structural validators and +# normalization run BEFORE pydantic parses fields (no coercion divergence), +# and the document form is the bridge in both directions. + +AttrsT = TypeVar("AttrsT") + + +class NamedConfig(BaseModel): + """Pydantic mirror of a normalized metadata field (name + configuration).""" + + name: str + configuration: dict[str, JSONValue] = {} + + +class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): + """A pydantic-native, attribute-typed view of a v3 array metadata document. + + The library is the engine: every input is canonicalized and structurally + validated by `ArrayMetadataModelV3.from_json` before pydantic sees the + fields, and `to_document` / `to_metadata_model` emit through the library. + """ + + model_config = ConfigDict(frozen=True) + + zarr_format: int = 3 + node_type: str = "array" + shape: tuple[int, ...] + data_type: NamedConfig + chunk_grid: NamedConfig + chunk_key_encoding: NamedConfig + fill_value: JSONValue + codecs: tuple[NamedConfig, ...] + attributes: AttrsT + dimension_names: tuple[str | None, ...] | None = None + storage_transformers: tuple[NamedConfig, ...] = () + + @model_validator(mode="before") + @classmethod + def _canonicalize(cls, data: object) -> object: + """Route every input document through the library's validation and + normalization; pydantic then parses only canonical documents.""" + if isinstance(data, Mapping): + doc = dict(ArrayMetadataModelV3.from_json(data).to_json()) + doc.setdefault("attributes", {}) + return doc + return data + + def to_metadata_model(self) -> ArrayMetadataModelV3: + """Bridge back to the canonical model, via the document form. + + In the document, "no dimension names" is key-absence, not null; the + pydantic-side None translates to dropping the key. + """ + doc = self.model_dump() + if doc["dimension_names"] is None: + del doc["dimension_names"] + return ArrayMetadataModelV3.from_json(doc) + + def to_document(self) -> dict[str, object]: + """The canonical document (omit-empty conventions applied).""" + return dict(self.to_metadata_model().to_json()) + + +class MicroscopyAttrs(BaseModel): + """Example of consumer-typed attributes, pydantic-zarr style.""" + + resolution_um: float + + +def test_spec_typed_attributes() -> None: + """The generic parameter types the attributes, so consumers get validated, + attribute-level access — the pydantic-zarr ArraySpec pattern.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + assert spec.attributes.resolution_um == 0.5 + assert spec.data_type == NamedConfig(name="uint8") + + +def test_spec_engine_validates_before_pydantic() -> None: + """The library's structural validation runs before pydantic's parsing, so + coercion cannot re-open validator holes (contrast with the native + introspection test above, where [True, -5] coerced to (1, -5)).""" + with pytest.raises(ValidationError, match="shape"): + ArrayMetadataV3Spec[MicroscopyAttrs].model_validate( + dict(VALID_DOC) | {"shape": [True, -5], "attributes": {"resolution_um": 0.5}} + ) + + +def test_spec_bridges_to_canonical_model_and_document() -> None: + """to_metadata_model / to_document round-trip through the document form, + and the emitted document matches what the library itself would emit.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + model = spec.to_metadata_model() + assert isinstance(model, ArrayMetadataModelV3) + assert spec.to_document() == dict(model.to_json()) + # and back: the document revalidates to an equal spec + assert ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(spec.to_document()) == spec + + +def test_spec_json_schema_generation() -> None: + """A real BaseModel means model_json_schema works — the capability the + opaque InstanceOf pattern cannot provide.""" + schema = ArrayMetadataV3Spec[MicroscopyAttrs].model_json_schema() + assert schema["properties"]["shape"]["type"] == "array" + assert "MicroscopyAttrs" in schema["$defs"] From 9193bf47bbf5b596550db6e3a75e754855aef378 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:14:40 +0200 Subject: [PATCH 18/48] test(zarr-metadata): pin that a null dimension_names field is invalid Spec: 'If specified, must be an array of strings or null objects... If dimension_names is not specified, all dimensions are unnamed.' The null object is a permitted element (an unnamed dimension), never the field value; key absence is the only spelling of 'not specified'. Pins the validator's existing rejection so it is not later 'fixed' to accept null-as-absence, and documents that in-memory None maps to key absence on serialization. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/tests/model/test_array.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index e73739929a..3a13aa6bea 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1244,3 +1244,17 @@ def test_must_understand_fields_empty_when_all_waived() -> None: extra_fields={"ext_a": {"name": "a", "must_understand": False}} ) assert model.must_understand_fields == {} + + +def test_dimension_names_null_field_rejected() -> None: + """A dimension_names field whose VALUE is null is invalid: the spec permits + null as an element (an unnamed dimension), never as the field value — "not + specified" is spelled by omitting the key. Consumers bridging from an + in-memory None sentinel must drop the key, not write null.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"dimension_names": None} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] + # and the model's own None spelling correctly maps to key absence + assert ( + "dimension_names" not in ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + ) From eaf2587280f0e2aed664d68bed96e5dfcdfeb601 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:34:41 +0200 Subject: [PATCH 19/48] feat(zarr-metadata): optional pydantic integration as zarr_metadata.pydantic Gamed out three shapes with prototypes before choosing: - dunders on the core classes (works, verified pydantic 2.0-2.13, but puts a framework protocol in the dependency-free layer); - pydantic-aware SUBCLASSES in a namespace (rejected on empirical failures: identity split breaks equality, core instances are rejected by subclass-typed fields, and nested construction produces core-class children unless every cross-reference is overridden); - Annotated field types over the CORE classes in an opt-in module (chosen): instances are the core classes so interop is free, pydantic imports eagerly at the module (loud failure when absent), core stays framework-free, and pydantic-protocol risk is quarantined to one clearly-labeled module. The module exports one field type per model. Validation delegates to from_json (structural validation and normalization cannot be bypassed by pydantic coercion), instances pass through, serialization emits the canonical document, and WithJsonSchema describes the accepted document form so model_json_schema works. Tests cover all seven field types, core-instance interop, error quality, JSON schema, roundtrip, and that importing zarr_metadata does not import pydantic. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 7 + .../src/zarr_metadata/pydantic.py | 131 ++++++++++++++++++ .../tests/model/test_pydantic_module.py | 125 +++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 packages/zarr-metadata/src/zarr_metadata/pydantic.py create mode 100644 packages/zarr-metadata/tests/model/test_pydantic_module.py diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index f35625558b..44b7c1d892 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -29,3 +29,10 @@ not explicitly waived with `must_understand: false` (fields are implicitly must-understand per the spec). Readers discharge the spec's fail-to-open duty by subtracting the extension names they recognize; the model only partitions by obligation, since recognition is reader-specific. + +Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it +requires pydantic v2; the core package does not depend on it): one `Annotated` +field type per model, validating raw documents through `from_json`, passing +core-model instances through unchanged, and serializing via `to_json`. The +instances are the core model classes, so values interoperate freely with +non-pydantic code. diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py new file mode 100644 index 0000000000..5ec837ee23 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -0,0 +1,131 @@ +"""Optional pydantic (v2) integration: field types over the core models. + +Importing this module requires pydantic; the core package deliberately does +not depend on it, so this module is never imported by `zarr_metadata` itself. + +Each exported name is an `Annotated` field type over the corresponding core +model class — the instances ARE the core classes, so values interoperate +freely with non-pydantic code (equality, isinstance, nesting). Validation +delegates to the library: a raw document routes through `from_json` (the +single source of truth for structural validation and normalization, so +pydantic's field-level coercion can never bypass it), an existing model +instance passes through unchanged, and serialization emits the canonical +document via `to_json`. `MetadataValidationError` subclasses `ValueError`, +so a failed parse surfaces as a pydantic `ValidationError` carrying the +loc-annotated problem messages. + +Usage: + + import zarr_metadata.pydantic as zmp + + class ArrayManifest(BaseModel): + path: str + metadata: zmp.ArrayMetadataV3 + +Static type checkers see each field type as its core model class, so +`manifest.metadata` is an `ArrayMetadataModelV3`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, TypeVar + +from pydantic import BeforeValidator, InstanceOf, PlainSerializer, WithJsonSchema + +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, + NamedConfigModelV3, +) + +if TYPE_CHECKING: + from collections.abc import Callable + +_M = TypeVar("_M") + + +def _coerce_to(cls: type[_M]) -> Callable[[object], _M]: + """A validator that passes instances through and parses anything else.""" + + def coerce(value: object) -> _M: + if isinstance(value, cls): + return value + return cls.from_json(value) # type: ignore[attr-defined, no-any-return] + + return coerce + + +# JSON schemas describe the DOCUMENT form each field accepts (the validation +# input), not the in-memory model shape. +_DOCUMENT_SCHEMA = {"type": "object"} +_FIELD_SCHEMA = {"anyOf": [{"type": "string"}, {"type": "object"}]} + +ArrayMetadataV3 = Annotated[ + InstanceOf[ArrayMetadataModelV3], + BeforeValidator(_coerce_to(ArrayMetadataModelV3)), + PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV3"}), +] +"""Field type for a v3 array metadata document (`zarr.json` content).""" + +ArrayMetadataV2 = Annotated[ + InstanceOf[ArrayMetadataModelV2], + BeforeValidator(_coerce_to(ArrayMetadataModelV2)), + PlainSerializer(ArrayMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV2"}), +] +"""Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" + +GroupMetadataV3 = Annotated[ + InstanceOf[GroupMetadataModelV3], + BeforeValidator(_coerce_to(GroupMetadataModelV3)), + PlainSerializer(GroupMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV3"}), +] +"""Field type for a v3 group metadata document (`zarr.json` content).""" + +GroupMetadataV2 = Annotated[ + InstanceOf[GroupMetadataModelV2], + BeforeValidator(_coerce_to(GroupMetadataModelV2)), + PlainSerializer(GroupMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV2"}), +] +"""Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" + +ConsolidatedMetadataV3 = Annotated[ + InstanceOf[ConsolidatedMetadataModelV3], + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3)), + PlainSerializer(ConsolidatedMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV3"}), +] +"""Field type for v3 inline consolidated metadata.""" + +ConsolidatedMetadataV2 = Annotated[ + InstanceOf[ConsolidatedMetadataModelV2], + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2)), + PlainSerializer(ConsolidatedMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV2"}), +] +"""Field type for a v2 `.zmetadata` document.""" + +MetadataFieldV3 = Annotated[ + InstanceOf[NamedConfigModelV3], + BeforeValidator(_coerce_to(NamedConfigModelV3)), + PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), + WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), +] +"""Field type for one v3 metadata field (bare name string or name + configuration).""" + +__all__ = [ + "ArrayMetadataV2", + "ArrayMetadataV3", + "ConsolidatedMetadataV2", + "ConsolidatedMetadataV3", + "GroupMetadataV2", + "GroupMetadataV3", + "MetadataFieldV3", +] diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py new file mode 100644 index 0000000000..6202020dd6 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -0,0 +1,125 @@ +"""Tests for `zarr_metadata.pydantic`, the optional pydantic field-type module. + +The hand-rolled recipes in `test_pydantic.py` document how the integration +works; this module ships it. Instances are the CORE model classes (no +parallel hierarchy), so values interoperate freely with non-pydantic code. +""" + +import pytest +from pydantic import BaseModel, TypeAdapter, ValidationError + +import zarr_metadata.pydantic as zmp +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, + NamedConfigModelV3, +) + +V3_ARRAY_DOC = dict(ArrayMetadataModelV3.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ArrayMetadataModelV2.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} +V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} +V3_CONSOLIDATED_DOC = { + "kind": "inline", + "must_understand": False, + "metadata": {"a": dict(V3_ARRAY_DOC)}, +} +V2_CONSOLIDATED_DOC = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}}, +} + +FIELD_CASES = [ + pytest.param(zmp.ArrayMetadataV3, ArrayMetadataModelV3, V3_ARRAY_DOC, id="array-v3"), + pytest.param(zmp.ArrayMetadataV2, ArrayMetadataModelV2, V2_ARRAY_DOC, id="array-v2"), + pytest.param(zmp.GroupMetadataV3, GroupMetadataModelV3, V3_GROUP_DOC, id="group-v3"), + pytest.param(zmp.GroupMetadataV2, GroupMetadataModelV2, V2_GROUP_DOC, id="group-v2"), + pytest.param( + zmp.ConsolidatedMetadataV3, + ConsolidatedMetadataModelV3, + V3_CONSOLIDATED_DOC, + id="consolidated-v3", + ), + pytest.param( + zmp.ConsolidatedMetadataV2, + ConsolidatedMetadataModelV2, + V2_CONSOLIDATED_DOC, + id="consolidated-v2", + ), + pytest.param(zmp.MetadataFieldV3, NamedConfigModelV3, {"name": "bytes"}, id="field-v3"), +] + + +@pytest.mark.parametrize(("field_type", "model_cls", "doc"), FIELD_CASES) +def test_field_type_validates_and_dumps_canonically( + field_type: object, model_cls: type, doc: dict[str, object] +) -> None: + """Each field type parses its raw document into the CORE model class, + passes existing instances through unchanged, and dumps the canonical + document via to_json.""" + adapter = TypeAdapter(field_type) + model = adapter.validate_python(doc) + assert type(model) is model_cls + assert adapter.validate_python(model) is model + assert adapter.dump_python(model) == dict(model.to_json()) + + +def test_core_instances_interoperate() -> None: + """A core model instance (e.g. handed out by zarr-python) drops straight + into a pydantic field — the reason the module ships Annotated aliases over + the core classes rather than pydantic-aware subclasses.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + core = ArrayMetadataModelV3.from_json(V3_ARRAY_DOC) + manifest = Manifest(metadata=core) + assert manifest.metadata is core + + +def test_validation_error_carries_problems() -> None: + """A defective document fails with the library's loc-annotated messages.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + doc = dict(V3_ARRAY_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError, match="chunk_key_encoding: missing required key"): + Manifest(metadata=doc) # type: ignore[arg-type] + + +def test_json_schema_generation() -> None: + """model_json_schema works, describing the document form each field accepts.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + codec: zmp.MetadataFieldV3 + + schema = Manifest.model_json_schema() + assert schema["properties"]["metadata"] == {"type": "object", "title": "ArrayMetadataV3"} + assert schema["properties"]["codec"]["anyOf"] == [{"type": "string"}, {"type": "object"}] + + +def test_json_roundtrip() -> None: + """model_dump_json output re-validates to an equal pydantic model.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + manifest = Manifest(metadata=V3_ARRAY_DOC) # type: ignore[arg-type] + assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest + + +def test_core_package_does_not_import_pydantic() -> None: + """Importing zarr_metadata (in a fresh interpreter) must not import + pydantic: the integration is opt-in via zarr_metadata.pydantic.""" + import subprocess + import sys + + code = "import sys, zarr_metadata; assert 'pydantic' not in sys.modules, 'leaked'" + subprocess.run([sys.executable, "-c", code], check=True) From bd80d7dfaf754628c95ac33206e9955adcc47df1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:46:23 +0200 Subject: [PATCH 20/48] fix(zarr-metadata): create_default derives the chunk grid from shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_default(shape=(100, 100)) silently kept the scalar default's 0-d chunk grid (chunk_shape: ()), producing a structurally-valid but semantically inconsistent document — a footgun for every test fixture built on it. When shape is overridden and the grid is not, the default is now one regular chunk covering the array (v3 chunk_shape == shape, v2 chunks == shape); an explicit chunk_grid/chunks override still wins. update() stays a dumb dataclasses.replace, per its documented contract. One existing whole-document test literal carried exactly this inconsistency (shape (10,) with chunk_shape ()) and was updated. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 +++ .../src/zarr_metadata/model/_array.py | 14 ++++++-- .../zarr-metadata/tests/model/test_array.py | 34 ++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 44b7c1d892..088d6e11a6 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -36,3 +36,8 @@ field type per model, validating raw documents through `from_json`, passing core-model instances through unchanged, and serializing via `to_json`. The instances are the core model classes, so values interoperate freely with non-pydantic code. + +`create_default` keeps its output self-consistent: overriding `shape` without +a chunk grid derives one regular chunk covering the array (v3 +`chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the +scalar default's 0-d grid. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 7a4487b7f4..af680e8d3f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -168,8 +168,14 @@ def create_default( The default is a structurally-valid scalar `uint8` array — the array analog of `list()` returning `[]`. Any field can be overridden by keyword - (the same fields accepted by `update`). + (the same fields accepted by `update`). Overriding `shape` without + `chunk_grid` derives a consistent default grid: one regular chunk + covering the array (`chunk_shape` equal to `shape`). """ + if "shape" in overrides and "chunk_grid" not in overrides: + overrides["chunk_grid"] = NamedConfigModelV3( + name="regular", configuration={"chunk_shape": tuple(overrides["shape"])} + ) default = cls( shape=(), fill_value=0, @@ -353,8 +359,12 @@ def create_default( The default is a structurally-valid scalar `uint8` (`"|u1"`) array — the array analog of `list()` returning `[]`. Any field can be overridden by - keyword (the same fields accepted by `update`). + keyword (the same fields accepted by `update`). Overriding `shape` + without `chunks` derives `chunks` equal to `shape` (one chunk covering + the array). """ + if "shape" in overrides and "chunks" not in overrides: + overrides["chunks"] = tuple(overrides["shape"]) default = cls( shape=(), dtype="|u1", diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 3a13aa6bea..535541d180 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -180,7 +180,7 @@ def test_v3_to_json_emits_canonical_document() -> None: "shape": (10,), "fill_value": 0, "data_type": {"name": "int32", "configuration": {}}, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": ()}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, "codecs": ({"name": "bytes", "configuration": {}},), "chunk_key_encoding": {"name": "default", "configuration": {}}, } @@ -1258,3 +1258,35 @@ def test_dimension_names_null_field_rejected() -> None: assert ( "dimension_names" not in ArrayMetadataModelV3.create_default(dimension_names=None).to_json() ) + + +# --- create_default derives the chunk grid from shape ------------------------ + + +def test_v3_create_default_chunk_grid_follows_shape() -> None: + """Overriding shape without chunk_grid derives a consistent default grid: + one chunk covering the array (chunk_shape == shape), instead of silently + keeping the scalar default's 0-d grid.""" + model = ArrayMetadataModelV3.create_default(shape=(100, 100)) + assert model.chunk_grid == NamedConfigModelV3( + name="regular", configuration={"chunk_shape": (100, 100)} + ) + + +def test_v3_create_default_explicit_chunk_grid_respected() -> None: + """An explicit chunk_grid override wins over the shape-derived default.""" + grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) + model = ArrayMetadataModelV3.create_default(shape=(100, 100), chunk_grid=grid) + assert model.chunk_grid == grid + + +def test_v2_create_default_chunks_follow_shape() -> None: + """Overriding shape without chunks derives chunks == shape.""" + model = ArrayMetadataModelV2.create_default(shape=(100, 100)) + assert model.chunks == (100, 100) + + +def test_v2_create_default_explicit_chunks_respected() -> None: + """An explicit chunks override wins over the shape-derived default.""" + model = ArrayMetadataModelV2.create_default(shape=(100, 100), chunks=(10, 10)) + assert model.chunks == (10, 10) From bdec803ac47707e69eb2121b26ca07238c8a1d3f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:49:31 +0200 Subject: [PATCH 21/48] test(zarr-metadata): pin zero-length-dimension case of the derived chunk grid The spec's constraint is conditional ('non-zero when the corresponding dimensions of the arrays have non-zero length'), so chunk_shape == shape is sound for every shape, including empty dimensions. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/tests/model/test_array.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 535541d180..a5ffbb7080 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1290,3 +1290,12 @@ def test_v2_create_default_explicit_chunks_respected() -> None: """An explicit chunks override wins over the shape-derived default.""" model = ArrayMetadataModelV2.create_default(shape=(100, 100), chunks=(10, 10)) assert model.chunks == (10, 10) + + +def test_v3_create_default_zero_length_dimensions() -> None: + """chunk_shape == shape is spec-sound even with zero-length dimensions: + 'The chunk shape elements are non-zero when the corresponding dimensions + of the arrays have non-zero length' — the constraint is conditional, so a + zero chunk length is permitted exactly where the dimension is empty.""" + model = ArrayMetadataModelV3.create_default(shape=(0, 3)) + assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) From 965ea029cd0d5e5421c5b34a6e01b0d65f1db854 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:53:20 +0200 Subject: [PATCH 22/48] docs(zarr-metadata): document that create_default's derivation is one-way MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overriding shape without a grid derives the grid; the reverse does not hold. A user-supplied chunk_grid is an extension point taken verbatim — deriving shape from it would require interpreting grid configurations, which the model layer never does and cannot do for unrecognized grid names. Pinned by test so the asymmetry reads as a decision, not an oversight; the v2 model documents the same one-way rule for chunks for cross-version consistency. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_array.py | 13 +++++++++++++ packages/zarr-metadata/tests/model/test_array.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index af680e8d3f..cfd43571ec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -171,6 +171,14 @@ def create_default( (the same fields accepted by `update`). Overriding `shape` without `chunk_grid` derives a consistent default grid: one regular chunk covering the array (`chunk_shape` equal to `shape`). + + The derivation is deliberately one-way. A user-supplied `chunk_grid` + is an extension point and is taken verbatim — deriving `shape` from + it would require interpreting the grid's configuration, which this + layer never does (and cannot do for unrecognized grid names). So + overriding `chunk_grid` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. """ if "shape" in overrides and "chunk_grid" not in overrides: overrides["chunk_grid"] = NamedConfigModelV3( @@ -362,6 +370,11 @@ def create_default( keyword (the same fields accepted by `update`). Overriding `shape` without `chunks` derives `chunks` equal to `shape` (one chunk covering the array). + + The derivation is deliberately one-way, matching the v3 model: + overriding `chunks` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. """ if "shape" in overrides and "chunks" not in overrides: overrides["chunks"] = tuple(overrides["shape"]) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index a5ffbb7080..3f7f35c50c 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1299,3 +1299,17 @@ def test_v3_create_default_zero_length_dimensions() -> None: zero chunk length is permitted exactly where the dimension is empty.""" model = ArrayMetadataModelV3.create_default(shape=(0, 3)) assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) + + +def test_create_default_derivation_is_one_way() -> None: + """Overriding the chunk grid (v3) or chunks (v2) without shape leaves the + scalar default shape=() untouched: a user-supplied chunk_grid is an + extension point taken verbatim, and deriving shape from it would require + interpreting grid configurations, which the model layer never does.""" + grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) + v3 = ArrayMetadataModelV3.create_default(chunk_grid=grid) + assert v3.shape == () + assert v3.chunk_grid == grid + v2 = ArrayMetadataModelV2.create_default(chunks=(10, 10)) + assert v2.shape == () + assert v2.chunks == (10, 10) From d4a0f8eecf7a88a25eb6267cca38f8b938c837dd Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 19:02:29 +0200 Subject: [PATCH 23/48] refactor(zarr-metadata): eliminate every type-ignore comment Audited all 24 (15 src, 9 tests); each was either obsolete, replaceable by a sound cast, or avoidable by better-typed code: - Two fill_value arg-type ignores were factually obsolete: their justifying comment said 'fill_value: object in upstream TypedDict', but 0.3.0 narrowed it to JSONValue. - Eight pre-existing call-arg/reportInvalidTypeForm ignores on the PEP 728 TypedDicts and the recursive JSONValue alias were mypy-dialect suppressions that the checker of record (pyright strict with enableExperimentalFeatures) never needed; mypy has never checked this package. - The two extra_fields comprehensions are a genuine checker limitation (a key filter cannot narrow a PEP 728 TypedDict's item-value union), now expressed as casts whose comments state the soundness claim instead of suppressing the diagnostic. - pydantic.py's generic coercer factory takes the parse callable explicitly instead of calling from_json through type[_M]. - NamedConfigModelV3.from_json casts the validated configuration (sound since configuration values are now deep-validated as JSON). - Tests: _build_v2/_build_v3 gained real Unpack[...Partial] signatures; raw-document pydantic inputs go through model_validate (the idiomatic entry point for untyped data) instead of ignoring constructor signatures; the frozen-dataclass test uses setattr for its intentional runtime error. src and tests/model now carry zero type-ignore comments. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/_common.py | 2 +- .../src/zarr_metadata/model/_array.py | 25 ++++++++++++------- .../src/zarr_metadata/model/_group.py | 16 ++++++++---- .../src/zarr_metadata/pydantic.py | 20 +++++++-------- .../src/zarr_metadata/v2/codec.py | 2 +- .../src/zarr_metadata/v3/array.py | 6 ++--- .../src/zarr_metadata/v3/codec/crc32c.py | 2 +- .../src/zarr_metadata/v3/group.py | 4 +-- .../zarr-metadata/tests/model/test_array.py | 13 ++++++---- .../tests/model/test_pydantic.py | 8 +++--- .../tests/model/test_pydantic_module.py | 4 +-- 11 files changed, 59 insertions(+), 43 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 598a12e80c..b335cd0bd6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -13,7 +13,7 @@ JSONValue = TypeAliasType( "JSONValue", - "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", # type: ignore[reportInvalidTypeForm] + "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", ) """A recursive type alias for JSON-encodable values. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index cfd43571ec..991dd77d3f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -63,8 +63,13 @@ def from_json(cls, data: object) -> NamedConfigModelV3: field = parse_metadata_field_v3(data) if isinstance(field, str): return cls(name=field, configuration={}) - configuration = arrays_to_tuples(dict(field.get("configuration", {}))) - return cls(name=field["name"], configuration=configuration) # type: ignore[arg-type] + # Sound cast: parse_metadata_field_v3 checked the configuration is a + # string-keyed mapping of JSON values; arrays_to_tuples only converts + # lists to tuples within that shape. + configuration = cast( + "dict[str, JSONValue]", arrays_to_tuples(dict(field.get("configuration", {}))) + ) + return cls(name=field["name"], configuration=configuration) MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 @@ -260,14 +265,16 @@ def to_json(self) -> ArrayMetadataV3: @classmethod def from_json(cls, data: object) -> ArrayMetadataModelV3: parsed = parse_array_metadata_v3(arrays_to_tuples(data)) - extra_fields: dict[str, ExtensionFieldV3] = { - k: v # type: ignore[misc] - for k, v in parsed.items() - if k not in ARRAY_METADATA_STANDARD_KEYS_V3 - } + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ExtensionFieldV3]", + {k: v for k, v in parsed.items() if k not in ARRAY_METADATA_STANDARD_KEYS_V3}, + ) return cls( shape=parsed["shape"], - fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + fill_value=parsed["fill_value"], data_type=NamedConfigModelV3.from_json(parsed["data_type"]), chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), @@ -418,7 +425,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: shape=parsed["shape"], dtype=parsed["dtype"], chunks=parsed["chunks"], - fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + fill_value=parsed["fill_value"], order=parsed["order"], compressor=parsed["compressor"], filters=parsed["filters"], diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index b8cca3849d..0980096a8b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -149,11 +149,17 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: if consolidated_raw is None else ConsolidatedMetadataModelV3.from_json(consolidated_raw) ) - extra_fields: dict[str, ExtensionFieldV3] = { - k: v # type: ignore[misc] - for k, v in parsed.items() - if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 - } + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ExtensionFieldV3]", + { + k: v + for k, v in parsed.items() + if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 + }, + ) return cls( attributes=dict(parsed.get("attributes", {})), consolidated_metadata=consolidated, diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 5ec837ee23..3eada76316 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -48,13 +48,13 @@ class ArrayManifest(BaseModel): _M = TypeVar("_M") -def _coerce_to(cls: type[_M]) -> Callable[[object], _M]: - """A validator that passes instances through and parses anything else.""" +def _coerce_to(cls: type[_M], parse: Callable[[object], _M]) -> Callable[[object], _M]: + """A validator that passes instances of `cls` through and parses anything else.""" def coerce(value: object) -> _M: if isinstance(value, cls): return value - return cls.from_json(value) # type: ignore[attr-defined, no-any-return] + return parse(value) return coerce @@ -66,7 +66,7 @@ def coerce(value: object) -> _M: ArrayMetadataV3 = Annotated[ InstanceOf[ArrayMetadataModelV3], - BeforeValidator(_coerce_to(ArrayMetadataModelV3)), + BeforeValidator(_coerce_to(ArrayMetadataModelV3, ArrayMetadataModelV3.from_json)), PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV3"}), ] @@ -74,7 +74,7 @@ def coerce(value: object) -> _M: ArrayMetadataV2 = Annotated[ InstanceOf[ArrayMetadataModelV2], - BeforeValidator(_coerce_to(ArrayMetadataModelV2)), + BeforeValidator(_coerce_to(ArrayMetadataModelV2, ArrayMetadataModelV2.from_json)), PlainSerializer(ArrayMetadataModelV2.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV2"}), ] @@ -82,7 +82,7 @@ def coerce(value: object) -> _M: GroupMetadataV3 = Annotated[ InstanceOf[GroupMetadataModelV3], - BeforeValidator(_coerce_to(GroupMetadataModelV3)), + BeforeValidator(_coerce_to(GroupMetadataModelV3, GroupMetadataModelV3.from_json)), PlainSerializer(GroupMetadataModelV3.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV3"}), ] @@ -90,7 +90,7 @@ def coerce(value: object) -> _M: GroupMetadataV2 = Annotated[ InstanceOf[GroupMetadataModelV2], - BeforeValidator(_coerce_to(GroupMetadataModelV2)), + BeforeValidator(_coerce_to(GroupMetadataModelV2, GroupMetadataModelV2.from_json)), PlainSerializer(GroupMetadataModelV2.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV2"}), ] @@ -98,7 +98,7 @@ def coerce(value: object) -> _M: ConsolidatedMetadataV3 = Annotated[ InstanceOf[ConsolidatedMetadataModelV3], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3)), + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3, ConsolidatedMetadataModelV3.from_json)), PlainSerializer(ConsolidatedMetadataModelV3.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV3"}), ] @@ -106,7 +106,7 @@ def coerce(value: object) -> _M: ConsolidatedMetadataV2 = Annotated[ InstanceOf[ConsolidatedMetadataModelV2], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2)), + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2, ConsolidatedMetadataModelV2.from_json)), PlainSerializer(ConsolidatedMetadataModelV2.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV2"}), ] @@ -114,7 +114,7 @@ def coerce(value: object) -> _M: MetadataFieldV3 = Annotated[ InstanceOf[NamedConfigModelV3], - BeforeValidator(_coerce_to(NamedConfigModelV3)), + BeforeValidator(_coerce_to(NamedConfigModelV3, NamedConfigModelV3.from_json)), PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py index 6d194b7e29..8f7f2b3f88 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py @@ -10,7 +10,7 @@ from zarr_metadata._common import JSONValue -class CodecMetadataV2(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class CodecMetadataV2(TypedDict, extra_items=JSONValue): """ A numcodecs configuration dict, used as a v2 compressor or filter. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index a8b0fa3358..a3fec24a6c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -9,7 +9,7 @@ from zarr_metadata.v3._common import MetadataV3 -class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class ExtensionFieldV3(TypedDict, extra_items=JSONValue): """ Required shape of any extension field on a v3 metadata document. @@ -41,7 +41,7 @@ class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-a must_understand: bool -class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 array metadata document (the `zarr.json` content for an array). @@ -63,7 +63,7 @@ class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ dimension_names: NotRequired[tuple[str | None, ...]] -class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): """ Partial form of `ArrayMetadataV3`: every field is `NotRequired`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index ea35ae5f1d..6b9b46c43d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -18,7 +18,7 @@ """Literal type of the `name` field of the `crc32c` codec.""" -class Empty(TypedDict, closed=True): # type: ignore[call-arg] +class Empty(TypedDict, closed=True): """An empty mapping""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index 27186b6059..be990b1ae7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -12,7 +12,7 @@ from zarr_metadata.v3.array import ExtensionFieldV3 -class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 group metadata document (the `zarr.json` content for a group). @@ -26,7 +26,7 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): """ Partial form of `GroupMetadataV3`: every field is `NotRequired`. diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 3f7f35c50c..f1ce967b99 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING import pytest +from typing_extensions import Unpack from tests.model._cases import Expect, ExpectFail from zarr_metadata.model import ( @@ -814,12 +815,12 @@ def test_parse_metadata_field_v3( # invalid cases (a subset check, so accumulation of OTHER problems is allowed). -def _build_v3(**overrides: object) -> dict[str, object]: - return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) # type: ignore[arg-type] +def _build_v3(**overrides: Unpack[ArrayMetadataModelV3Partial]) -> dict[str, object]: + return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) -def _build_v2(**overrides: object) -> dict[str, object]: - return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) # type: ignore[arg-type] +def _build_v2(**overrides: Unpack[ArrayMetadataModelV2Partial]) -> dict[str, object]: + return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: @@ -1008,7 +1009,9 @@ def test_validation_problem_is_frozen() -> None: """ValidationProblem is immutable (frozen dataclass).""" p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") with pytest.raises(dataclasses.FrozenInstanceError): - p.message = "y" # type: ignore[misc] + # setattr: assigning to a frozen field is an intentional runtime error, + # spelled dynamically so it is not also a static type error. + setattr(p, "message", "y") # noqa: B010 def test_metadata_validation_error_holds_problems() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index fa4fb5f590..c1cd3010d4 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -90,7 +90,7 @@ class ArrayManifest(BaseModel): def test_raw_document_is_validated_into_a_model() -> None: """A raw metadata document on a pydantic field is parsed by from_json, with the library's normalization applied (tuples, canonical field form).""" - manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) assert isinstance(manifest.metadata, ArrayMetadataModelV3) assert manifest.metadata.shape == (10,) assert manifest.metadata.data_type.name == "uint8" @@ -109,14 +109,14 @@ def test_invalid_document_surfaces_problems_in_validation_error() -> None: doc = dict(VALID_DOC) del doc["chunk_key_encoding"] with pytest.raises(ValidationError) as exc_info: - ArrayManifest(path="a/b", metadata=doc) # type: ignore[arg-type] + ArrayManifest.model_validate({"path": "a/b", "metadata": doc}) assert "chunk_key_encoding: missing required key" in str(exc_info.value) def test_dump_emits_canonical_document() -> None: """model_dump serializes the field via to_json — the canonical document, not pydantic's field-by-field view of the dataclass.""" - manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) dumped = manifest.model_dump() assert dumped["metadata"] == manifest.metadata.to_json() # the bare-string data_type was normalized to the canonical object form @@ -126,7 +126,7 @@ def test_dump_emits_canonical_document() -> None: def test_json_roundtrip_through_pydantic() -> None: """model_dump_json output re-validates to an equal manifest (JSON emits tuples as arrays; from_json converts them back).""" - manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) assert revived == manifest diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 6202020dd6..81b9b962af 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -90,7 +90,7 @@ class Manifest(BaseModel): doc = dict(V3_ARRAY_DOC) del doc["chunk_key_encoding"] with pytest.raises(ValidationError, match="chunk_key_encoding: missing required key"): - Manifest(metadata=doc) # type: ignore[arg-type] + Manifest.model_validate({"metadata": doc}) def test_json_schema_generation() -> None: @@ -111,7 +111,7 @@ def test_json_roundtrip() -> None: class Manifest(BaseModel): metadata: zmp.ArrayMetadataV3 - manifest = Manifest(metadata=V3_ARRAY_DOC) # type: ignore[arg-type] + manifest = Manifest.model_validate({"metadata": V3_ARRAY_DOC}) assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest From 0a7f674ac29f5cd51dee19ae73d37a44f1b7bb4c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 19:14:23 +0200 Subject: [PATCH 24/48] fix(zarr-metadata): absent v2 dimension_separator means '.', not '/' roborev job 426 (branch review) found that ArrayMetadataModelV2 normalized an ABSENT dimension_separator key to '/', inherited verbatim from the zng prototype. The v2 convention's default is '.': a consumer deriving chunk keys from the model against a real-world v2 array written with the default separator would have looked for '0/0' instead of '0.0'. No test caught it because every fixture started from create_default(), which always carries an explicit separator. Absence is normalized to an explicit '.' -- a semantics-preserving spelling normalization consistent with the model's existing canonical forms (bare-string metadata fields, missing configuration). The field is deliberately NOT modeled as Optional: the document grammar has no null spelling for this key, and a None in the model invites writing 'dimension_separator': null into documents. Pinned by three tests, including explicit-null rejection. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 +++ .../src/zarr_metadata/model/_array.py | 12 ++++-- .../zarr-metadata/tests/model/test_array.py | 37 +++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 088d6e11a6..3274fd2e3b 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -41,3 +41,8 @@ non-pydantic code. a chunk grid derives one regular chunk covering the array (v3 `chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the scalar default's 0-d grid. + +A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 +convention's default `"."` (the model previously normalized absence to `"/"`, +which would misaddress the chunks of real-world default-separator arrays). +The value is never null: absent, `"."`, or `"/"` are the only spellings. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 991dd77d3f..6d1d2c3d6b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -340,7 +340,9 @@ class ArrayMetadataModelV2: A canonical, lossless representation of the `.zarray` content plus the sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are held in their raw JSON forms and are never interpreted; `fill_value` is - held verbatim in its JSON form. + held verbatim in its JSON form. One spelling normalization: a `.zarray` + that omits `dimension_separator` means `"."` by the v2 convention, and + the model holds and re-emits that value explicitly. """ zarr_format: Literal[2] = field(default=2, init=False) @@ -351,7 +353,11 @@ class ArrayMetadataModelV2: order: ArrayOrderV2 compressor: CodecMetadataV2 | None filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: ArrayDimensionSeparatorV2 = field(default="/") + # "." is the v2 convention's default for an ABSENT dimension_separator key; + # from_json normalizes absence to it (a semantics-preserving spelling + # normalization, like the v3 bare-string metadata-field form). The value + # is never None: the document grammar has no null spelling for this field. + dimension_separator: ArrayDimensionSeparatorV2 = field(default=".") attributes: dict[str, JSONValue] def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: @@ -429,7 +435,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: order=parsed["order"], compressor=parsed["compressor"], filters=parsed["filters"], - dimension_separator=parsed.get("dimension_separator", "/"), + dimension_separator=parsed.get("dimension_separator", "."), attributes=dict(parsed.get("attributes", {})), ) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index f1ce967b99..4805fc2674 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1316,3 +1316,40 @@ def test_create_default_derivation_is_one_way() -> None: v2 = ArrayMetadataModelV2.create_default(chunks=(10, 10)) assert v2.shape == () assert v2.chunks == (10, 10) + + +# --- v2 dimension_separator default (roborev job 426) ------------------------- + + +def test_v2_absent_dimension_separator_means_dot() -> None: + """A .zarray that omits dimension_separator uses the v2 convention default + '.', not '/': chunk keys of real-world default-separator v2 arrays look + like '0.0'. The model normalizes the absent key to an explicit '.' — a + semantics-preserving spelling normalization.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) + del doc["dimension_separator"] + model = ArrayMetadataModelV2.from_json(doc) + assert model.dimension_separator == "." + assert model.to_json()["dimension_separator"] == "." + + +def test_v2_from_key_value_without_separator_means_dot() -> None: + """The .zarray store-file path applies the same '.' default for an absent + dimension_separator key.""" + doc = { + k: v + for k, v in ArrayMetadataModelV2.create_default().to_json().items() + if k not in ("dimension_separator", "attributes") + } + import json as _json + + model = ArrayMetadataModelV2.from_key_value({".zarray": _json.dumps(doc).encode()}) + assert model.dimension_separator == "." + + +def test_v2_null_dimension_separator_rejected() -> None: + """dimension_separator may be absent, '.', or '/' — never null: the + document grammar has no null spelling for this field.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": None} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] From 69ae90192af1bafc07a7905f5f8b7b384a86d883 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 20:58:07 +0200 Subject: [PATCH 25/48] feat(zarr-metadata): UNSET sentinel for absent optional document keys Establishes the models' None/absence invariant: None in a model always corresponds to a JSON null in the document (a v2 compressor/filters value, an unnamed dimension inside dimension_names), and UNSET always means the key is absent. The two are never interchangeable. Applied to the two fields that used None as an absence marker: dimension_names (ArrayMetadataModelV3) and consolidated_metadata (GroupMetadataModelV3). For dimension_names this also preserves a semantic distinction d-v-b identified: an absent field ("there are no dimension names") and an explicit all-null array ("every dimension has a name, which is null") are different documents; both spellings now round-trip faithfully and compare unequal. Normalizing absence to the all-null form was considered and rejected: the spellings' interpretations coincide but interpretation-equivalence is the resolve layer's business, and collapsing document-level distinctions on that basis is the layer violation this package exists to avoid. Verified that current zarr-python never writes "consolidated_metadata": null (GroupMetadata.to_dict pops the key), so None there was purely an absence marker, not a document spelling. UnsetType is a single-member enum (identity-checkable, repr "UNSET", deliberately truthy so `if not x` cannot silently treat it as absent); UNSET and UnsetType are exported from zarr_metadata.model and the package front door. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 8 +++++ .../src/zarr_metadata/__init__.py | 4 +++ .../src/zarr_metadata/model/__init__.py | 3 ++ .../src/zarr_metadata/model/_array.py | 11 +++--- .../src/zarr_metadata/model/_group.py | 15 ++++---- .../src/zarr_metadata/model/_sentinel.py | 30 ++++++++++++++++ .../zarr-metadata/tests/model/test_array.py | 36 +++++++++++++++---- .../zarr-metadata/tests/model/test_group.py | 5 +-- .../tests/model/test_pydantic.py | 4 ++- .../zarr-metadata/tests/test_public_api.py | 2 ++ 10 files changed, 96 insertions(+), 22 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 3274fd2e3b..29c893b3c1 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -46,3 +46,11 @@ A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 convention's default `"."` (the model previously normalized absence to `"/"`, which would misaddress the chunks of real-world default-separator arrays). The value is never null: absent, `"."`, or `"/"` are the only spellings. + +Optional document keys use the `UNSET` sentinel, never `None`: in a model, +`None` always corresponds to a JSON `null` in the document (a v2 +`compressor`, an unnamed dimension inside `dimension_names`), and `UNSET` +always means the key is absent. This keeps semantically distinct spellings +distinct — an absent `dimension_names` ("there are no dimension names") and +an explicit `[null, null]` ("every dimension has a name, which is null") are +different documents and round-trip as such. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 354f2a97f5..4e0cfd4e36 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -2,6 +2,7 @@ from zarr_metadata._common import JSONValue, NamedConfigV3 from zarr_metadata.model import ( + UNSET, ArrayMetadataModelV2, ArrayMetadataModelV2Partial, ArrayMetadataModelV3, @@ -16,6 +17,7 @@ MetadataValidationError, NamedConfigModelV3, ProblemKind, + UnsetType, ValidationProblem, ) from zarr_metadata.v2.array import ( @@ -248,6 +250,7 @@ "UINT16_DATA_TYPE_NAME", "UINT32_DATA_TYPE_NAME", "UINT64_DATA_TYPE_NAME", + "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", @@ -353,6 +356,7 @@ "Uint32FillValue", "Uint64DataTypeName", "Uint64FillValue", + "UnsetType", "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 42fbc233d8..76367eae32 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -34,6 +34,7 @@ GroupMetadataModelV3, GroupMetadataModelV3Partial, ) +from zarr_metadata.model._sentinel import UNSET, UnsetType from zarr_metadata.model._validation import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V2, @@ -82,6 +83,7 @@ "GROUP_METADATA_STANDARD_KEYS_V3", "GROUP_METADATA_STORE_KEY_V2", "GROUP_METADATA_STORE_KEY_V3", + "UNSET", "ArrayMetadataModelV2", "ArrayMetadataModelV2Partial", "ArrayMetadataModelV3", @@ -96,6 +98,7 @@ "MetadataValidationError", "NamedConfigModelV3", "ProblemKind", + "UnsetType", "ValidationProblem", "is_array_metadata_v2", "is_array_metadata_v3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 6d1d2c3d6b..604c9a7d1e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -10,6 +10,7 @@ from typing_extensions import TypedDict, Unpack +from zarr_metadata.model._sentinel import UNSET, UnsetType from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -133,7 +134,7 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | None + dimension_names: tuple[str | None, ...] | UnsetType attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -159,7 +160,7 @@ class ArrayMetadataModelV3: chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | None + dimension_names: tuple[str | None, ...] | UnsetType attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -196,7 +197,7 @@ def create_default( chunk_grid=NamedConfigModelV3(name="regular", configuration={"chunk_shape": ()}), codecs=(NamedConfigModelV3(name="bytes", configuration={}),), chunk_key_encoding=NamedConfigModelV3(name="default", configuration={}), - dimension_names=None, + dimension_names=UNSET, attributes={}, storage_transformers=(), extra_fields={}, @@ -246,7 +247,7 @@ def to_json(self) -> ArrayMetadataV3: "codecs": tuple(codec.to_json() for codec in self.codecs), "chunk_key_encoding": self.chunk_key_encoding.to_json(), } - if self.dimension_names is not None: + if self.dimension_names is not UNSET: out["dimension_names"] = self.dimension_names if len(self.attributes) > 0: out["attributes"] = self.attributes @@ -279,7 +280,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV3: chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), chunk_key_encoding=NamedConfigModelV3.from_json(parsed["chunk_key_encoding"]), - dimension_names=parsed.get("dimension_names"), + dimension_names=parsed.get("dimension_names", UNSET), attributes=dict(parsed.get("attributes", {})), storage_transformers=tuple( NamedConfigModelV3.from_json(t) for t in parsed.get("storage_transformers", ()) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 0980096a8b..29f32bd330 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -15,6 +15,7 @@ ArrayMetadataModelV3, must_understand_subset, ) +from zarr_metadata.model._sentinel import UNSET, UnsetType from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -63,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] @@ -80,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -107,7 +108,7 @@ def create_default( analog of `list()` returning `[]`. Any field can be overridden by keyword (the same fields accepted by `update`). """ - default = cls(attributes={}, consolidated_metadata=None, extra_fields={}) + default = cls(attributes={}, consolidated_metadata=UNSET, extra_fields={}) return default.update(**overrides) def update(self, **kwargs: Unpack[GroupMetadataModelV3Partial]) -> GroupMetadataModelV3: @@ -128,7 +129,7 @@ def to_json(self) -> GroupMetadataV3: } if len(self.attributes) > 0: out["attributes"] = self.attributes - if self.consolidated_metadata is not None: + if self.consolidated_metadata is not UNSET: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is # not assignable to `ExtensionFieldV3`; see the discussion on @@ -143,10 +144,10 @@ def to_json(self) -> GroupMetadataV3: @classmethod def from_json(cls, data: object) -> GroupMetadataModelV3: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) - consolidated_raw = parsed.get(CONSOLIDATED_METADATA_KEY_V3) + consolidated_raw: object = parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET) consolidated = ( - None - if consolidated_raw is None + UNSET + if consolidated_raw is UNSET else ConsolidatedMetadataModelV3.from_json(consolidated_raw) ) # Sound cast: the TypedDict types all non-standard keys as its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py new file mode 100644 index 0000000000..b0cab883ca --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -0,0 +1,30 @@ +"""The absence sentinel for optional metadata-document keys. + +The models observe one invariant: `None` in a model always corresponds to a +JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed +dimension inside `dimension_names`), and `UNSET` always means the document +key is absent. The two are never interchangeable: for keys where the spec +gives `null` no meaning (`dimension_names` itself, `consolidated_metadata`), +a model `None` would invite serializing an invalid `key: null` spelling, +while `UNSET` cannot leak into a document at all. + +Check with identity: `if model.dimension_names is UNSET: ...`. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Final, Literal + + +class UnsetType(Enum): + """The type of `UNSET`; use in annotations as `T | UnsetType`.""" + + UNSET = "UNSET" + + def __repr__(self) -> str: + return "UNSET" + + +UNSET: Final[Literal[UnsetType.UNSET]] = UnsetType.UNSET +"""Marks a metadata-document key as absent. Test with `is UNSET`.""" diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 4805fc2674..d9af8aeff6 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -13,6 +13,7 @@ ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V3, ARRAY_METADATA_STANDARD_KEYS_V3, + UNSET, ArrayMetadataModelV2, ArrayMetadataModelV2Partial, ArrayMetadataModelV3, @@ -196,8 +197,8 @@ def test_v3_dimension_names_included_when_present() -> None: def test_v3_dimension_names_omitted_when_none() -> None: - """V3 to_json omits dimension_names when they are None.""" - out = ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + """V3 to_json omits dimension_names when they are UNSET.""" + out = ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() assert "dimension_names" not in out @@ -213,7 +214,7 @@ def test_v3_attributes_included_when_dimension_names_is_none() -> None: """ out: dict[str, object] = dict( ArrayMetadataModelV3.create_default( - dimension_names=None, attributes={"foo": "bar"} + dimension_names=UNSET, attributes={"foo": "bar"} ).to_json() ) assert out["attributes"] == {"foo": "bar"} @@ -482,13 +483,13 @@ def test_v3_from_json_reconstructs_required_fields() -> None: def test_v3_from_json_defaults_for_omitted_optionals() -> None: """V3 from_json supplies defaults for omitted optional fields.""" doc = ArrayMetadataModelV3.create_default( - attributes={}, storage_transformers=(), dimension_names=None + attributes={}, storage_transformers=(), dimension_names=UNSET ).to_json() # to_json omits these entirely; from_json must restore defaults model = ArrayMetadataModelV3.from_json(doc) assert model.attributes == {} assert model.storage_transformers == () - assert model.dimension_names is None + assert model.dimension_names is UNSET def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: @@ -570,7 +571,7 @@ def test_from_key_value_missing_key_raises( ArrayMetadataModelV3, ArrayMetadataModelV3.create_default( attributes={}, - dimension_names=None, + dimension_names=UNSET, storage_transformers=(), extra_fields={}, ), @@ -1259,7 +1260,8 @@ def test_dimension_names_null_field_rejected() -> None: assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] # and the model's own None spelling correctly maps to key absence assert ( - "dimension_names" not in ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + "dimension_names" + not in ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() ) @@ -1353,3 +1355,23 @@ def test_v2_null_dimension_separator_rejected() -> None: doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": None} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_dimension_names_absent_and_all_null_are_distinct() -> None: + """An absent dimension_names field and an explicit all-null one are + semantically different documents: the explicit form says every dimension + has a name, which is null; absence says there are no dimension names. + The model preserves the distinction (UNSET vs a tuple of Nones), and both + spellings round-trip faithfully.""" + absent_doc = dict(ArrayMetadataModelV3.create_default(shape=(2, 3)).to_json()) + explicit_doc = absent_doc | {"dimension_names": (None, None)} + + absent = ArrayMetadataModelV3.from_json(absent_doc) + explicit = ArrayMetadataModelV3.from_json(explicit_doc) + + assert absent.dimension_names is UNSET + assert explicit.dimension_names == (None, None) + assert absent != explicit + assert "dimension_names" not in absent.to_json() + assert absent.to_json() == absent_doc + assert explicit.to_json() == explicit_doc diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index d2399220be..fcf57bddde 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -5,6 +5,7 @@ import pytest +from zarr_metadata.model import UNSET from zarr_metadata.model._array import ArrayMetadataModelV3 from zarr_metadata.model._group import ( ConsolidatedMetadataModelV2, @@ -62,7 +63,7 @@ def test_group_v3_extra_fields_overlap_rejected() -> None: with pytest.raises(ValueError, match="Extra fields"): GroupMetadataModelV3( attributes={}, - consolidated_metadata=None, + consolidated_metadata=UNSET, extra_fields={"node_type": {"name": "x", "must_understand": False}}, ) @@ -72,7 +73,7 @@ def test_group_v3_consolidated_extra_field_rejected() -> None: with pytest.raises(ValueError, match="Extra fields"): GroupMetadataModelV3( attributes={}, - consolidated_metadata=None, + consolidated_metadata=UNSET, extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, ) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index c1cd3010d4..65db4ae419 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -149,6 +149,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: cross-field checks. This test documents why the delegation pattern above is the recommended integration.""" from zarr_metadata._common import JSONValue + from zarr_metadata.model import UnsetType from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 @@ -162,6 +163,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: "ArrayMetadataV3": ArrayMetadataV3, "NamedConfigModelV3": NamedConfigModelV3, "MetadataFieldModelV3": NamedConfigModelV3, + "UnsetType": UnsetType, }, ) model_shaped = { @@ -171,7 +173,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, "codecs": [{"name": "bytes", "configuration": {}}], "chunk_key_encoding": {"name": "default", "configuration": {}}, - "dimension_names": None, + "dimension_names": UnsetType.UNSET, "attributes": {}, "storage_transformers": [], "extra_fields": {}, diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index fb2f574a4f..16a540b3f9 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -55,6 +55,8 @@ def _group_rank(s: str) -> int: "ValidationProblem", "MetadataValidationError", "ProblemKind", + "UNSET", + "UnsetType", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From 672d2bf016fdc39adbf8f23a3fadc3e40daa68ce Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:05:38 +0200 Subject: [PATCH 26/48] fix(zarr-metadata): accept and preserve the wild consolidated_metadata null Historical zarr-python versions wrote "consolidated_metadata": null into group documents for groups without consolidated metadata, so real stores contain the spelling; the validator was rejecting those documents ("expected a mapping"). Per the None/UNSET invariant, the field is now honestly three-state: UNSET (key absent), None (the document's literal null, preserved on round-trip), or a ConsolidatedMetadataModelV3. Interpreting null as absence is the consumer's call, not a document rewrite by this layer. Also records an implementation constraint on the sentinel itself: typing_extensions.Sentinel (PEP 661) is the intended spelling, but pyright 1.1.411 degrades a Sentinel to Unknown in dataclass FIELD annotations (function signatures work), verified by probe both with and without enableExperimentalFeatures. Using it would reintroduce suppressions at every use site under the strict gate, so UNSET stays a single-member enum, with the Sentinel switch documented in _sentinel.py for when pyright catches up. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 ++- .../src/zarr_metadata/model/_group.py | 32 +++++++++++++------ .../src/zarr_metadata/model/_sentinel.py | 16 +++++++--- .../src/zarr_metadata/model/_validation.py | 4 ++- .../zarr-metadata/tests/model/test_group.py | 18 +++++++++++ .../tests/model/test_pydantic.py | 4 +-- 6 files changed, 60 insertions(+), 19 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 29c893b3c1..4cb35ad9cf 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -53,4 +53,7 @@ Optional document keys use the `UNSET` sentinel, never `None`: in a model, always means the key is absent. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are -different documents and round-trip as such. +different documents and round-trip as such. A group's +`consolidated_metadata` is accordingly three-state: absent (`UNSET`), +present, or the literal `null` written by historical zarr-python versions, +which is accepted and preserved rather than rejected or silently rewritten. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 29f32bd330..826492e26e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType extra_fields: dict[str, ExtensionFieldV3] @@ -81,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -133,9 +133,14 @@ def to_json(self) -> GroupMetadataV3: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is # not assignable to `ExtensionFieldV3`; see the discussion on - # `zarr_metadata.v3.consolidated`. + # `zarr_metadata.v3.consolidated`. A model None is the document's + # literal null spelling (written by historical zarr-python versions) + # and round-trips as such. out[CONSOLIDATED_METADATA_KEY_V3] = cast( - "ExtensionFieldV3", self.consolidated_metadata.to_json() + "ExtensionFieldV3", + None + if self.consolidated_metadata is None + else self.consolidated_metadata.to_json(), ) for key, value in self.extra_fields.items(): out[key] = value @@ -144,12 +149,19 @@ def to_json(self) -> GroupMetadataV3: @classmethod def from_json(cls, data: object) -> GroupMetadataModelV3: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) - consolidated_raw: object = parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET) - consolidated = ( - UNSET - if consolidated_raw is UNSET - else ConsolidatedMetadataModelV3.from_json(consolidated_raw) - ) + # Cast to object: the TypedDict's extra_items type does not admit null, + # but wild documents (historical zarr-python) contain it. + consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) + consolidated: ConsolidatedMetadataModelV3 | None | UnsetType + if consolidated_raw is UNSET: + consolidated = UNSET + elif consolidated_raw is None: + # Historical zarr-python versions wrote consolidated_metadata: null + # for groups without consolidated metadata; the spelling is + # preserved (interpreting it as absence is the consumer's call). + consolidated = None + else: + consolidated = ConsolidatedMetadataModelV3.from_json(consolidated_raw) # Sound cast: the TypedDict types all non-standard keys as its # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value # type is the union over ALL keys because the key filter cannot narrow it. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index b0cab883ca..29eaa104ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -2,13 +2,19 @@ The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed -dimension inside `dimension_names`), and `UNSET` always means the document -key is absent. The two are never interchangeable: for keys where the spec -gives `null` no meaning (`dimension_names` itself, `consolidated_metadata`), -a model `None` would invite serializing an invalid `key: null` spelling, -while `UNSET` cannot leak into a document at all. +dimension inside `dimension_names`, a group's historical +`consolidated_metadata: null`), and `UNSET` always means the document key is +absent. The two are never interchangeable, so a model value can never leak +into a document as a spelling the writer did not intend. Check with identity: `if model.dimension_names is UNSET: ...`. + +Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended +spelling, but pyright (1.1.411) degrades a Sentinel to `Unknown` in dataclass +FIELD annotations (function signatures work), which would force suppressions +under this package's strict gate at every use site. The single-member enum +gives the same identity semantics with exact `Literal` narrowing; switch to +`Sentinel` once pyright supports it in dataclass fields. """ from __future__ import annotations diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index fc781c7899..4558b57ef4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -518,7 +518,9 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) - if "consolidated_metadata" in doc: + if "consolidated_metadata" in doc and doc["consolidated_metadata"] is not None: + # consolidated_metadata: null is accepted: historical zarr-python + # versions wrote it for groups without consolidated metadata. problems.extend( _prefix( "consolidated_metadata", diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index fcf57bddde..4286976962 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -351,3 +351,21 @@ def test_group_must_understand_fields_partition() -> None: } ) assert set(model.must_understand_fields) == {"implicit"} + + +def test_group_v3_null_consolidated_metadata_preserved() -> None: + """Historical zarr-python versions wrote consolidated_metadata: null for + groups without consolidated metadata, so null is a real wild spelling: + it is accepted and preserved on round-trip (model None = document null), + distinct from key absence (UNSET). Interpreting null as "no consolidated + metadata" is the consumer's call, not a document rewrite.""" + null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} + absent_doc = {"zarr_format": 3, "node_type": "group"} + assert validate_group_metadata_v3(null_doc) == [] + null_model = GroupMetadataModelV3.from_json(null_doc) + absent_model = GroupMetadataModelV3.from_json(absent_doc) + assert null_model.consolidated_metadata is None + assert absent_model.consolidated_metadata is UNSET + assert null_model != absent_model + assert null_model.to_json() == null_doc + assert absent_model.to_json() == absent_doc diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 65db4ae419..c10d1cb471 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -149,7 +149,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: cross-field checks. This test documents why the delegation pattern above is the recommended integration.""" from zarr_metadata._common import JSONValue - from zarr_metadata.model import UnsetType + from zarr_metadata.model import UNSET, UnsetType from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 @@ -173,7 +173,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, "codecs": [{"name": "bytes", "configuration": {}}], "chunk_key_encoding": {"name": "default", "configuration": {}}, - "dimension_names": UnsetType.UNSET, + "dimension_names": UNSET, "attributes": {}, "storage_transformers": [], "extra_fields": {}, From fa7b7cf3660afa02e16b633a6505d10c1b4dac91 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:10:31 +0200 Subject: [PATCH 27/48] fix(zarr-metadata): repair consolidated_metadata null to absence, not preserve it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit d-v-b: the bugged spelling should not be preserved or honored. The three-state field reverts to two states (model | UNSET): a document carrying "consolidated_metadata": null — written by a historical zarr-python bug — remains readable (the validator accepts it so real stores open), but the spelling gets no model representation: it is read as absence and never written back. This is the one deliberate exception to faithful round-tripping, pinned as such: from_json(null_doc) equals from_json(absent_doc), and to_json omits the key. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 8 +++--- .../src/zarr_metadata/model/_group.py | 25 +++++++------------ .../src/zarr_metadata/model/_sentinel.py | 5 ++-- .../src/zarr_metadata/model/_validation.py | 5 ++-- .../zarr-metadata/tests/model/test_group.py | 23 +++++++---------- 5 files changed, 27 insertions(+), 39 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 4cb35ad9cf..c4efbe6f01 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -53,7 +53,7 @@ Optional document keys use the `UNSET` sentinel, never `None`: in a model, always means the key is absent. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are -different documents and round-trip as such. A group's -`consolidated_metadata` is accordingly three-state: absent (`UNSET`), -present, or the literal `null` written by historical zarr-python versions, -which is accepted and preserved rather than rejected or silently rewritten. +different documents and round-trip as such. The `consolidated_metadata: null` +written by a historical zarr-python bug is the one deliberate exception to +faithful round-tripping: those stores remain readable, but the bug spelling +is repaired to absence on read and never written back. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 826492e26e..5c2c2287fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] @@ -81,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -133,14 +133,9 @@ def to_json(self) -> GroupMetadataV3: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is # not assignable to `ExtensionFieldV3`; see the discussion on - # `zarr_metadata.v3.consolidated`. A model None is the document's - # literal null spelling (written by historical zarr-python versions) - # and round-trips as such. + # `zarr_metadata.v3.consolidated`. out[CONSOLIDATED_METADATA_KEY_V3] = cast( - "ExtensionFieldV3", - None - if self.consolidated_metadata is None - else self.consolidated_metadata.to_json(), + "ExtensionFieldV3", self.consolidated_metadata.to_json() ) for key, value in self.extra_fields.items(): out[key] = value @@ -152,14 +147,12 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: # Cast to object: the TypedDict's extra_items type does not admit null, # but wild documents (historical zarr-python) contain it. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) - consolidated: ConsolidatedMetadataModelV3 | None | UnsetType - if consolidated_raw is UNSET: + consolidated: ConsolidatedMetadataModelV3 | UnsetType + if consolidated_raw is UNSET or consolidated_raw is None: + # consolidated_metadata: null was written by a historical + # zarr-python bug; it gets no model representation. It is read as + # absence and never written back — repaired, not preserved. consolidated = UNSET - elif consolidated_raw is None: - # Historical zarr-python versions wrote consolidated_metadata: null - # for groups without consolidated metadata; the spelling is - # preserved (interpreting it as absence is the consumer's call). - consolidated = None else: consolidated = ConsolidatedMetadataModelV3.from_json(consolidated_raw) # Sound cast: the TypedDict types all non-standard keys as its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 29eaa104ab..74f5feec64 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -2,9 +2,8 @@ The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed -dimension inside `dimension_names`, a group's historical -`consolidated_metadata: null`), and `UNSET` always means the document key is -absent. The two are never interchangeable, so a model value can never leak +dimension inside `dimension_names`), and `UNSET` always means the document +key is absent. The two are never interchangeable, so a model value can never leak into a document as a spelling the writer did not intend. Check with identity: `if model.dimension_names is UNSET: ...`. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 4558b57ef4..fae09187a4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -519,8 +519,9 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) if "consolidated_metadata" in doc and doc["consolidated_metadata"] is not None: - # consolidated_metadata: null is accepted: historical zarr-python - # versions wrote it for groups without consolidated metadata. + # consolidated_metadata: null (a historical zarr-python bug) is + # structurally accepted so those stores remain readable, but the model + # repairs it to absence on read and never writes it back. problems.extend( _prefix( "consolidated_metadata", diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 4286976962..0062bf7620 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -353,19 +353,14 @@ def test_group_must_understand_fields_partition() -> None: assert set(model.must_understand_fields) == {"implicit"} -def test_group_v3_null_consolidated_metadata_preserved() -> None: - """Historical zarr-python versions wrote consolidated_metadata: null for - groups without consolidated metadata, so null is a real wild spelling: - it is accepted and preserved on round-trip (model None = document null), - distinct from key absence (UNSET). Interpreting null as "no consolidated - metadata" is the consumer's call, not a document rewrite.""" +def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: + """consolidated_metadata: null was written by a historical zarr-python bug. + Those stores must remain readable, but the bug spelling is not honored: + it is read as absence (UNSET) and never written back — the round-trip + deliberately repairs the document rather than preserving the bug.""" null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} - absent_doc = {"zarr_format": 3, "node_type": "group"} assert validate_group_metadata_v3(null_doc) == [] - null_model = GroupMetadataModelV3.from_json(null_doc) - absent_model = GroupMetadataModelV3.from_json(absent_doc) - assert null_model.consolidated_metadata is None - assert absent_model.consolidated_metadata is UNSET - assert null_model != absent_model - assert null_model.to_json() == null_doc - assert absent_model.to_json() == absent_doc + model = GroupMetadataModelV3.from_json(null_doc) + assert model.consolidated_metadata is UNSET + assert "consolidated_metadata" not in model.to_json() + assert model == GroupMetadataModelV3.from_json({"zarr_format": 3, "node_type": "group"}) From da3094deadf0d3dd88189b00d70eaace19a80e2c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:14:45 +0200 Subject: [PATCH 28/48] docs(zarr-metadata): pin the Sentinel blocker to pyright regression #11115 Investigated: the Unknown-degradation of typing_extensions.Sentinel is a confirmed upstream pyright regression, not by-design. Introduced in 1.1.405 (verified: 1.1.404 is clean on the same probe, 1.1.411 fails), affects reads of any class-body attribute annotation (dataclass or plain class), does not affect function signatures or module variables, and Final on the sentinel does not help. Tracked as microsoft/pyright#11115 (open, bug+regression); #11467 closed as its duplicate. The enum sentinel stays until the fix lands. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_sentinel.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 74f5feec64..e83d49852a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -9,11 +9,13 @@ Check with identity: `if model.dimension_names is UNSET: ...`. Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended -spelling, but pyright (1.1.411) degrades a Sentinel to `Unknown` in dataclass -FIELD annotations (function signatures work), which would force suppressions -under this package's strict gate at every use site. The single-member enum -gives the same identity semantics with exact `Literal` narrowing; switch to -`Sentinel` once pyright supports it in dataclass fields. +spelling, but a confirmed pyright regression (1.1.405 through at least +1.1.411; worked in <= 1.1.404) degrades a Sentinel to `Unknown` when read +from any class-body attribute annotation — dataclass or not; function +signatures and module variables are unaffected, and `Final` on the sentinel +does not help. Tracked as https://github.com/microsoft/pyright/issues/11115. +The single-member enum gives the same identity semantics with exact +`Literal` narrowing; switch to `Sentinel` once that regression is fixed. """ from __future__ import annotations From a9edd1faf9f9f0e801ea1faacdc659335904608b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:17:38 +0200 Subject: [PATCH 29/48] docs(zarr-metadata): sentinel switch is blocked by mypy too, not just pyright MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pinning a working pyright (<= 1.1.404) in CI was considered and does not suffice: the pin controls one of four checker surfaces. Contributor IDEs (Pylance bundles current pyright) and downstream consumers' pyright read the py.typed inline annotations with their own versions, and decisively, mypy 2.1.0 has no PEP 661 support at all — a sentinel in type position is a hard [valid-type] error, which would degrade these fields to Any for mypy consumers, including zarr-python itself. The enum is currently the only spelling with exact types on every surface; switch when pyright#11115 is fixed AND mypy implements PEP 661. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_sentinel.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index e83d49852a..2b77b44b95 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -9,13 +9,17 @@ Check with identity: `if model.dimension_names is UNSET: ...`. Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended -spelling, but a confirmed pyright regression (1.1.405 through at least -1.1.411; worked in <= 1.1.404) degrades a Sentinel to `Unknown` when read -from any class-body attribute annotation — dataclass or not; function -signatures and module variables are unaffected, and `Final` on the sentinel -does not help. Tracked as https://github.com/microsoft/pyright/issues/11115. -The single-member enum gives the same identity semantics with exact -`Literal` narrowing; switch to `Sentinel` once that regression is fixed. +spelling, but two independent checker gaps block it. Pyright: a confirmed +regression (1.1.405 through at least 1.1.411; worked in <= 1.1.404, +https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to +`Unknown` when read from any class-body attribute annotation. Mypy (2.1.0): +no PEP 661 support at all — a sentinel in type position is a hard +`[valid-type]` error, so downstream mypy users (zarr-python itself) would +see these fields as `Any`. Pinning a working pyright in this package's CI +would fix neither contributors' IDEs nor downstream checkers reading the +py.typed annotations. The single-member enum gives the same identity +semantics with exact `Literal` narrowing on every checker; switch to +`Sentinel` once the pyright regression is fixed AND mypy implements PEP 661. """ from __future__ import annotations From 934095e8bf77fc0f7b4c87d4de0a66405ec19432 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:20:01 +0200 Subject: [PATCH 30/48] docs(zarr-metadata): PEP 661 is Final (Python 3.15), not a draft Corrects the sentinel implementation note: PEP 661 was accepted 2026-04-23 and ships as stdlib sentinel in Python 3.15. The two checker gaps blocking the Sentinel spelling (pyright regression #11115, mypy not yet implementing the PEP) are therefore temporary gaps against a Final standard, and the enum is a stopgap with a defined end state. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_sentinel.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 2b77b44b95..5fa80004d2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -8,18 +8,20 @@ Check with identity: `if model.dimension_names is UNSET: ...`. -Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended -spelling, but two independent checker gaps block it. Pyright: a confirmed -regression (1.1.405 through at least 1.1.411; worked in <= 1.1.404, +Implementation note: `typing_extensions.Sentinel` (PEP 661, Final as of +2026-04-23, stdlib in Python 3.15) is the intended spelling, but two +independent checker gaps block it for now. Pyright: a confirmed regression +(1.1.405 through at least 1.1.411; worked in <= 1.1.404, https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to `Unknown` when read from any class-body attribute annotation. Mypy (2.1.0): -no PEP 661 support at all — a sentinel in type position is a hard +has not yet implemented PEP 661 — a sentinel in type position is a hard `[valid-type]` error, so downstream mypy users (zarr-python itself) would see these fields as `Any`. Pinning a working pyright in this package's CI would fix neither contributors' IDEs nor downstream checkers reading the py.typed annotations. The single-member enum gives the same identity semantics with exact `Literal` narrowing on every checker; switch to -`Sentinel` once the pyright regression is fixed AND mypy implements PEP 661. +`Sentinel` once the pyright regression is fixed and mypy support lands — +both expected, now that the PEP is Final. """ from __future__ import annotations From 800546aed3db4129d2bee2843859f5ee91127cf5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:25:46 +0200 Subject: [PATCH 31/48] docs(zarr-metadata): ty fully supports typed sentinels; pyright/mypy are the laggards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ty 0.0.56 types the Sentinel spelling perfectly in dataclass fields: exact T | UNSET unions, both-direction is/is-not narrowing, and wrong-typed constructor arguments rejected (verified with reveal_type, so it is real inference, not silent Any). The checker matrix for sentinel-in-type-position is therefore ty full / pyright regressed (#11115) / mypy not implemented — recorded so the switch decision has current calibration. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/src/zarr_metadata/model/_sentinel.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 5fa80004d2..749e353c8a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -18,10 +18,12 @@ `[valid-type]` error, so downstream mypy users (zarr-python itself) would see these fields as `Any`. Pinning a working pyright in this package's CI would fix neither contributors' IDEs nor downstream checkers reading the -py.typed annotations. The single-member enum gives the same identity -semantics with exact `Literal` narrowing on every checker; switch to -`Sentinel` once the pyright regression is fixed and mypy support lands — -both expected, now that the PEP is Final. +py.typed annotations. For calibration: ty (0.0.56) +already types the Sentinel spelling perfectly — exact unions and +`is`/`is not` narrowing in dataclass fields — so the standard is landing; +pyright and mypy are the laggards. The single-member enum gives the same +identity semantics with exact `Literal` narrowing on every checker; switch +to `Sentinel` once the pyright regression is fixed and mypy support lands. """ from __future__ import annotations From f1fba4123cf50c404af13daaeafa86af801fbf06 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:34:32 +0200 Subject: [PATCH 32/48] feat(zarr-metadata): adopt the PEP 661 sentinel for UNSET MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit d-v-b's call: PEP 661 is Final, ty already types the sentinel spelling exactly, mypy support is in review (python/mypy#21647) and treated as imminent, and pyright has a known-good version — so use the standard sentinel today rather than carrying the enum stopgap. - UNSET is now typing_extensions.Sentinel("UNSET"), used directly in type expressions (tuple[str | None, ...] | UNSET); the UnsetType companion enum is gone from the API. - typing_extensions floor bumped to 4.14 (where Sentinel arrived). - CI pins pyright==1.1.404, the last version before the class-attribute sentinel regression (microsoft/pyright#11115); pyproject documents the same pin for local runs. 0 errors on the pin; ty checks the sentinel fields clean (its 2 remaining diagnostics are its incomplete PEP 728 extra_items write support, unrelated). - Known short-term cost, accepted deliberately: mypy-checked consumers need cast/type-ignore at narrowing sites until mypy#21647 merges, and contributors' Pylance may show phantom Unknowns until the pyright fix ships. Recorded in _sentinel.py and the changelog. - The pydantic native-introspection test reverts to documenting that introspection is unsupported (pydantic 2.13 cannot schema a Sentinel); the delegation patterns are unaffected. Assisted-by: ClaudeCode:claude-fable-5 --- .github/workflows/zarr-metadata.yml | 5 +- packages/zarr-metadata/changes/210.feature.md | 12 ++- packages/zarr-metadata/pyproject.toml | 5 +- .../src/zarr_metadata/__init__.py | 2 - .../src/zarr_metadata/model/__init__.py | 3 +- .../src/zarr_metadata/model/_array.py | 6 +- .../src/zarr_metadata/model/_group.py | 8 +- .../src/zarr_metadata/model/_sentinel.py | 48 ++++------- .../tests/model/test_pydantic.py | 79 ++++++++----------- .../zarr-metadata/tests/test_public_api.py | 1 - 10 files changed, 71 insertions(+), 98 deletions(-) diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 95e8251227..e660483220 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -82,7 +82,10 @@ jobs: - name: Sync test dependency group run: uv sync --group test --python 3.11 - name: Run pyright - run: uv run --group test --with pyright pyright src + # Pinned to the last version that types PEP 661 sentinels in class + # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115). + # Unpin when the fix lands. + run: uv run --group test --with 'pyright==1.1.404' pyright src zarr-metadata-complete: name: zarr-metadata complete diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index c4efbe6f01..12f57c1525 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -47,10 +47,14 @@ convention's default `"."` (the model previously normalized absence to `"/"`, which would misaddress the chunks of real-world default-separator arrays). The value is never null: absent, `"."`, or `"/"` are the only spellings. -Optional document keys use the `UNSET` sentinel, never `None`: in a model, -`None` always corresponds to a JSON `null` in the document (a v2 -`compressor`, an unnamed dimension inside `dimension_names`), and `UNSET` -always means the key is absent. This keeps semantically distinct spellings +Optional document keys use `UNSET` — a PEP 661 sentinel +(`typing_extensions.Sentinel`), usable directly in type expressions — never +`None`: in a model, `None` always corresponds to a JSON `null` in the +document (a v2 `compressor`, an unnamed dimension inside `dimension_names`), +and `UNSET` always means the key is absent. Checker note: ty types the +sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115 +is fixed (this package's CI pins it); mypy users need a `cast` or +`type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are different documents and round-trip as such. The `consolidated_metadata: null` diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 05667d59e3..87f35db1db 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ ] keywords = ["zarr"] dependencies = [ - "typing_extensions>=4.13", + "typing_extensions>=4.14", ] [project.urls] @@ -82,6 +82,9 @@ checks = [ "PR06", ] +# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in +# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel +# relies on. Use the same pin locally; unpin when the fix lands. [tool.pyright] include = ["src"] enableExperimentalFeatures = true diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 4e0cfd4e36..4c1328bfe1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -17,7 +17,6 @@ MetadataValidationError, NamedConfigModelV3, ProblemKind, - UnsetType, ValidationProblem, ) from zarr_metadata.v2.array import ( @@ -356,7 +355,6 @@ "Uint32FillValue", "Uint64DataTypeName", "Uint64FillValue", - "UnsetType", "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 76367eae32..1257109465 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -34,7 +34,7 @@ GroupMetadataModelV3, GroupMetadataModelV3Partial, ) -from zarr_metadata.model._sentinel import UNSET, UnsetType +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V2, @@ -98,7 +98,6 @@ "MetadataValidationError", "NamedConfigModelV3", "ProblemKind", - "UnsetType", "ValidationProblem", "is_array_metadata_v2", "is_array_metadata_v3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 604c9a7d1e..bfa57ddb34 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -10,7 +10,7 @@ from typing_extensions import TypedDict, Unpack -from zarr_metadata.model._sentinel import UNSET, UnsetType +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -134,7 +134,7 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | UnsetType + dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -160,7 +160,7 @@ class ArrayMetadataModelV3: chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | UnsetType + dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 5c2c2287fc..e3a006699b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -15,7 +15,7 @@ ArrayMetadataModelV3, must_understand_subset, ) -from zarr_metadata.model._sentinel import UNSET, UnsetType +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET extra_fields: dict[str, ExtensionFieldV3] @@ -81,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -147,7 +147,7 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: # Cast to object: the TypedDict's extra_items type does not admit null, # but wild documents (historical zarr-python) contain it. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) - consolidated: ConsolidatedMetadataModelV3 | UnsetType + consolidated: ConsolidatedMetadataModelV3 | UNSET if consolidated_raw is UNSET or consolidated_raw is None: # consolidated_metadata: null was written by a historical # zarr-python bug; it gets no model representation. It is read as diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 749e353c8a..eeae0da97a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -3,43 +3,27 @@ The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed dimension inside `dimension_names`), and `UNSET` always means the document -key is absent. The two are never interchangeable, so a model value can never leak -into a document as a spelling the writer did not intend. +key is absent. The two are never interchangeable, so a model value can never +leak into a document as a spelling the writer did not intend. Check with identity: `if model.dimension_names is UNSET: ...`. -Implementation note: `typing_extensions.Sentinel` (PEP 661, Final as of -2026-04-23, stdlib in Python 3.15) is the intended spelling, but two -independent checker gaps block it for now. Pyright: a confirmed regression -(1.1.405 through at least 1.1.411; worked in <= 1.1.404, -https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to -`Unknown` when read from any class-body attribute annotation. Mypy (2.1.0): -has not yet implemented PEP 661 — a sentinel in type position is a hard -`[valid-type]` error, so downstream mypy users (zarr-python itself) would -see these fields as `Any`. Pinning a working pyright in this package's CI -would fix neither contributors' IDEs nor downstream checkers reading the -py.typed annotations. For calibration: ty (0.0.56) -already types the Sentinel spelling perfectly — exact unions and -`is`/`is not` narrowing in dataclass fields — so the standard is landing; -pyright and mypy are the laggards. The single-member enum gives the same -identity semantics with exact `Literal` narrowing on every checker; switch -to `Sentinel` once the pyright regression is fixed and mypy support lands. +Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python +3.15): ty types this spelling exactly, including `is`/`is not` narrowing. +Pyright supports it but a regression (1.1.405+, tracked as +https://github.com/microsoft/pyright/issues/11115) degrades class-attribute +reads to `Unknown`, so this package pins pyright to the last good version +until the fix lands. Mypy support is in review +(https://github.com/python/mypy/pull/21647); until it merges, mypy-checked +consumers of these fields need a `cast` or `type: ignore` at narrowing +sites. This is a deliberate short-term cost: the sentinel is the standard, +and the checkers are converging on it. """ from __future__ import annotations -from enum import Enum -from typing import Final, Literal +from typing_extensions import Sentinel - -class UnsetType(Enum): - """The type of `UNSET`; use in annotations as `T | UnsetType`.""" - - UNSET = "UNSET" - - def __repr__(self) -> str: - return "UNSET" - - -UNSET: Final[Literal[UnsetType.UNSET]] = UnsetType.UNSET -"""Marks a metadata-document key as absent. Test with `is UNSET`.""" +UNSET = Sentinel("UNSET") +"""Marks a metadata-document key as absent (PEP 661 sentinel; usable directly +in type expressions, e.g. `tuple[str, ...] | UNSET`). Test with `is UNSET`.""" diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index c10d1cb471..7e80d52ee8 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -34,6 +34,8 @@ ConfigDict, InstanceOf, PlainSerializer, + PydanticSchemaGenerationError, + PydanticUserError, TypeAdapter, ValidationError, model_validator, @@ -142,59 +144,40 @@ def test_type_adapter_standalone() -> None: # --- the road not taken: native dataclass introspection ---------------------- -def test_native_dataclass_introspection_is_possible_but_diverges() -> None: - """Pydantic CAN introspect the model dataclass after a namespace rebuild, - but that path validates the model shape, not the document: it rejects the - document form, coerces booleans into dimensions, and skips the library's - cross-field checks. This test documents why the delegation pattern above - is the recommended integration.""" +def test_native_dataclass_introspection_is_not_supported() -> None: + """Pydantic cannot field-introspect the model dataclasses: the UNSET + sentinel (PEP 661, typing_extensions.Sentinel) in the optional-field + annotations has no pydantic schema (as of pydantic 2.13), so even the + rebuild-with-namespace recipe fails. Introspection was already the wrong + tool before the sentinel existed — it validated the model shape rather + than the document, and its lax coercion re-opened validator holes (e.g. + shape=[True, -5] coerced to (1, -5)) — so the delegation patterns above + are the only supported integrations. If this test ever fails because + pydantic learned to handle sentinels, revisit whether the introspection + path needs its divergences documented again.""" from zarr_metadata._common import JSONValue - from zarr_metadata.model import UNSET, UnsetType + from zarr_metadata.model import UNSET from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 - adapter = TypeAdapter(ArrayMetadataModelV3) - adapter.rebuild( - force=True, - _types_namespace={ - "JSONValue": JSONValue, - "ExtensionFieldV3": ExtensionFieldV3, - "MetadataV3": MetadataV3, - "ArrayMetadataV3": ArrayMetadataV3, - "NamedConfigModelV3": NamedConfigModelV3, - "MetadataFieldModelV3": NamedConfigModelV3, - "UnsetType": UnsetType, - }, - ) - model_shaped = { - "shape": [10], - "fill_value": 0, - "data_type": {"name": "uint8", "configuration": {}}, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, - "codecs": [{"name": "bytes", "configuration": {}}], - "chunk_key_encoding": {"name": "default", "configuration": {}}, - "dimension_names": UNSET, - "attributes": {}, - "storage_transformers": [], - "extra_fields": {}, - } - # model-shaped data validates, nested named configs and all - model = adapter.validate_python(model_shaped) - assert isinstance(model.data_type, NamedConfigModelV3) - - # divergence 1: the DOCUMENT form is rejected — no from_json normalization - with pytest.raises(ValidationError): - adapter.validate_python(model_shaped | {"data_type": "uint8"}) - - # divergence 2: lax coercion re-opens holes the library validators close - coerced = adapter.validate_python(model_shaped | {"shape": [True, -5]}) - assert coerced.shape == (1, -5) # from_json would reject both entries - - # __post_init__ invariants DO still run under pydantic construction - with pytest.raises(ValidationError, match="Extra fields"): - adapter.validate_python( - model_shaped | {"extra_fields": {"shape": {"must_understand": False}}} + def build_and_use() -> None: + adapter = TypeAdapter(ArrayMetadataModelV3) + adapter.rebuild( + force=True, + _types_namespace={ + "JSONValue": JSONValue, + "ExtensionFieldV3": ExtensionFieldV3, + "MetadataV3": MetadataV3, + "ArrayMetadataV3": ArrayMetadataV3, + "NamedConfigModelV3": NamedConfigModelV3, + "MetadataFieldModelV3": NamedConfigModelV3, + "UNSET": UNSET, + }, ) + adapter.validate_python({}) + + with pytest.raises((AttributeError, PydanticSchemaGenerationError, PydanticUserError)): + build_and_use() # --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 16a540b3f9..ee26e80c5a 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -56,7 +56,6 @@ def _group_rank(s: str) -> int: "MetadataValidationError", "ProblemKind", "UNSET", - "UnsetType", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From 8156ac0df92ea796cba83c64c555f9290af71ee7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:50:51 +0200 Subject: [PATCH 33/48] fix(zarr-metadata): .zattrs presence is part of the store, not an artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the last flagged round-trip question from the initial port: to_key_value on the v2 models always emitted a .zattrs key, so a store that never had one gained a file on round-trip. Per d-v-b's ruling, attributes on ArrayMetadataModelV2/GroupMetadataModelV2 is now `dict[str, JSONValue] | UNSET`: UNSET means no .zattrs file (and no attributes key in the merged document form) and emits nothing, while any dict — including an explicit empty {} — means the file exists and is emitted. The two spellings stay distinct through round-trips, per the None/UNSET invariant; create_default defaults to UNSET (a fresh minimal node has no .zattrs). Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 6 +++ .../src/zarr_metadata/model/_array.py | 50 +++++++++++-------- .../src/zarr_metadata/model/_group.py | 45 +++++++++-------- .../zarr-metadata/tests/model/test_array.py | 39 ++++++++++----- .../zarr-metadata/tests/model/test_group.py | 17 +++++-- 5 files changed, 98 insertions(+), 59 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 12f57c1525..f711d46af5 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -61,3 +61,9 @@ different documents and round-trip as such. The `consolidated_metadata: null` written by a historical zarr-python bug is the one deliberate exception to faithful round-tripping: those stores remain readable, but the bug spelling is repaired to absence on read and never written back. + +The v2 models treat the `.zattrs` file's presence as part of the store: +`attributes` is `UNSET` when no `.zattrs` file exists (and `to_key_value` +emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a +file. Previously `to_key_value` always emitted `.zattrs`, silently adding a +file to stores that never had one. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index bfa57ddb34..f68c1b0f5e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -331,7 +331,7 @@ class ArrayMetadataModelV2Partial(TypedDict, total=False): compressor: CodecMetadataV2 | None filters: tuple[CodecMetadataV2, ...] | None dimension_separator: ArrayDimensionSeparatorV2 - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET @dataclass(frozen=True, slots=True, kw_only=True) @@ -341,9 +341,12 @@ class ArrayMetadataModelV2: A canonical, lossless representation of the `.zarray` content plus the sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are held in their raw JSON forms and are never interpreted; `fill_value` is - held verbatim in its JSON form. One spelling normalization: a `.zarray` - that omits `dimension_separator` means `"."` by the v2 convention, and - the model holds and re-emits that value explicitly. + held verbatim in its JSON form. `attributes` is `UNSET` when no + `.zattrs` file (or merged `attributes` key) exists — distinct from an + explicit empty `.zattrs`, which is `{}` and round-trips as a file. One + spelling normalization: a `.zarray` that omits `dimension_separator` + means `"."` by the v2 convention, and the model holds and re-emits that + value explicitly. """ zarr_format: Literal[2] = field(default=2, init=False) @@ -359,7 +362,7 @@ class ArrayMetadataModelV2: # normalization, like the v3 bare-string metadata-field form). The value # is never None: the document grammar has no null spelling for this field. dimension_separator: ArrayDimensionSeparatorV2 = field(default=".") - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: """ @@ -400,15 +403,16 @@ def create_default( order="C", compressor=None, filters=None, - attributes={}, + attributes=UNSET, ) return default.update(**overrides) def to_json(self) -> ArrayMetadataV2: - """Return the merged in-memory document form, INCLUDING `attributes`. + """Return the merged in-memory document form. - This is not the on-disk `.zarray` content: a conforming `.zarray` must - exclude `attributes` (they live in the sibling `.zattrs` file). Use + `attributes` is included when set (even empty). This is not the + on-disk `.zarray` content: a conforming `.zarray` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ out: ArrayMetadataV2 = { @@ -419,10 +423,11 @@ def to_json(self) -> ArrayMetadataV2: "chunks": self.chunks, "fill_value": self.fill_value, "dimension_separator": self.dimension_separator, - "attributes": self.attributes, "compressor": self.compressor, "filters": self.filters, } + if self.attributes is not UNSET: + out["attributes"] = self.attributes return out @classmethod @@ -437,24 +442,25 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: compressor=parsed["compressor"], filters=parsed["filters"], dimension_separator=parsed.get("dimension_separator", "."), - attributes=dict(parsed.get("attributes", {})), + attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET), ) @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) - zattrs: dict[str, JSONValue] = ( - load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) - if ATTRIBUTES_STORE_KEY_V2 in mapping - else {} - ) - return cls.from_json({**zarray, "attributes": zattrs}) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + return cls.from_json({**zarray, "attributes": zattrs}) + return cls.from_json(dict(zarray)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zarray` - # document must exclude them. + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} - return { - ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8"), - ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), - } + out = {ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8")} + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( + "utf-8" + ) + return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index e3a006699b..222b0b3bf9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -260,7 +260,7 @@ class GroupMetadataModelV2Partial(TypedDict, total=False): `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. """ - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET @dataclass(frozen=True, slots=True, kw_only=True) @@ -269,11 +269,14 @@ class GroupMetadataModelV2: A canonical, lossless representation of the `.zgroup` content plus the sibling `.zattrs` attributes, folded into a single in-memory value - (mirroring the merged `GroupMetadataV2` document form). + (mirroring the merged `GroupMetadataV2` document form). `attributes` is + `UNSET` when no `.zattrs` file (or merged `attributes` key) exists — + distinct from an explicit empty `.zattrs`, which is `{}` and round-trips + as a file. """ zarr_format: Literal[2] = field(default=2, init=False) - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET @classmethod def create_default( @@ -286,7 +289,7 @@ def create_default( analog of `list()` returning `[]`. Any field can be overridden by keyword (the same fields accepted by `update`). """ - default = cls(attributes={}) + default = cls(attributes=UNSET) return default.update(**overrides) def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadataModelV2: @@ -301,40 +304,42 @@ def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadata return dataclasses.replace(self, **kwargs) def to_json(self) -> GroupMetadataV2: - """Return the merged in-memory document form, INCLUDING `attributes`. + """Return the merged in-memory document form. - This is not the on-disk `.zgroup` content: a conforming `.zgroup` must - exclude `attributes` (they live in the sibling `.zattrs` file). Use + `attributes` is included when set (even empty). This is not the + on-disk `.zgroup` content: a conforming `.zgroup` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ out: GroupMetadataV2 = {"zarr_format": self.zarr_format} - if len(self.attributes) > 0: + if self.attributes is not UNSET: out["attributes"] = self.attributes return out @classmethod def from_json(cls, data: object) -> GroupMetadataModelV2: parsed = parse_group_metadata_v2(arrays_to_tuples(data)) - return cls(attributes=dict(parsed.get("attributes", {}))) + return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) - zattrs: dict[str, JSONValue] = ( - load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) - if ATTRIBUTES_STORE_KEY_V2 in mapping - else {} - ) - return cls.from_json({**zgroup, "attributes": zattrs}) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + return cls.from_json({**zgroup, "attributes": zattrs}) + return cls.from_json(dict(zgroup)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zgroup` - # document must exclude them. + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} - return { - GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8"), - ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), - } + out = {GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8")} + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( + "utf-8" + ) + return out @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index d9af8aeff6..ff27923635 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -326,7 +326,7 @@ def test_v2_create_default_is_valid_empty_array() -> None: assert m.fill_value == 0 assert m.compressor is None assert m.filters is None - assert m.attributes == {} + assert m.attributes is UNSET assert validate_array_metadata_v2(m.to_json()) == [] assert ArrayMetadataModelV2.from_json(m.to_json()) == m @@ -671,12 +671,16 @@ def test_v2_from_json_reconstructs_fields() -> None: assert model.attributes == {"a": 1} -def test_v2_from_json_defaults_when_attributes_absent() -> None: - """V2 from_json defaults attributes to empty when the key is absent.""" - doc = ArrayMetadataModelV2.create_default().to_json() - del doc["attributes"] - model = ArrayMetadataModelV2.from_json(doc) - assert model.attributes == {} +def test_v2_from_json_attributes_absent_is_unset() -> None: + """V2 from_json reads an absent attributes key as UNSET, distinct from an + explicit empty mapping.""" + absent = ArrayMetadataModelV2.from_json(ArrayMetadataModelV2.create_default().to_json()) + explicit = ArrayMetadataModelV2.from_json( + ArrayMetadataModelV2.create_default(attributes={}).to_json() + ) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert absent != explicit # --- ArrayMetadataModelV2.from_key_value -------------------------------- @@ -690,12 +694,21 @@ def test_v2_from_key_value_remerges_zattrs() -> None: assert model.shape == (10,) -def test_v2_from_key_value_absent_zattrs_gives_empty_attributes() -> None: - """V2 from_key_value yields empty attributes when .zattrs is absent.""" - kv: dict[str, bytes] = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) - del kv[".zattrs"] - model = ArrayMetadataModelV2.from_key_value(kv) - assert model.attributes == {} +def test_v2_zattrs_presence_round_trips() -> None: + """The .zattrs file's presence is part of the store: an absent file reads + as UNSET and emits no .zattrs; an explicit empty file reads as {} and + emits .zattrs — the two stores stay distinct through a round-trip.""" + explicit_kv = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) + assert ".zattrs" in explicit_kv + absent_kv = dict(explicit_kv) + del absent_kv[".zattrs"] + + absent = ArrayMetadataModelV2.from_key_value(absent_kv) + explicit = ArrayMetadataModelV2.from_key_value(explicit_kv) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert ".zattrs" not in absent.to_key_value() + assert ".zattrs" in explicit.to_key_value() def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 0062bf7620..58b92f5d71 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -116,10 +116,19 @@ def test_group_v2_key_value_split() -> None: assert GroupMetadataModelV2.from_key_value(kv) == model -def test_group_v2_from_key_value_without_zattrs() -> None: - """A v2 group with no .zattrs file parses with empty attributes.""" - model = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) - assert model.attributes == {} +def test_group_v2_zattrs_presence_round_trips() -> None: + """A v2 group with no .zattrs file parses with UNSET attributes and emits + no .zattrs; an explicit empty .zattrs stays a file — the stores remain + distinct through a round-trip.""" + absent = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + assert absent.attributes is UNSET + assert ".zattrs" not in absent.to_key_value() + explicit = GroupMetadataModelV2.from_key_value( + {".zgroup": b'{"zarr_format": 2}', ".zattrs": b"{}"} + ) + assert explicit.attributes == {} + assert ".zattrs" in explicit.to_key_value() + assert absent != explicit def test_group_v2_json_roundtrip() -> None: From 340a17d9f91547f788790ac54aee74a8587dc761 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 8 Jul 2026 10:35:50 +0200 Subject: [PATCH 34/48] fix(zarr-metadata): require typing_extensions>=4.16 so UNSET pickles by reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Models hold the UNSET sentinel as field values (dimension_names, attributes), so any object graph containing a model must survive pickling and deep-copying. A sentinel's contract is identity — state-based pickling would produce impostor objects that fail every `is UNSET` check — which is why typing_extensions <= 4.15 refused to pickle sentinels at all. typing_extensions 4.16 implements Sentinel.__reduce__ as pickling by reference (a lookup of the sentinel's name on its defining module), the same mechanism enum members use, so the singleton identity survives the round trip. Bump the floor and pin the behavior with tests: identity across pickle/copy/deepcopy, models holding UNSET round-tripping, and a guard that a non-importable sentinel still fails loudly rather than pickling by state. Co-Authored-By: Claude Fable 5 --- packages/zarr-metadata/pyproject.toml | 6 +- .../src/zarr_metadata/model/_sentinel.py | 8 ++ .../tests/model/test_sentinel.py | 95 +++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/tests/model/test_sentinel.py diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 87f35db1db..60f5c4aa95 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -32,7 +32,11 @@ classifiers = [ ] keywords = ["zarr"] dependencies = [ - "typing_extensions>=4.14", + # >=4.16: first release where `Sentinel` pickles by reference + # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model + # holding it — can cross process boundaries and be deep-copied with its + # singleton identity intact. 4.15 and earlier refuse to pickle sentinels. + "typing_extensions>=4.16", ] [project.urls] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index eeae0da97a..ad71e216fa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -8,6 +8,14 @@ Check with identity: `if model.dimension_names is UNSET: ...`. +Because the contract is identity, the sentinel must never be reconstructed +from state: pickling and copying work by *reference* (typing_extensions >= +4.16 implements `Sentinel.__reduce__` as a lookup of the sentinel's name on +its defining module), so `pickle.loads(pickle.dumps(UNSET)) is UNSET` holds +across process boundaries, and models holding `UNSET` pickle and deep-copy +freely. Earlier typing_extensions releases refused to pickle sentinels +outright — hence the `>=4.16` floor in this package's dependencies. + Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python 3.15): ty types this spelling exactly, including `is`/`is not` narrowing. Pyright supports it but a regression (1.1.405+, tracked as diff --git a/packages/zarr-metadata/tests/model/test_sentinel.py b/packages/zarr-metadata/tests/model/test_sentinel.py new file mode 100644 index 0000000000..78aa60e208 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_sentinel.py @@ -0,0 +1,95 @@ +"""Tests for the pickling and copying behavior of the `UNSET` sentinel. + +The sentinel's contract is identity, so it must never be reconstructed from +state. typing_extensions >= 4.16 pickles sentinels by reference (a lookup of +the sentinel's name on its defining module), which preserves the singleton +across process boundaries; these tests pin that behavior, since models hold +`UNSET` as field values and must survive pickling and deep-copying. + +The model round-trip tests compare whole structures: dataclass equality +compares every field, and `UNSET` compares by identity, so an impostor +sentinel produced by state-based pickling would fail the equality check. +""" + +from __future__ import annotations + +import copy +import pickle + +import pytest +from typing_extensions import Sentinel + +from zarr_metadata.model import ( + UNSET, + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, +) + +# Whole-model cases covering the states we know are problematic for +# serialization: every optional-key field in the UNSET (absent) state, the +# same fields in the present state (including present-but-empty, which must +# stay distinct from absent), and UNSET nested inside consolidated metadata. +MODEL_CASES = { + "array-v3-dimension-names-unset": ArrayMetadataModelV3.create_default(shape=(4,)), + "array-v3-dimension-names-set": ArrayMetadataModelV3.create_default(shape=(2, 2)).update( + dimension_names=("x", None) + ), + "array-v2-attributes-unset": ArrayMetadataModelV2.create_default(shape=(4,)), + "array-v2-attributes-empty": ArrayMetadataModelV2.create_default(shape=(4,), attributes={}), + "group-v2-attributes-unset": GroupMetadataModelV2.create_default(), + "group-v2-attributes-set": GroupMetadataModelV2.create_default(attributes={"a": 1}), + "group-v3-consolidated-unset": GroupMetadataModelV3.create_default(), + "group-v3-consolidated-with-unset-inside": GroupMetadataModelV3.create_default( + consolidated_metadata=ConsolidatedMetadataModelV3( + metadata={ + "child": ArrayMetadataModelV3.create_default(shape=(4,)), + "subgroup": GroupMetadataModelV3.create_default(), + } + ) + ), +} + + +def test_unset_pickle_round_trip_preserves_identity() -> None: + restored = pickle.loads(pickle.dumps(UNSET)) + assert restored is UNSET + + +def test_unset_copy_preserves_identity() -> None: + assert copy.copy(UNSET) is UNSET + assert copy.deepcopy(UNSET) is UNSET + + +@pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) +def test_model_pickle_round_trip( + model: ArrayMetadataModelV2 + | ArrayMetadataModelV3 + | GroupMetadataModelV2 + | GroupMetadataModelV3, +) -> None: + restored = pickle.loads(pickle.dumps(model)) + assert restored == model + + +@pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) +def test_model_deepcopy( + model: ArrayMetadataModelV2 + | ArrayMetadataModelV3 + | GroupMetadataModelV2 + | GroupMetadataModelV3, +) -> None: + assert copy.deepcopy(model) == model + + +def test_non_importable_sentinel_fails_to_pickle() -> None: + """Sentinels pickle by reference, never by state. A sentinel that is not + an importable attribute of its module has no reference to pickle, so + dumping it must fail loudly — a successful dump here would mean the + implementation regressed to state-based pickling, which would produce + identity-breaking impostor objects on the receiving side.""" + local_sentinel = Sentinel("local_sentinel") + with pytest.raises((pickle.PicklingError, TypeError)): + pickle.dumps(local_sentinel) From 06668a74dbb73725f6efae9ea2aa04ccc792ec2c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 8 Jul 2026 11:48:53 +0200 Subject: [PATCH 35/48] refactor(zarr-metadata): format-version-first naming; dataclasses primary, JSON suffix for documents Applies the naming decisions from the PR discussion: ZarrV2/ZarrV3 moves to the front of every type name so a format version cannot be misread as a class revision, and the model dataclasses take the bare entity names (ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV3ConsolidatedMetadata, ZarrV3NamedConfig, role alias ZarrV3MetadataField) while the TypedDict document forms carry a JSON suffix (ZarrV3ArrayMetadataJSON, ..., ZarrV3MetadataFieldJSON, ZarrV3NamedConfigJSON). The zarr_metadata.pydantic field types take the bare entity names, matching the model classes they validate into; the module now references the model module qualified to keep those names free. Raw-layer names released in 0.3.0 are renamed without aliases (pre-1.0), documented in changes/4119.removal.md. Validation problem messages name documents in plain English instead of type names. snake_case function names (validate_array_metadata_v3, ...) and SCREAMING_SNAKE constants are deliberately untouched: the revision ambiguity the rename fixes does not arise for them, and renaming them is a separate decision. Co-Authored-By: Claude Fable 5 --- packages/zarr-metadata/README.md | 4 +- packages/zarr-metadata/changes/210.feature.md | 10 +- .../zarr-metadata/changes/4119.removal.md | 23 ++ .../src/zarr_metadata/__init__.py | 116 +++--- .../src/zarr_metadata/_common.py | 2 +- .../src/zarr_metadata/model/__init__.py | 48 +-- .../src/zarr_metadata/model/_array.py | 180 +++++---- .../src/zarr_metadata/model/_group.py | 120 +++--- .../src/zarr_metadata/model/_validation.py | 64 ++-- .../src/zarr_metadata/pydantic.py | 102 +++-- .../src/zarr_metadata/v2/__init__.py | 28 +- .../src/zarr_metadata/v2/array.py | 60 +-- .../src/zarr_metadata/v2/codec.py | 4 +- .../src/zarr_metadata/v2/consolidated.py | 4 +- .../src/zarr_metadata/v2/group.py | 16 +- .../src/zarr_metadata/v3/__init__.py | 18 +- .../src/zarr_metadata/v3/_common.py | 8 +- .../src/zarr_metadata/v3/array.py | 46 +-- .../src/zarr_metadata/v3/codec/__init__.py | 2 +- .../src/zarr_metadata/v3/codec/cast_value.py | 4 +- .../v3/codec/sharding_indexed.py | 6 +- .../src/zarr_metadata/v3/consolidated.py | 12 +- .../src/zarr_metadata/v3/data_type/struct.py | 4 +- .../src/zarr_metadata/v3/group.py | 20 +- .../zarr-metadata/tests/model/test_array.py | 361 +++++++++--------- .../zarr-metadata/tests/model/test_group.py | 100 ++--- .../tests/model/test_pydantic.py | 48 +-- .../tests/model/test_pydantic_module.py | 50 +-- .../tests/model/test_sentinel.py | 42 +- .../tests/test_partial_equivalence.py | 16 +- .../zarr-metadata/tests/test_public_api.py | 60 +-- .../tests/v2/consolidated/test_fixtures.py | 4 +- .../tests/v3/array/test_fixtures.py | 6 +- .../tests/v3/array/with_extra_field.json | 2 +- .../tests/v3/consolidated/test_fixtures.py | 4 +- .../tests/v3/group/test_fixtures.py | 4 +- 36 files changed, 803 insertions(+), 795 deletions(-) create mode 100644 packages/zarr-metadata/changes/4119.removal.md diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index a842e07886..f2884782fd 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -20,12 +20,12 @@ Zarr metadata. Pair them with a runtime validator like ```python import json from pydantic import TypeAdapter -from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON with open("zarr.json", "rb") as f: raw = json.load(f) -metadata = TypeAdapter(ArrayMetadataV3).validate_python(raw) +metadata = TypeAdapter(ZarrV3ArrayMetadataJSON).validate_python(raw) ``` ## What this is *not* diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index f711d46af5..fda1a88cf7 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -1,12 +1,12 @@ -Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, -`ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`, -`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `NamedConfigModelV3`) +Added `zarr_metadata.model`: frozen-dataclass models (`ZarrV2ArrayMetadata`, +`ZarrV3ArrayMetadata`, `ZarrV2GroupMetadata`, `ZarrV3GroupMetadata`, +`ZarrV2ConsolidatedMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV3NamedConfig`) that are canonical, lossless representations of Zarr metadata documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, storage transformers) is held as a name + configuration pair; nothing is interpreted. Model fields -are annotated with the role alias `MetadataFieldModelV3` (today exactly -`NamedConfigModelV3`), so the annotations convey the logical meaning of the +are annotated with the role alias `ZarrV3MetadataField` (today exactly +`ZarrV3NamedConfig`), so the annotations convey the logical meaning of the field and stay put if the spec ever adds a new field form. Validation is strict about what the types declare: v2 `dtype` / `order` / diff --git a/packages/zarr-metadata/changes/4119.removal.md b/packages/zarr-metadata/changes/4119.removal.md new file mode 100644 index 0000000000..0c90f59131 --- /dev/null +++ b/packages/zarr-metadata/changes/4119.removal.md @@ -0,0 +1,23 @@ +The document (TypedDict) types are renamed to put the format version at the +front of the name and to mark the JSON-document form with a `JSON` suffix, +so a format version can never be misread as a class revision and the bare +entity names are reserved for the `zarr_metadata.model` dataclasses: + +- `ArrayMetadataV2` → `ZarrV2ArrayMetadataJSON` (and `...Partial` accordingly) +- `ArrayMetadataV3` → `ZarrV3ArrayMetadataJSON` (and `...Partial` accordingly) +- `GroupMetadataV2` → `ZarrV2GroupMetadataJSON` (and `...Partial` accordingly) +- `GroupMetadataV3` → `ZarrV3GroupMetadataJSON` (and `...Partial` accordingly) +- `ConsolidatedMetadataV2` → `ZarrV2ConsolidatedMetadataJSON` +- `ConsolidatedMetadataV3` → `ZarrV3ConsolidatedMetadataJSON` +- `NamedConfigV3` → `ZarrV3NamedConfigJSON` +- `MetadataV3` → `ZarrV3MetadataFieldJSON` (the union of the bare-name and + named-configuration spellings of one metadata field) +- `ExtensionFieldV3` → `ZarrV3ExtensionField` +- `CodecMetadataV2` → `ZarrV2CodecMetadata` +- `DataTypeMetadataV2` → `ZarrV2DataTypeMetadata` +- `ArrayOrderV2` → `ZarrV2ArrayOrder` +- `ArrayDimensionSeparatorV2` → `ZarrV2ArrayDimensionSeparator` + +The old names are removed, not aliased. The `zarr_metadata.pydantic` field +types take the bare entity names (`ZarrV3ArrayMetadata`, ...), matching the +model classes they validate into. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 4c1328bfe1..ab7bce2ff2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,40 +1,48 @@ from importlib.metadata import version -from zarr_metadata._common import JSONValue, NamedConfigV3 +from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON from zarr_metadata.model import ( UNSET, - ArrayMetadataModelV2, - ArrayMetadataModelV2Partial, - ArrayMetadataModelV3, - ArrayMetadataModelV3Partial, - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV2Partial, - GroupMetadataModelV3, - GroupMetadataModelV3Partial, - MetadataFieldModelV3, MetadataValidationError, - NamedConfigModelV3, ProblemKind, ValidationProblem, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, ) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, ARRAY_ORDER_V2, - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayMetadataV2Partial, - ArrayOrderV2, - DataTypeMetadataV2, ZArrayMetadata, + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayMetadataJSONPartial, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, ) from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.codec import CodecMetadataV2 -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial, ZGroupMetadata -from zarr_metadata.v3._common import MetadataV3 -from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial, ExtensionFieldV3 +from zarr_metadata.v2.codec import ZarrV2CodecMetadata +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ( + ZarrV2GroupMetadataJSON, + ZarrV2GroupMetadataJSONPartial, + ZGroupMetadata, +) +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ( + ZarrV3ArrayMetadataJSON, + ZarrV3ArrayMetadataJSONPartial, + ZarrV3ExtensionField, +) from zarr_metadata.v3.chunk_grid.rectilinear import ( RECTILINEAR_CHUNK_GRID_NAME, RectilinearChunkGridMetadata, @@ -104,7 +112,7 @@ TransposeCodecName, ) from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata, ZstdCodecName -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON from zarr_metadata.v3.data_type.bool import ( BOOL_DATA_TYPE_NAME, BoolDataTypeName, @@ -203,7 +211,7 @@ Uint64DataTypeName, Uint64FillValue, ) -from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial __version__ = version("zarr-metadata") @@ -253,16 +261,6 @@ "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", - "ArrayDimensionSeparatorV2", - "ArrayMetadataModelV2", - "ArrayMetadataModelV2Partial", - "ArrayMetadataModelV3", - "ArrayMetadataModelV3Partial", - "ArrayMetadataV2", - "ArrayMetadataV2Partial", - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ArrayOrderV2", "BloscCName", "BloscCodecMetadata", "BloscCodecName", @@ -277,37 +275,22 @@ "CastRoundingMode", "CastValueCodecMetadata", "CastValueCodecName", - "CodecMetadataV2", "Complex64DataTypeName", "Complex64FillValue", "Complex128DataTypeName", "Complex128FillValue", - "ConsolidatedMetadataModelV2", - "ConsolidatedMetadataModelV3", - "ConsolidatedMetadataV2", - "ConsolidatedMetadataV3", "Crc32cCodecMetadata", "Crc32cCodecName", - "DataTypeMetadataV2", "DefaultChunkKeyEncodingMetadata", "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingSeparator", "Endianness", - "ExtensionFieldV3", "Float16DataTypeName", "Float16FillValue", "Float32DataTypeName", "Float32FillValue", "Float64DataTypeName", "Float64FillValue", - "GroupMetadataModelV2", - "GroupMetadataModelV2Partial", - "GroupMetadataModelV3", - "GroupMetadataModelV3Partial", - "GroupMetadataV2", - "GroupMetadataV2Partial", - "GroupMetadataV3", - "GroupMetadataV3Partial", "GzipCodecMetadata", "GzipCodecName", "Int8DataTypeName", @@ -319,11 +302,7 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", - "MetadataFieldModelV3", - "MetadataV3", "MetadataValidationError", - "NamedConfigModelV3", - "NamedConfigV3", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", @@ -362,6 +341,35 @@ "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", + "ZarrV2ArrayMetadataPartial", + "ZarrV2ArrayOrder", + "ZarrV2CodecMetadata", + "ZarrV2ConsolidatedMetadata", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2DataTypeMetadata", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", + "ZarrV2GroupMetadataPartial", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ArrayMetadataPartial", + "ZarrV3ConsolidatedMetadata", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", + "ZarrV3GroupMetadataPartial", + "ZarrV3MetadataField", + "ZarrV3MetadataFieldJSON", + "ZarrV3NamedConfig", + "ZarrV3NamedConfigJSON", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index b335cd0bd6..b66183f875 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -24,7 +24,7 @@ """ -class NamedConfigV3(TypedDict): +class ZarrV3NamedConfigJSON(TypedDict): """ Externally-tagged union member for a metadata field. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 1257109465..1889b20300 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -15,24 +15,24 @@ ARRAY_METADATA_STORE_KEY_V2, ARRAY_METADATA_STORE_KEY_V3, ATTRIBUTES_STORE_KEY_V2, - ArrayMetadataModelV2, - ArrayMetadataModelV2Partial, - ArrayMetadataModelV3, - ArrayMetadataModelV3Partial, - MetadataFieldModelV3, - NamedConfigModelV3, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, ) from zarr_metadata.model._group import ( CONSOLIDATED_METADATA_KEY_V3, CONSOLIDATED_METADATA_STORE_KEY_V2, GROUP_METADATA_STORE_KEY_V2, GROUP_METADATA_STORE_KEY_V3, - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV2Partial, - GroupMetadataModelV3, - GroupMetadataModelV3Partial, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, ) from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( @@ -84,21 +84,21 @@ "GROUP_METADATA_STORE_KEY_V2", "GROUP_METADATA_STORE_KEY_V3", "UNSET", - "ArrayMetadataModelV2", - "ArrayMetadataModelV2Partial", - "ArrayMetadataModelV3", - "ArrayMetadataModelV3Partial", - "ConsolidatedMetadataModelV2", - "ConsolidatedMetadataModelV3", - "GroupMetadataModelV2", - "GroupMetadataModelV2Partial", - "GroupMetadataModelV3", - "GroupMetadataModelV3Partial", - "MetadataFieldModelV3", "MetadataValidationError", - "NamedConfigModelV3", "ProblemKind", "ValidationProblem", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataPartial", + "ZarrV2ConsolidatedMetadata", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataPartial", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataPartial", + "ZarrV3ConsolidatedMetadata", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataPartial", + "ZarrV3MetadataField", + "ZarrV3NamedConfig", "is_array_metadata_v2", "is_array_metadata_v3", "is_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index f68c1b0f5e..1f4a2ded4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -25,30 +25,30 @@ if TYPE_CHECKING: from zarr_metadata._common import JSONValue from zarr_metadata.v2.array import ( - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayOrderV2, - DataTypeMetadataV2, + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, ) - from zarr_metadata.v2.codec import CodecMetadataV2 - from zarr_metadata.v3._common import MetadataV3 - from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 + from zarr_metadata.v2.codec import ZarrV2CodecMetadata + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField -ArrayMetadataStoreKeyV3 = Literal["zarr.json"] -ARRAY_METADATA_STORE_KEY_V3: Final[ArrayMetadataStoreKeyV3] = "zarr.json" +ZarrV3ArrayMetadataStoreKey = Literal["zarr.json"] +ARRAY_METADATA_STORE_KEY_V3: Final[ZarrV3ArrayMetadataStoreKey] = "zarr.json" -ArrayMetadataStoreKeyV2 = Literal[".zarray"] -ARRAY_METADATA_STORE_KEY_V2: Final[ArrayMetadataStoreKeyV2] = ".zarray" +ZarrV2ArrayMetadataStoreKey = Literal[".zarray"] +ARRAY_METADATA_STORE_KEY_V2: Final[ZarrV2ArrayMetadataStoreKey] = ".zarray" -AttributesStoreKeyV2 = Literal[".zattrs"] -ATTRIBUTES_STORE_KEY_V2: Final[AttributesStoreKeyV2] = ".zattrs" +ZarrV2AttributesStoreKey = Literal[".zattrs"] +ATTRIBUTES_STORE_KEY_V2: Final[ZarrV2AttributesStoreKey] = ".zattrs" @dataclass(frozen=True, slots=True, kw_only=True) -class NamedConfigModelV3: +class ZarrV3NamedConfig: """A v3 metadata field in normalized form: a name plus a configuration. - This is the in-memory model of `MetadataV3` (a bare name string or a + This is the in-memory model of `ZarrV3MetadataFieldJSON` (a bare name string or a `{name, configuration}` mapping): the bare-name and missing-configuration forms normalize to an empty configuration. """ @@ -56,11 +56,11 @@ class NamedConfigModelV3: name: str configuration: dict[str, JSONValue] - def to_json(self) -> MetadataV3: + def to_json(self) -> ZarrV3MetadataFieldJSON: return {"name": self.name, "configuration": self.configuration} @classmethod - def from_json(cls, data: object) -> NamedConfigModelV3: + def from_json(cls, data: object) -> ZarrV3NamedConfig: field = parse_metadata_field_v3(data) if isinstance(field, str): return cls(name=field, configuration={}) @@ -73,23 +73,23 @@ def from_json(cls, data: object) -> NamedConfigModelV3: return cls(name=field["name"], configuration=configuration) -MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 +ZarrV3MetadataField: TypeAlias = ZarrV3NamedConfig """The in-memory model of one field of a v3 metadata document. This is the role-named alias for annotation positions: model fields and -consumer signatures should say `MetadataFieldModelV3` (the logical meaning) -rather than `NamedConfigModelV3` (the serialized form the field currently +consumer signatures should say `ZarrV3MetadataField` (the logical meaning) +rather than `ZarrV3NamedConfig` (the serialized form the field currently takes). Today every metadata field normalizes to a named configuration, so -the alias is exactly `NamedConfigModelV3`; if a future spec revision adds a +the alias is exactly `ZarrV3NamedConfig`; if a future spec revision adds a field form that cannot be normalized to name + configuration, this alias widens to a union and annotation sites do not change. Mirrors the raw-layer -split between `NamedConfigV3` (shape) and `MetadataV3` (field union). +split between `ZarrV3NamedConfigJSON` (shape) and `ZarrV3MetadataFieldJSON` (field union). """ def must_understand_subset( - extra_fields: Mapping[str, ExtensionFieldV3], -) -> dict[str, ExtensionFieldV3]: + extra_fields: Mapping[str, ZarrV3ExtensionField], +) -> dict[str, ZarrV3ExtensionField]: """The subset of `extra_fields` the reader is obligated to understand. Per the v3 spec, an extension field is implicitly `must_understand: True` @@ -98,11 +98,11 @@ def must_understand_subset( explicitly `must_understand: false`. A non-mapping field value cannot carry the explicit waiver, so it always requires understanding (the runtime isinstance check defends against values looser than the declared - `ExtensionFieldV3`). + `ZarrV3ExtensionField`). """ fields = cast("Mapping[str, object]", extra_fields) return cast( - "dict[str, ExtensionFieldV3]", + "dict[str, ZarrV3ExtensionField]", { name: value for name, value in fields.items() @@ -114,13 +114,13 @@ def must_understand_subset( ) -class ArrayMetadataModelV3Partial(TypedDict, total=False): +class ZarrV3ArrayMetadataPartial(TypedDict, total=False): """ - Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. + Partial form of the constructor-settable fields of `ZarrV3ArrayMetadata`. Every key is optional and typed with the model's own (not serialized) value types, so it describes valid keyword arguments to - `ArrayMetadataModelV3.update`. The `init=False` fields `zarr_format` and + `ZarrV3ArrayMetadata.update`. The `init=False` fields `zarr_format` and `node_type` are intentionally excluded, since they cannot be passed to `dataclasses.replace`. @@ -130,24 +130,24 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): shape: tuple[int, ...] fill_value: JSONValue - data_type: MetadataFieldModelV3 - chunk_grid: MetadataFieldModelV3 - codecs: tuple[MetadataFieldModelV3, ...] - chunk_key_encoding: MetadataFieldModelV3 + data_type: ZarrV3MetadataField + chunk_grid: ZarrV3MetadataField + codecs: tuple[ZarrV3MetadataField, ...] + chunk_key_encoding: ZarrV3MetadataField dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] - storage_transformers: tuple[MetadataFieldModelV3, ...] - extra_fields: dict[str, ExtensionFieldV3] + storage_transformers: tuple[ZarrV3MetadataField, ...] + extra_fields: dict[str, ZarrV3ExtensionField] @dataclass(frozen=True, slots=True, kw_only=True) -class ArrayMetadataModelV3: +class ZarrV3ArrayMetadata: """In-memory model of a v3 array metadata document. A canonical, lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, - `codecs`, `storage_transformers`) are held as `MetadataFieldModelV3` - values (currently always `NamedConfigModelV3` name + configuration pairs) + `codecs`, `storage_transformers`) are held as `ZarrV3MetadataField` + values (currently always `ZarrV3NamedConfig` name + configuration pairs) and are never interpreted; `fill_value` is held verbatim in its JSON form. """ @@ -156,19 +156,17 @@ class ArrayMetadataModelV3: node_type: Literal["array"] = field(default="array", init=False) shape: tuple[int, ...] fill_value: JSONValue - data_type: MetadataFieldModelV3 - chunk_grid: MetadataFieldModelV3 - codecs: tuple[MetadataFieldModelV3, ...] - chunk_key_encoding: MetadataFieldModelV3 + data_type: ZarrV3MetadataField + chunk_grid: ZarrV3MetadataField + codecs: tuple[ZarrV3MetadataField, ...] + chunk_key_encoding: ZarrV3MetadataField dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] - storage_transformers: tuple[MetadataFieldModelV3, ...] - extra_fields: dict[str, ExtensionFieldV3] + storage_transformers: tuple[ZarrV3MetadataField, ...] + extra_fields: dict[str, ZarrV3ExtensionField] @classmethod - def create_default( - cls, **overrides: Unpack[ArrayMetadataModelV3Partial] - ) -> ArrayMetadataModelV3: + def create_default(cls, **overrides: Unpack[ZarrV3ArrayMetadataPartial]) -> ZarrV3ArrayMetadata: """ Create a default (empty) v3 array metadata model, with optional overrides. @@ -187,16 +185,16 @@ def create_default( responsibility. """ if "shape" in overrides and "chunk_grid" not in overrides: - overrides["chunk_grid"] = NamedConfigModelV3( + overrides["chunk_grid"] = ZarrV3NamedConfig( name="regular", configuration={"chunk_shape": tuple(overrides["shape"])} ) default = cls( shape=(), fill_value=0, - data_type=NamedConfigModelV3(name="uint8", configuration={}), - chunk_grid=NamedConfigModelV3(name="regular", configuration={"chunk_shape": ()}), - codecs=(NamedConfigModelV3(name="bytes", configuration={}),), - chunk_key_encoding=NamedConfigModelV3(name="default", configuration={}), + data_type=ZarrV3NamedConfig(name="uint8", configuration={}), + chunk_grid=ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": ()}), + codecs=(ZarrV3NamedConfig(name="bytes", configuration={}),), + chunk_key_encoding=ZarrV3NamedConfig(name="default", configuration={}), dimension_names=UNSET, attributes={}, storage_transformers=(), @@ -204,12 +202,12 @@ def create_default( ) return default.update(**overrides) - def update(self, **kwargs: Unpack[ArrayMetadataModelV3Partial]) -> ArrayMetadataModelV3: + def update(self, **kwargs: Unpack[ZarrV3ArrayMetadataPartial]) -> ZarrV3ArrayMetadata: """ - Return a new `ArrayMetadataModelV3` with the given fields updated. + Return a new `ZarrV3ArrayMetadata` with the given fields updated. Only the constructor-settable fields listed in - `ArrayMetadataModelV3Partial` can be updated; any attempt to update + `ZarrV3ArrayMetadataPartial` can be updated; any attempt to update other fields (including the fixed `zarr_format` / `node_type`) is rejected at the type level. Each given field fully replaces its previous value, including `extra_fields`. @@ -230,14 +228,14 @@ def __post_init__(self) -> None: [ ValidationProblem( ("extra_fields",), - "Extra fields cannot overlap with standard ArrayMetadataV3 fields", + "Extra fields cannot overlap with standard Zarr V3 array metadata fields", "invalid_value", ) ] ) - def to_json(self) -> ArrayMetadataV3: - out: ArrayMetadataV3 = { + def to_json(self) -> ZarrV3ArrayMetadataJSON: + out: ZarrV3ArrayMetadataJSON = { "zarr_format": self.zarr_format, "node_type": self.node_type, "shape": self.shape, @@ -264,32 +262,32 @@ def to_json(self) -> ArrayMetadataV3: return out @classmethod - def from_json(cls, data: object) -> ArrayMetadataModelV3: + def from_json(cls, data: object) -> ZarrV3ArrayMetadata: parsed = parse_array_metadata_v3(arrays_to_tuples(data)) # Sound cast: the TypedDict types all non-standard keys as its - # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # `extra_items` (`ZarrV3ExtensionField`); the comprehension's inferred value # type is the union over ALL keys because the key filter cannot narrow it. extra_fields = cast( - "dict[str, ExtensionFieldV3]", + "dict[str, ZarrV3ExtensionField]", {k: v for k, v in parsed.items() if k not in ARRAY_METADATA_STANDARD_KEYS_V3}, ) return cls( shape=parsed["shape"], fill_value=parsed["fill_value"], - data_type=NamedConfigModelV3.from_json(parsed["data_type"]), - chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), - codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), - chunk_key_encoding=NamedConfigModelV3.from_json(parsed["chunk_key_encoding"]), + data_type=ZarrV3NamedConfig.from_json(parsed["data_type"]), + chunk_grid=ZarrV3NamedConfig.from_json(parsed["chunk_grid"]), + codecs=tuple(ZarrV3NamedConfig.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=ZarrV3NamedConfig.from_json(parsed["chunk_key_encoding"]), dimension_names=parsed.get("dimension_names", UNSET), attributes=dict(parsed.get("attributes", {})), storage_transformers=tuple( - NamedConfigModelV3.from_json(t) for t in parsed.get("storage_transformers", ()) + ZarrV3NamedConfig.from_json(t) for t in parsed.get("storage_transformers", ()) ), extra_fields=extra_fields, ) @property - def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with @@ -301,7 +299,7 @@ def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: return must_understand_subset(self.extra_fields) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: @@ -310,12 +308,12 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes } -class ArrayMetadataModelV2Partial(TypedDict, total=False): +class ZarrV2ArrayMetadataPartial(TypedDict, total=False): """ - Partial form of the constructor-settable fields of `ArrayMetadataModelV2`. + Partial form of the constructor-settable fields of `ZarrV2ArrayMetadata`. Every key is optional and typed with the model's own value types, so it - describes valid keyword arguments to `ArrayMetadataModelV2.update` and + describes valid keyword arguments to `ZarrV2ArrayMetadata.update` and `create_default`. The `init=False` field `zarr_format` is intentionally excluded, since it cannot be passed to `dataclasses.replace`. @@ -324,18 +322,18 @@ class ArrayMetadataModelV2Partial(TypedDict, total=False): """ shape: tuple[int, ...] - dtype: DataTypeMetadataV2 + dtype: ZarrV2DataTypeMetadata chunks: tuple[int, ...] fill_value: JSONValue - order: ArrayOrderV2 - compressor: CodecMetadataV2 | None - filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: ArrayDimensionSeparatorV2 + order: ZarrV2ArrayOrder + compressor: ZarrV2CodecMetadata | None + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: ZarrV2ArrayDimensionSeparator attributes: dict[str, JSONValue] | UNSET @dataclass(frozen=True, slots=True, kw_only=True) -class ArrayMetadataModelV2: +class ZarrV2ArrayMetadata: """In-memory model of a v2 array metadata document. A canonical, lossless representation of the `.zarray` content plus the @@ -351,34 +349,32 @@ class ArrayMetadataModelV2: zarr_format: Literal[2] = field(default=2, init=False) shape: tuple[int, ...] - dtype: DataTypeMetadataV2 + dtype: ZarrV2DataTypeMetadata chunks: tuple[int, ...] fill_value: JSONValue - order: ArrayOrderV2 - compressor: CodecMetadataV2 | None - filters: tuple[CodecMetadataV2, ...] | None + order: ZarrV2ArrayOrder + compressor: ZarrV2CodecMetadata | None + filters: tuple[ZarrV2CodecMetadata, ...] | None # "." is the v2 convention's default for an ABSENT dimension_separator key; # from_json normalizes absence to it (a semantics-preserving spelling # normalization, like the v3 bare-string metadata-field form). The value # is never None: the document grammar has no null spelling for this field. - dimension_separator: ArrayDimensionSeparatorV2 = field(default=".") + dimension_separator: ZarrV2ArrayDimensionSeparator = field(default=".") attributes: dict[str, JSONValue] | UNSET - def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: + def update(self, **kwargs: Unpack[ZarrV2ArrayMetadataPartial]) -> ZarrV2ArrayMetadata: """ - Return a new `ArrayMetadataModelV2` with the given fields updated. + Return a new `ZarrV2ArrayMetadata` with the given fields updated. Only the constructor-settable fields listed in - `ArrayMetadataModelV2Partial` can be updated; the fixed `zarr_format` is + `ZarrV2ArrayMetadataPartial` can be updated; the fixed `zarr_format` is rejected at the type level. Each given field fully replaces its previous value. """ return dataclasses.replace(self, **kwargs) @classmethod - def create_default( - cls, **overrides: Unpack[ArrayMetadataModelV2Partial] - ) -> ArrayMetadataModelV2: + def create_default(cls, **overrides: Unpack[ZarrV2ArrayMetadataPartial]) -> ZarrV2ArrayMetadata: """ Create a default (empty) v2 array metadata model, with optional overrides. @@ -407,7 +403,7 @@ def create_default( ) return default.update(**overrides) - def to_json(self) -> ArrayMetadataV2: + def to_json(self) -> ZarrV2ArrayMetadataJSON: """Return the merged in-memory document form. `attributes` is included when set (even empty). This is not the @@ -415,7 +411,7 @@ def to_json(self) -> ArrayMetadataV2: `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ - out: ArrayMetadataV2 = { + out: ZarrV2ArrayMetadataJSON = { "zarr_format": self.zarr_format, "shape": self.shape, "dtype": self.dtype, @@ -431,7 +427,7 @@ def to_json(self) -> ArrayMetadataV2: return out @classmethod - def from_json(cls, data: object) -> ArrayMetadataModelV2: + def from_json(cls, data: object) -> ZarrV2ArrayMetadata: parsed = parse_array_metadata_v2(arrays_to_tuples(data)) return cls( shape=parsed["shape"], @@ -446,7 +442,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: ) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping: zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 222b0b3bf9..dc544bd19e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -12,7 +12,7 @@ from zarr_metadata.model._array import ( ATTRIBUTES_STORE_KEY_V2, - ArrayMetadataModelV3, + ZarrV3ArrayMetadata, must_understand_subset, ) from zarr_metadata.model._sentinel import UNSET @@ -29,19 +29,19 @@ if TYPE_CHECKING: from zarr_metadata._common import JSONValue - from zarr_metadata.v2.group import GroupMetadataV2 - from zarr_metadata.v3.array import ExtensionFieldV3 - from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 - from zarr_metadata.v3.group import GroupMetadataV3 + from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON + from zarr_metadata.v3.array import ZarrV3ExtensionField + from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON + from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -GroupMetadataStoreKeyV3 = Literal["zarr.json"] -GROUP_METADATA_STORE_KEY_V3: Final[GroupMetadataStoreKeyV3] = "zarr.json" +ZarrV3GroupMetadataStoreKey = Literal["zarr.json"] +GROUP_METADATA_STORE_KEY_V3: Final[ZarrV3GroupMetadataStoreKey] = "zarr.json" -GroupMetadataStoreKeyV2 = Literal[".zgroup"] -GROUP_METADATA_STORE_KEY_V2: Final[GroupMetadataStoreKeyV2] = ".zgroup" +ZarrV2GroupMetadataStoreKey = Literal[".zgroup"] +GROUP_METADATA_STORE_KEY_V2: Final[ZarrV2GroupMetadataStoreKey] = ".zgroup" -ConsolidatedMetadataStoreKeyV2 = Literal[".zmetadata"] -CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ConsolidatedMetadataStoreKeyV2] = ".zmetadata" +ZarrV2ConsolidatedMetadataStoreKey = Literal[".zmetadata"] +CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ZarrV2ConsolidatedMetadataStoreKey] = ".zmetadata" # The key under which consolidated metadata is embedded in a v3 group document. # This is a reference-implementation convention (not a spec artifact), stored @@ -49,12 +49,12 @@ CONSOLIDATED_METADATA_KEY_V3: Final = "consolidated_metadata" -class GroupMetadataModelV3Partial(TypedDict, total=False): +class ZarrV3GroupMetadataPartial(TypedDict, total=False): """ - Partial form of the constructor-settable fields of `GroupMetadataModelV3`. + Partial form of the constructor-settable fields of `ZarrV3GroupMetadata`. Every key is optional and typed with the model's own value types, so it - describes valid keyword arguments to `GroupMetadataModelV3.update` and + describes valid keyword arguments to `ZarrV3GroupMetadata.update` and `create_default`. The `init=False` fields `zarr_format` and `node_type` are intentionally excluded, since they cannot be passed to `dataclasses.replace`. @@ -64,12 +64,12 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET - extra_fields: dict[str, ExtensionFieldV3] + consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + extra_fields: dict[str, ZarrV3ExtensionField] @dataclass(frozen=True, slots=True, kw_only=True) -class GroupMetadataModelV3: +class ZarrV3GroupMetadata: """In-memory model of a v3 group metadata document. A canonical, lossless representation of the `zarr.json` content for a @@ -81,8 +81,8 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET - extra_fields: dict[str, ExtensionFieldV3] + consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + extra_fields: dict[str, ZarrV3ExtensionField] def __post_init__(self) -> None: reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} @@ -91,16 +91,14 @@ def __post_init__(self) -> None: [ ValidationProblem( ("extra_fields",), - "Extra fields cannot overlap with standard GroupMetadataV3 fields", + "Extra fields cannot overlap with standard Zarr V3 group metadata fields", "invalid_value", ) ] ) @classmethod - def create_default( - cls, **overrides: Unpack[GroupMetadataModelV3Partial] - ) -> GroupMetadataModelV3: + def create_default(cls, **overrides: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMetadata: """ Create a default (empty) v3 group metadata model, with optional overrides. @@ -111,19 +109,19 @@ def create_default( default = cls(attributes={}, consolidated_metadata=UNSET, extra_fields={}) return default.update(**overrides) - def update(self, **kwargs: Unpack[GroupMetadataModelV3Partial]) -> GroupMetadataModelV3: + def update(self, **kwargs: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMetadata: """ - Return a new `GroupMetadataModelV3` with the given fields updated. + Return a new `ZarrV3GroupMetadata` with the given fields updated. Only the constructor-settable fields listed in - `GroupMetadataModelV3Partial` can be updated; the fixed `zarr_format` / + `ZarrV3GroupMetadataPartial` can be updated; the fixed `zarr_format` / `node_type` are rejected at the type level. Each given field fully replaces its previous value, including `extra_fields`. """ return dataclasses.replace(self, **kwargs) - def to_json(self) -> GroupMetadataV3: - out: GroupMetadataV3 = { + def to_json(self) -> ZarrV3GroupMetadataJSON: + out: ZarrV3GroupMetadataJSON = { "zarr_format": self.zarr_format, "node_type": self.node_type, } @@ -132,34 +130,34 @@ def to_json(self) -> GroupMetadataV3: if self.consolidated_metadata is not UNSET: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is - # not assignable to `ExtensionFieldV3`; see the discussion on + # not assignable to `ZarrV3ExtensionField`; see the discussion on # `zarr_metadata.v3.consolidated`. out[CONSOLIDATED_METADATA_KEY_V3] = cast( - "ExtensionFieldV3", self.consolidated_metadata.to_json() + "ZarrV3ExtensionField", self.consolidated_metadata.to_json() ) for key, value in self.extra_fields.items(): out[key] = value return out @classmethod - def from_json(cls, data: object) -> GroupMetadataModelV3: + def from_json(cls, data: object) -> ZarrV3GroupMetadata: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) # Cast to object: the TypedDict's extra_items type does not admit null, # but wild documents (historical zarr-python) contain it. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) - consolidated: ConsolidatedMetadataModelV3 | UNSET + consolidated: ZarrV3ConsolidatedMetadata | UNSET if consolidated_raw is UNSET or consolidated_raw is None: # consolidated_metadata: null was written by a historical # zarr-python bug; it gets no model representation. It is read as # absence and never written back — repaired, not preserved. consolidated = UNSET else: - consolidated = ConsolidatedMetadataModelV3.from_json(consolidated_raw) + consolidated = ZarrV3ConsolidatedMetadata.from_json(consolidated_raw) # Sound cast: the TypedDict types all non-standard keys as its - # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # `extra_items` (`ZarrV3ExtensionField`); the comprehension's inferred value # type is the union over ALL keys because the key filter cannot narrow it. extra_fields = cast( - "dict[str, ExtensionFieldV3]", + "dict[str, ZarrV3ExtensionField]", { k: v for k, v in parsed.items() @@ -173,7 +171,7 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: ) @property - def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with @@ -185,7 +183,7 @@ def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: return must_understand_subset(self.extra_fields) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: @@ -195,7 +193,7 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes @dataclass(frozen=True, slots=True, kw_only=True) -class ConsolidatedMetadataModelV3: +class ZarrV3ConsolidatedMetadata: """In-memory model of v3 inline consolidated metadata. Models the reference-implementation convention where consolidated metadata @@ -207,7 +205,7 @@ class ConsolidatedMetadataModelV3: kind: Literal["inline"] = field(default="inline", init=False) must_understand: bool = False - metadata: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] + metadata: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] def __post_init__(self) -> None: if self.must_understand is not False: @@ -222,7 +220,7 @@ def __post_init__(self) -> None: ] ) - def to_json(self) -> ConsolidatedMetadataV3: + def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: # `must_understand` is emitted as the literal False: the field is typed # permissively as `bool`, but `__post_init__` guarantees the value. return { @@ -232,27 +230,27 @@ def to_json(self) -> ConsolidatedMetadataV3: } @classmethod - def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: + def from_json(cls, data: object) -> ZarrV3ConsolidatedMetadata: problems = validate_consolidated_metadata_v3(data) if problems: raise MetadataValidationError(problems) env = cast("Mapping[str, object]", data) - entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} + entries: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] = {} for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): node_type = cast("Mapping[str, object]", entry).get("node_type") if node_type == "array": - entries[key] = ArrayMetadataModelV3.from_json(entry) + entries[key] = ZarrV3ArrayMetadata.from_json(entry) else: - entries[key] = GroupMetadataModelV3.from_json(entry) + entries[key] = ZarrV3GroupMetadata.from_json(entry) return cls(metadata=entries) -class GroupMetadataModelV2Partial(TypedDict, total=False): +class ZarrV2GroupMetadataPartial(TypedDict, total=False): """ - Partial form of the constructor-settable fields of `GroupMetadataModelV2`. + Partial form of the constructor-settable fields of `ZarrV2GroupMetadata`. Every key is optional and typed with the model's own value types, so it - describes valid keyword arguments to `GroupMetadataModelV2.update` and + describes valid keyword arguments to `ZarrV2GroupMetadata.update` and `create_default`. The `init=False` field `zarr_format` is intentionally excluded, since it cannot be passed to `dataclasses.replace`. @@ -264,12 +262,12 @@ class GroupMetadataModelV2Partial(TypedDict, total=False): @dataclass(frozen=True, slots=True, kw_only=True) -class GroupMetadataModelV2: +class ZarrV2GroupMetadata: """In-memory model of a v2 group metadata document. A canonical, lossless representation of the `.zgroup` content plus the sibling `.zattrs` attributes, folded into a single in-memory value - (mirroring the merged `GroupMetadataV2` document form). `attributes` is + (mirroring the merged `ZarrV2GroupMetadataJSON` document form). `attributes` is `UNSET` when no `.zattrs` file (or merged `attributes` key) exists — distinct from an explicit empty `.zattrs`, which is `{}` and round-trips as a file. @@ -279,9 +277,7 @@ class GroupMetadataModelV2: attributes: dict[str, JSONValue] | UNSET @classmethod - def create_default( - cls, **overrides: Unpack[GroupMetadataModelV2Partial] - ) -> GroupMetadataModelV2: + def create_default(cls, **overrides: Unpack[ZarrV2GroupMetadataPartial]) -> ZarrV2GroupMetadata: """ Create a default (empty) v2 group metadata model, with optional overrides. @@ -292,18 +288,18 @@ def create_default( default = cls(attributes=UNSET) return default.update(**overrides) - def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadataModelV2: + def update(self, **kwargs: Unpack[ZarrV2GroupMetadataPartial]) -> ZarrV2GroupMetadata: """ - Return a new `GroupMetadataModelV2` with the given fields updated. + Return a new `ZarrV2GroupMetadata` with the given fields updated. Only the constructor-settable fields listed in - `GroupMetadataModelV2Partial` can be updated; the fixed `zarr_format` + `ZarrV2GroupMetadataPartial` can be updated; the fixed `zarr_format` is rejected at the type level. Each given field fully replaces its previous value. """ return dataclasses.replace(self, **kwargs) - def to_json(self) -> GroupMetadataV2: + def to_json(self) -> ZarrV2GroupMetadataJSON: """Return the merged in-memory document form. `attributes` is included when set (even empty). This is not the @@ -311,18 +307,18 @@ def to_json(self) -> GroupMetadataV2: `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ - out: GroupMetadataV2 = {"zarr_format": self.zarr_format} + out: ZarrV2GroupMetadataJSON = {"zarr_format": self.zarr_format} if self.attributes is not UNSET: out["attributes"] = self.attributes return out @classmethod - def from_json(cls, data: object) -> GroupMetadataModelV2: + def from_json(cls, data: object) -> ZarrV2GroupMetadata: parsed = parse_group_metadata_v2(arrays_to_tuples(data)) return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping: zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) @@ -343,7 +339,7 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes @dataclass(frozen=True, slots=True, kw_only=True) -class ConsolidatedMetadataModelV2: +class ZarrV2ConsolidatedMetadata: """In-memory model of a v2 `.zmetadata` document. The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, @@ -363,7 +359,7 @@ def to_json(self) -> dict[str, JSONValue]: } @classmethod - def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: + def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: if not isinstance(data, Mapping): raise MetadataValidationError( [ValidationProblem((), "expected a mapping", "invalid_type")] @@ -393,7 +389,7 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: return cls(metadata=entries_tupled) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ConsolidatedMetadataModelV2: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetadata: return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index fae09187a4..d2ef94da52 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -21,11 +21,11 @@ from typing_extensions import TypeIs from zarr_metadata._common import JSONValue -from zarr_metadata.v2.array import ArrayMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2 -from zarr_metadata.v3._common import MetadataV3 -from zarr_metadata.v3.array import ArrayMetadataV3 -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] """Machine-readable classification of a `ValidationProblem`. @@ -112,33 +112,33 @@ def parse_json(value: object) -> JSONValue: # this set is an extension field. Built from the TypedDict's required/optional # key sets (which resolve inherited keys, unlike `__annotations__`). ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( - ArrayMetadataV3.__required_keys__ + ZarrV3ArrayMetadataJSON.__required_keys__ ) ARRAY_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( - ArrayMetadataV3.__optional_keys__ + ZarrV3ArrayMetadataJSON.__optional_keys__ ) ARRAY_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 ) ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( - ArrayMetadataV2.__required_keys__ + ZarrV2ArrayMetadataJSON.__required_keys__ ) # The standard top-level keys of a v3 group metadata document. Anything outside # this set is an extension field. GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( - GroupMetadataV3.__required_keys__ + ZarrV3GroupMetadataJSON.__required_keys__ ) GROUP_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( - GroupMetadataV3.__optional_keys__ + ZarrV3GroupMetadataJSON.__optional_keys__ ) GROUP_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( GROUP_METADATA_REQUIRED_KEYS_V3 | GROUP_METADATA_OPTIONAL_KEYS_V3 ) GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( - GroupMetadataV2.__required_keys__ + ZarrV2GroupMetadataJSON.__required_keys__ ) @@ -196,17 +196,17 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: return problems -def is_metadata_field_v3(value: object) -> TypeIs[MetadataV3]: +def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: """Whether `value` is a v3 metadata field: a bare name or a named config.""" return not validate_metadata_field_v3(value) -def parse_metadata_field_v3(value: object) -> MetadataV3: - """Return `value` narrowed to `MetadataV3`, or raise `MetadataValidationError`.""" +def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: + """Return `value` narrowed to `ZarrV3MetadataFieldJSON`, or raise `MetadataValidationError`.""" problems = validate_metadata_field_v3(value) if problems: raise MetadataValidationError(problems) - return cast(MetadataV3, value) + return cast(ZarrV3MetadataFieldJSON, value) def _is_int_sequence(value: object) -> bool: @@ -355,17 +355,17 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: return problems -def is_array_metadata_v3(value: object) -> TypeIs[ArrayMetadataV3]: +def is_array_metadata_v3(value: object) -> TypeIs[ZarrV3ArrayMetadataJSON]: """Whether `value` is a structurally-valid v3 array metadata document.""" return not validate_array_metadata_v3(value) -def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: - """Return `value` narrowed to `ArrayMetadataV3`, or raise `MetadataValidationError`.""" +def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: + """Return `value` narrowed to `ZarrV3ArrayMetadataJSON`, or raise `MetadataValidationError`.""" problems = validate_array_metadata_v3(value) if problems: raise MetadataValidationError(problems) - return cast(ArrayMetadataV3, value) + return cast(ZarrV3ArrayMetadataJSON, value) def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: @@ -436,17 +436,17 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: return problems -def is_array_metadata_v2(value: object) -> TypeIs[ArrayMetadataV2]: +def is_array_metadata_v2(value: object) -> TypeIs[ZarrV2ArrayMetadataJSON]: """Whether `value` is a structurally-valid v2 array metadata document.""" return not validate_array_metadata_v2(value) -def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: - """Return `value` narrowed to `ArrayMetadataV2`, or raise `MetadataValidationError`.""" +def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: + """Return `value` narrowed to `ZarrV2ArrayMetadataJSON`, or raise `MetadataValidationError`.""" problems = validate_array_metadata_v2(value) if problems: raise MetadataValidationError(problems) - return cast(ArrayMetadataV2, value) + return cast(ZarrV2ArrayMetadataJSON, value) def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: @@ -455,7 +455,7 @@ def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: Locs are value-relative (the caller prefixes with `consolidated_metadata` where appropriate). Entries recurse into the array and group document validators, so a validator verdict always agrees with what - `ConsolidatedMetadataModelV3.from_json` accepts. + `ZarrV3ConsolidatedMetadata.from_json` accepts. """ if not isinstance(value, Mapping): return [ValidationProblem((), "expected a mapping", "invalid_type")] @@ -531,17 +531,17 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: return problems -def is_group_metadata_v3(value: object) -> TypeIs[GroupMetadataV3]: +def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: """Whether `value` is a structurally-valid v3 group metadata document.""" return not validate_group_metadata_v3(value) -def parse_group_metadata_v3(value: object) -> GroupMetadataV3: - """Return `value` narrowed to `GroupMetadataV3`, or raise `MetadataValidationError`.""" +def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: + """Return `value` narrowed to `ZarrV3GroupMetadataJSON`, or raise `MetadataValidationError`.""" problems = validate_group_metadata_v3(value) if problems: raise MetadataValidationError(problems) - return cast(GroupMetadataV3, value) + return cast(ZarrV3GroupMetadataJSON, value) def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: @@ -560,17 +560,17 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: return problems -def is_group_metadata_v2(value: object) -> TypeIs[GroupMetadataV2]: +def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: """Whether `value` is a structurally-valid v2 group metadata document.""" return not validate_group_metadata_v2(value) -def parse_group_metadata_v2(value: object) -> GroupMetadataV2: - """Return `value` narrowed to `GroupMetadataV2`, or raise `MetadataValidationError`.""" +def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: + """Return `value` narrowed to `ZarrV2GroupMetadataJSON`, or raise `MetadataValidationError`.""" problems = validate_group_metadata_v2(value) if problems: raise MetadataValidationError(problems) - return cast(GroupMetadataV2, value) + return cast(ZarrV2GroupMetadataJSON, value) def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 3eada76316..8b88b49128 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -20,10 +20,10 @@ class ArrayManifest(BaseModel): path: str - metadata: zmp.ArrayMetadataV3 + metadata: zmp.ZarrV3ArrayMetadata Static type checkers see each field type as its core model class, so -`manifest.metadata` is an `ArrayMetadataModelV3`. +`manifest.metadata` is a `zarr_metadata.model.ZarrV3ArrayMetadata`. """ from __future__ import annotations @@ -32,15 +32,7 @@ class ArrayManifest(BaseModel): from pydantic import BeforeValidator, InstanceOf, PlainSerializer, WithJsonSchema -from zarr_metadata.model import ( - ArrayMetadataModelV2, - ArrayMetadataModelV3, - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV3, - NamedConfigModelV3, -) +from zarr_metadata import model as _model if TYPE_CHECKING: from collections.abc import Callable @@ -64,68 +56,72 @@ def coerce(value: object) -> _M: _DOCUMENT_SCHEMA = {"type": "object"} _FIELD_SCHEMA = {"anyOf": [{"type": "string"}, {"type": "object"}]} -ArrayMetadataV3 = Annotated[ - InstanceOf[ArrayMetadataModelV3], - BeforeValidator(_coerce_to(ArrayMetadataModelV3, ArrayMetadataModelV3.from_json)), - PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV3"}), +ZarrV3ArrayMetadata = Annotated[ + InstanceOf[_model.ZarrV3ArrayMetadata], + BeforeValidator(_coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json)), + PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3ArrayMetadata"}), ] """Field type for a v3 array metadata document (`zarr.json` content).""" -ArrayMetadataV2 = Annotated[ - InstanceOf[ArrayMetadataModelV2], - BeforeValidator(_coerce_to(ArrayMetadataModelV2, ArrayMetadataModelV2.from_json)), - PlainSerializer(ArrayMetadataModelV2.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV2"}), +ZarrV2ArrayMetadata = Annotated[ + InstanceOf[_model.ZarrV2ArrayMetadata], + BeforeValidator(_coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json)), + PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2ArrayMetadata"}), ] """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" -GroupMetadataV3 = Annotated[ - InstanceOf[GroupMetadataModelV3], - BeforeValidator(_coerce_to(GroupMetadataModelV3, GroupMetadataModelV3.from_json)), - PlainSerializer(GroupMetadataModelV3.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV3"}), +ZarrV3GroupMetadata = Annotated[ + InstanceOf[_model.ZarrV3GroupMetadata], + BeforeValidator(_coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json)), + PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3GroupMetadata"}), ] """Field type for a v3 group metadata document (`zarr.json` content).""" -GroupMetadataV2 = Annotated[ - InstanceOf[GroupMetadataModelV2], - BeforeValidator(_coerce_to(GroupMetadataModelV2, GroupMetadataModelV2.from_json)), - PlainSerializer(GroupMetadataModelV2.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV2"}), +ZarrV2GroupMetadata = Annotated[ + InstanceOf[_model.ZarrV2GroupMetadata], + BeforeValidator(_coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json)), + PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2GroupMetadata"}), ] """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" -ConsolidatedMetadataV3 = Annotated[ - InstanceOf[ConsolidatedMetadataModelV3], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3, ConsolidatedMetadataModelV3.from_json)), - PlainSerializer(ConsolidatedMetadataModelV3.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV3"}), +ZarrV3ConsolidatedMetadata = Annotated[ + InstanceOf[_model.ZarrV3ConsolidatedMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json) + ), + PlainSerializer(_model.ZarrV3ConsolidatedMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3ConsolidatedMetadata"}), ] """Field type for v3 inline consolidated metadata.""" -ConsolidatedMetadataV2 = Annotated[ - InstanceOf[ConsolidatedMetadataModelV2], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2, ConsolidatedMetadataModelV2.from_json)), - PlainSerializer(ConsolidatedMetadataModelV2.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV2"}), +ZarrV2ConsolidatedMetadata = Annotated[ + InstanceOf[_model.ZarrV2ConsolidatedMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json) + ), + PlainSerializer(_model.ZarrV2ConsolidatedMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2ConsolidatedMetadata"}), ] """Field type for a v2 `.zmetadata` document.""" -MetadataFieldV3 = Annotated[ - InstanceOf[NamedConfigModelV3], - BeforeValidator(_coerce_to(NamedConfigModelV3, NamedConfigModelV3.from_json)), - PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), - WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), +ZarrV3MetadataField = Annotated[ + InstanceOf[_model.ZarrV3NamedConfig], + BeforeValidator(_coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json)), + PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=dict), + WithJsonSchema(_FIELD_SCHEMA | {"title": "ZarrV3MetadataField"}), ] """Field type for one v3 metadata field (bare name string or name + configuration).""" __all__ = [ - "ArrayMetadataV2", - "ArrayMetadataV3", - "ConsolidatedMetadataV2", - "ConsolidatedMetadataV3", - "GroupMetadataV2", - "GroupMetadataV3", - "MetadataFieldV3", + "ZarrV2ArrayMetadata", + "ZarrV2ConsolidatedMetadata", + "ZarrV2GroupMetadata", + "ZarrV3ArrayMetadata", + "ZarrV3ConsolidatedMetadata", + "ZarrV3GroupMetadata", + "ZarrV3MetadataField", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py index 4e9a76125b..62dd8168ea 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py @@ -1,26 +1,26 @@ """Zarr v2 metadata types.""" from zarr_metadata.v2.array import ( - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayOrderV2, - DataTypeMetadataV2, ZArrayMetadata, + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, ) from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.codec import CodecMetadataV2 -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2, ZGroupMetadata +from zarr_metadata.v2.codec import ZarrV2CodecMetadata +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZGroupMetadata __all__ = [ - "ArrayDimensionSeparatorV2", - "ArrayMetadataV2", - "ArrayOrderV2", - "CodecMetadataV2", - "ConsolidatedMetadataV2", - "DataTypeMetadataV2", - "GroupMetadataV2", "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayOrder", + "ZarrV2CodecMetadata", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2DataTypeMetadata", + "ZarrV2GroupMetadataJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 999c341dc7..741f37acba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -6,9 +6,9 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v2.codec import CodecMetadataV2 +from zarr_metadata.v2.codec import ZarrV2CodecMetadata -DataTypeMetadataV2 = str | tuple[tuple[str, str] | tuple[str, str, tuple[int, ...]], ...] +ZarrV2DataTypeMetadata = str | tuple[tuple[str, str] | tuple[str, str, tuple[int, ...]], ...] """The v2 dtype representation. Either a numpy-style dtype string (e.g. `"CodecConfiguration`, etc., import directly from the leaf submodule. For the field-level "any codec entry" alias (used in array metadata's -`codecs` list and in sharding's inner pipelines), import `MetadataV3` +`codecs` list and in sharding's inner pipelines), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 7e9b071669..96c39e5916 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -9,7 +9,7 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON CAST_VALUE_CODEC_NAME: Final = "cast_value" """The `name` field value of the `cast_value` codec.""" @@ -71,7 +71,7 @@ class CastValueCodecConfiguration(TypedDict): bare-string primitive name or a `{name, configuration}` envelope. """ - data_type: MetadataV3 + data_type: ZarrV3MetadataFieldJSON rounding: NotRequired[CastRoundingMode] out_of_range: NotRequired[CastOutOfRangeMode] scalar_map: NotRequired[ScalarMap] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a1488f7c30..a8c9247ec4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -8,7 +8,7 @@ from typing_extensions import TypedDict -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" """The `name` field value of the `sharding_indexed` codec.""" @@ -40,8 +40,8 @@ class ShardingIndexedCodecConfiguration(TypedDict): """ chunk_shape: tuple[int, ...] - codecs: tuple[MetadataV3, ...] - index_codecs: tuple[MetadataV3, ...] + codecs: tuple[ZarrV3MetadataFieldJSON, ...] + index_codecs: tuple[ZarrV3MetadataFieldJSON, ...] index_location: NotRequired[ShardingIndexLocation] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index 486a0897a5..5419049bad 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -10,7 +10,7 @@ rules. Under the strict Zarr v3.1 reading, every extension field must also include a `name: str` key, which would make this shape — and every real-world consolidated metadata document in the wild — out of spec. -See `ExtensionFieldV3` and +See `ZarrV3ExtensionField` and https://github.com/zarr-developers/zarr-specs/issues/371 for the ongoing discussion. """ @@ -20,11 +20,11 @@ from typing_extensions import TypedDict -from zarr_metadata.v3.array import ArrayMetadataV3 -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -class ConsolidatedMetadataV3(TypedDict): +class ZarrV3ConsolidatedMetadataJSON(TypedDict): """ Inline consolidated metadata embedded in a v3 group. @@ -35,9 +35,9 @@ class ConsolidatedMetadataV3(TypedDict): kind: Literal["inline"] must_understand: Literal[False] - metadata: Mapping[str, ArrayMetadataV3 | GroupMetadataV3] + metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] __all__ = [ - "ConsolidatedMetadataV3", + "ZarrV3ConsolidatedMetadataJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 5291e5c309..b1b6b50308 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -10,7 +10,7 @@ from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON STRUCT_DATA_TYPE_NAME: Final = "struct" """The `name` field value of the `struct` data type.""" @@ -33,7 +33,7 @@ class StructField(TypedDict): """ name: ReadOnly[str] - data_type: ReadOnly[MetadataV3] + data_type: ReadOnly[ZarrV3MetadataFieldJSON] class StructConfiguration(TypedDict): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index be990b1ae7..029a530e6a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -9,14 +9,14 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3.array import ExtensionFieldV3 +from zarr_metadata.v3.array import ZarrV3ExtensionField -class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): +class ZarrV3GroupMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ Zarr v3 group metadata document (the `zarr.json` content for a group). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys are permitted if they conform to `ZarrV3ExtensionField`. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#group-metadata """ @@ -26,11 +26,11 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): +class ZarrV3GroupMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3ExtensionField): """ - Partial form of `GroupMetadataV3`: every field is `NotRequired`. + Partial form of `ZarrV3GroupMetadataJSON`: every field is `NotRequired`. - Field annotations and `extra_items=` mirror `GroupMetadataV3` exactly. + Field annotations and `extra_items=` mirror `ZarrV3GroupMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. @@ -40,12 +40,12 @@ class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV into a complete document elsewhere. The `NotRequired[...]` wrapper on `attributes` is intentional: keeping it - preserves byte-identical `__annotations__` with `GroupMetadataV3` so the + preserves byte-identical `__annotations__` with `ZarrV3GroupMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing that field (PEP 655 explicitly permits `NotRequired` inside `total=False`). - Drift between this type and `GroupMetadataV3` is prevented by + Drift between this type and `ZarrV3GroupMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ @@ -55,6 +55,6 @@ class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV __all__ = [ - "GroupMetadataV3", - "GroupMetadataV3Partial", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", ] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index ff27923635..6e5029a8fd 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -14,14 +14,14 @@ ARRAY_METADATA_REQUIRED_KEYS_V3, ARRAY_METADATA_STANDARD_KEYS_V3, UNSET, - ArrayMetadataModelV2, - ArrayMetadataModelV2Partial, - ArrayMetadataModelV3, - ArrayMetadataModelV3Partial, - MetadataFieldModelV3, MetadataValidationError, - NamedConfigModelV3, ValidationProblem, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, is_array_metadata_v2, is_array_metadata_v3, is_json, @@ -39,7 +39,7 @@ if TYPE_CHECKING: from zarr_metadata._common import JSONValue - from zarr_metadata.v2 import CodecMetadataV2 + from zarr_metadata.v2 import ZarrV2CodecMetadata # --- public exports -------------------------------------------------------- @@ -89,11 +89,11 @@ def test_expect_expectfail_smoke() -> None: def test_v3_from_json_error_lists_all_problems() -> None: """A malformed v3 document surfaces every problem via MetadataValidationError.problems.""" - doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) del doc["shape"] doc["data_type"] = 5 with pytest.raises(MetadataValidationError) as exc_info: - ArrayMetadataModelV3.from_json(doc) + ZarrV3ArrayMetadata.from_json(doc) locs = {p.loc for p in exc_info.value.problems} assert ("shape",) in locs assert ("data_type",) in locs @@ -117,21 +117,21 @@ def test_string_nan_fill_value_roundtrips() -> None: # unlike a raw float('nan') (which is an invalid fill_value the caller must # not pass). """A string 'NaN' fill_value round-trips cleanly (non-finite floats are the caller's responsibility).""" - m = ArrayMetadataModelV3.create_default(fill_value="NaN") - assert ArrayMetadataModelV3.from_json(m.to_json()) == m - assert ArrayMetadataModelV3.from_json(m.to_json()).fill_value == "NaN" + m = ZarrV3ArrayMetadata.create_default(fill_value="NaN") + assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m + assert ZarrV3ArrayMetadata.from_json(m.to_json()).fill_value == "NaN" -# --- NamedConfigModelV3.to_json ------------------------------------------------ +# --- ZarrV3NamedConfig.to_json ------------------------------------------------ ZARR_TO_JSON_CASES = [ Expect( - NamedConfigModelV3(name="regular", configuration={"chunk_shape": [1]}), + ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": [1]}), {"name": "regular", "configuration": {"chunk_shape": [1]}}, id="with-configuration", ), Expect( - NamedConfigModelV3(name="bytes", configuration={}), + ZarrV3NamedConfig(name="bytes", configuration={}), {"name": "bytes", "configuration": {}}, id="without-configuration", ), @@ -139,32 +139,32 @@ def test_string_nan_fill_value_roundtrips() -> None: @pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, dict[str, object]]) -> None: - """NamedConfigModelV3.to_json emits the canonical object form.""" +def test_zarr_metadata_v3_to_json(case: Expect[ZarrV3NamedConfig, dict[str, object]]) -> None: + """ZarrV3NamedConfig.to_json emits the canonical object form.""" assert case.input.to_json() == case.output -# --- NamedConfigModelV3.from_json ----------------------------------------------- +# --- ZarrV3NamedConfig.from_json ----------------------------------------------- ZARR_FROM_JSON_CASES = [ - Expect("bytes", NamedConfigModelV3(name="bytes", configuration={}), id="bare-string"), + Expect("bytes", ZarrV3NamedConfig(name="bytes", configuration={}), id="bare-string"), Expect( {"name": "regular", "configuration": {"chunk_shape": [1]}}, - NamedConfigModelV3(name="regular", configuration={"chunk_shape": (1,)}), + ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (1,)}), id="object-with-config", ), Expect( {"name": "bytes"}, - NamedConfigModelV3(name="bytes", configuration={}), + ZarrV3NamedConfig(name="bytes", configuration={}), id="object-without-config", ), ] @pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> None: - """NamedConfigModelV3.from_json parses both the bare-string and object forms.""" - assert NamedConfigModelV3.from_json(case.input) == case.output +def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrV3NamedConfig]) -> None: + """ZarrV3NamedConfig.from_json parses both the bare-string and object forms.""" + assert ZarrV3NamedConfig.from_json(case.input) == case.output # --- V3 baseline ----------------------------------------------------------- @@ -173,8 +173,8 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> def test_v3_to_json_emits_canonical_document() -> None: """V3 to_json emits exactly the expected document (which covers every spec-required key by construction).""" - out = ArrayMetadataModelV3.create_default( - shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) + out = ZarrV3ArrayMetadata.create_default( + shape=(10,), data_type=ZarrV3NamedConfig(name="int32", configuration={}) ).to_json() assert out == { "zarr_format": 3, @@ -191,14 +191,14 @@ def test_v3_to_json_emits_canonical_document() -> None: def test_v3_dimension_names_included_when_present() -> None: """V3 to_json includes dimension_names when they are set.""" out: dict[str, object] = dict( - ArrayMetadataModelV3.create_default(dimension_names=("x",)).to_json() + ZarrV3ArrayMetadata.create_default(dimension_names=("x",)).to_json() ) assert out["dimension_names"] == ("x",) def test_v3_dimension_names_omitted_when_none() -> None: """V3 to_json omits dimension_names when they are UNSET.""" - out = ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() + out = ZarrV3ArrayMetadata.create_default(dimension_names=UNSET).to_json() assert "dimension_names" not in out @@ -213,7 +213,7 @@ def test_v3_attributes_included_when_dimension_names_is_none() -> None: dimension names. """ out: dict[str, object] = dict( - ArrayMetadataModelV3.create_default( + ZarrV3ArrayMetadata.create_default( dimension_names=UNSET, attributes={"foo": "bar"} ).to_json() ) @@ -229,16 +229,16 @@ def test_v3_single_storage_transformer_included() -> None: Regression: the guard used ``> 1`` instead of ``> 0``, dropping a lone storage transformer. """ - st = NamedConfigModelV3(name="some_transformer", configuration={}) + st = ZarrV3NamedConfig(name="some_transformer", configuration={}) out: dict[str, object] = dict( - ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() + ZarrV3ArrayMetadata.create_default(storage_transformers=(st,)).to_json() ) assert out["storage_transformers"] == ({"name": "some_transformer", "configuration": {}},) def test_v3_no_storage_transformers_omitted() -> None: """V3 to_json omits storage_transformers when empty.""" - out = ArrayMetadataModelV3.create_default(storage_transformers=()).to_json() + out = ZarrV3ArrayMetadata.create_default(storage_transformers=()).to_json() assert "storage_transformers" not in out @@ -247,7 +247,7 @@ def test_v3_no_storage_transformers_omitted() -> None: def test_v3_extra_fields_merged() -> None: """V3 to_json merges extra_fields into the top-level document.""" - out = ArrayMetadataModelV3.create_default( + out = ZarrV3ArrayMetadata.create_default( extra_fields={"my_ext": {"must_understand": False}} ).to_json() assert out["my_ext"] == {"must_understand": False} @@ -256,7 +256,7 @@ def test_v3_extra_fields_merged() -> None: def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: """Constructing a V3 model with an extra field that collides with a standard key is rejected.""" with pytest.raises(ValueError): - ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + ZarrV3ArrayMetadata.create_default(extra_fields={"shape": {"must_understand": False}}) # --- V3 key/value ---------------------------------------------------------- @@ -264,7 +264,7 @@ def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: """V3 to_key_value produces valid JSON bytes under the zarr.json key.""" - kv = ArrayMetadataModelV3.create_default(attributes={"a": 1}).to_key_value() + kv = ZarrV3ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() assert set(kv) == {"zarr.json"} parsed = json.loads(kv["zarr.json"].decode("utf-8")) assert parsed["zarr_format"] == 3 @@ -298,29 +298,29 @@ def test_standard_keys_contains_known_fields_and_excludes_extensions() -> None: def test_v3_create_default_is_valid_empty_array() -> None: """V3 create_default builds a structurally valid empty array that round-trips.""" - m = ArrayMetadataModelV3.create_default() + m = ZarrV3ArrayMetadata.create_default() assert m.shape == () - assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) + assert m.data_type == ZarrV3NamedConfig(name="uint8", configuration={}) assert m.fill_value == 0 assert m.attributes == {} assert m.extra_fields == {} # the default document is structurally valid and round-trips assert validate_array_metadata_v3(m.to_json()) == [] - assert ArrayMetadataModelV3.from_json(m.to_json()) == m + assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m def test_v3_create_default_applies_overrides() -> None: """V3 create_default applies keyword overrides over the defaults.""" - m = ArrayMetadataModelV3.create_default(shape=(4, 4), attributes={"a": 1}) + m = ZarrV3ArrayMetadata.create_default(shape=(4, 4), attributes={"a": 1}) assert m.shape == (4, 4) assert m.attributes == {"a": 1} # un-overridden fields keep their defaults - assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) + assert m.data_type == ZarrV3NamedConfig(name="uint8", configuration={}) def test_v2_create_default_is_valid_empty_array() -> None: """V2 create_default builds a structurally valid empty array that round-trips.""" - m = ArrayMetadataModelV2.create_default() + m = ZarrV2ArrayMetadata.create_default() assert m.shape == () assert m.chunks == () assert m.fill_value == 0 @@ -328,12 +328,12 @@ def test_v2_create_default_is_valid_empty_array() -> None: assert m.filters is None assert m.attributes is UNSET assert validate_array_metadata_v2(m.to_json()) == [] - assert ArrayMetadataModelV2.from_json(m.to_json()) == m + assert ZarrV2ArrayMetadata.from_json(m.to_json()) == m def test_v2_create_default_applies_overrides() -> None: """V2 create_default applies keyword overrides over the defaults.""" - m = ArrayMetadataModelV2.create_default(shape=(8,), attributes={"k": "v"}) + m = ZarrV2ArrayMetadata.create_default(shape=(8,), attributes={"k": "v"}) assert m.shape == (8,) assert m.attributes == {"k": "v"} assert m.dtype == "|u1" # default dtype unchanged @@ -344,14 +344,14 @@ def test_v2_create_default_applies_overrides() -> None: # Cluster 3: update same-shape pairs across versions — parametrized UPDATE_NEW_INSTANCE_PARAMS = [ - pytest.param(ArrayMetadataModelV3, id="v3"), - pytest.param(ArrayMetadataModelV2, id="v2"), + pytest.param(ZarrV3ArrayMetadata, id="v3"), + pytest.param(ZarrV2ArrayMetadata, id="v2"), ] @pytest.mark.parametrize("model_cls", UPDATE_NEW_INSTANCE_PARAMS) def test_update_returns_new_instance( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], ) -> None: """update returns a new instance with the field replaced, leaving the original unchanged.""" base = model_cls.create_default(shape=(10,)) @@ -362,14 +362,14 @@ def test_update_returns_new_instance( UPDATE_NO_ARGS_PARAMS = [ - pytest.param(ArrayMetadataModelV3, id="v3"), - pytest.param(ArrayMetadataModelV2, id="v2"), + pytest.param(ZarrV3ArrayMetadata, id="v3"), + pytest.param(ZarrV2ArrayMetadata, id="v2"), ] @pytest.mark.parametrize("model_cls", UPDATE_NO_ARGS_PARAMS) def test_update_no_args_returns_equal_model( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], ) -> None: """update with no arguments returns a model equal to the original.""" base = model_cls.create_default() @@ -381,14 +381,14 @@ def test_update_no_args_returns_equal_model( def test_update_can_replace_extra_fields() -> None: """update can replace the extra_fields mapping.""" - base = ArrayMetadataModelV3.create_default(extra_fields={}) + base = ZarrV3ArrayMetadata.create_default(extra_fields={}) updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) assert updated.extra_fields == {"my_ext": {"must_understand": False}} def test_update_replaces_extra_fields_rather_than_merging() -> None: """update replaces extra_fields wholesale rather than merging.""" - base = ArrayMetadataModelV3.create_default(extra_fields={"a": {"must_understand": False}}) + base = ZarrV3ArrayMetadata.create_default(extra_fields={"a": {"must_understand": False}}) updated = base.update(extra_fields={"b": {"must_understand": True}}) assert updated.extra_fields == {"b": {"must_understand": True}} @@ -397,10 +397,10 @@ def test_partial_keys_match_settable_model_fields() -> None: """The partial TypedDict must list exactly the constructor-settable fields. Guards against drift: adding/removing a settable field on the model - without updating ``ArrayMetadataModelV3Partial`` fails here. + without updating ``ZarrV3ArrayMetadataPartial`` fails here. """ - settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV3) if f.init} - assert set(ArrayMetadataModelV3Partial.__annotations__) == settable + settable = {f.name for f in dataclasses.fields(ZarrV3ArrayMetadata) if f.init} + assert set(ZarrV3ArrayMetadataPartial.__annotations__) == settable # --- V2 model -------------------------------------------------------------- @@ -408,13 +408,13 @@ def test_partial_keys_match_settable_model_fields() -> None: def test_v2_partial_keys_match_settable_model_fields() -> None: """The v2 partial TypedDict must list exactly the settable fields.""" - settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV2) if f.init} - assert set(ArrayMetadataModelV2Partial.__annotations__) == settable + settable = {f.name for f in dataclasses.fields(ZarrV2ArrayMetadata) if f.init} + assert set(ZarrV2ArrayMetadataPartial.__annotations__) == settable def test_v2_to_key_value_splits_zarray_and_zattrs() -> None: """V2 to_key_value splits the document into .zarray and .zattrs.""" - kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() assert set(kv) == {".zarray", ".zattrs"} zarray = json.loads(kv[".zarray"].decode("utf-8")) zattrs = json.loads(kv[".zattrs"].decode("utf-8")) @@ -426,19 +426,17 @@ def test_v2_zarray_excludes_attributes() -> None: """The on-disk ``.zarray`` document must not contain user attributes. In v2, attributes live only in the sibling ``.zattrs`` file. The bundled - ``ArrayMetadataV2`` / ``to_json()`` carry attributes for convenience, but + ``ZarrV2ArrayMetadataJSON`` / ``to_json()`` carry attributes for convenience, but ``to_key_value()`` must split them out. """ - kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() zarray = json.loads(kv[".zarray"].decode("utf-8")) assert "attributes" not in zarray def test_v2_to_json_still_includes_attributes() -> None: """``to_json()`` is the bundled in-memory form and keeps attributes.""" - out: dict[str, object] = dict( - ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() - ) + out: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json()) assert out["attributes"] == {"a": 1} @@ -464,29 +462,29 @@ def test_arrays_to_tuples(case: Expect[object, object]) -> None: assert arrays_to_tuples(case.input) == case.output -# --- ArrayMetadataModelV3.from_json ---------------------------------------- +# --- ZarrV3ArrayMetadata.from_json ---------------------------------------- def test_v3_from_json_reconstructs_required_fields() -> None: """V3 from_json reconstructs the required fields from a document.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( shape=(7,), attributes={"a": 1}, - data_type=NamedConfigModelV3(name="int32", configuration={}), + data_type=ZarrV3NamedConfig(name="int32", configuration={}), ).to_json() - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) assert model.shape == (7,) - assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) + assert model.data_type == ZarrV3NamedConfig(name="int32", configuration={}) assert model.attributes == {"a": 1} def test_v3_from_json_defaults_for_omitted_optionals() -> None: """V3 from_json supplies defaults for omitted optional fields.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( attributes={}, storage_transformers=(), dimension_names=UNSET ).to_json() # to_json omits these entirely; from_json must restore defaults - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) assert model.attributes == {} assert model.storage_transformers == () assert model.dimension_names is UNSET @@ -494,36 +492,36 @@ def test_v3_from_json_defaults_for_omitted_optionals() -> None: def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: """V3 from_json routes unknown top-level keys into extra_fields.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( extra_fields={"my_ext": {"must_understand": False}} ).to_json() - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) assert model.extra_fields == {"my_ext": {"must_understand": False}} def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: """V3 from_json keeps standard keys out of extra_fields.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( shape=(10,), attributes={"a": 1}, dimension_names=("x",) ).to_json() - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) assert model.extra_fields == {} def test_v3_from_json_nested_arrays_in_attributes_become_tuples() -> None: """V3 from_json converts nested arrays in attributes into tuples.""" - doc = ArrayMetadataModelV3.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() - model = ArrayMetadataModelV3.from_json(doc) + doc = ZarrV3ArrayMetadata.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) assert model.attributes == {"scale": ((1, 2), (3, 4))} -# --- ArrayMetadataModelV3.from_key_value ---------------------------------- +# --- ZarrV3ArrayMetadata.from_key_value ---------------------------------- def test_v3_from_key_value_parses_zarr_json() -> None: """V3 from_key_value parses the zarr.json entry into a model.""" - kv = ArrayMetadataModelV3.create_default(shape=(3,)).to_key_value() - model = ArrayMetadataModelV3.from_key_value(kv) + kv = ZarrV3ArrayMetadata.create_default(shape=(3,)).to_key_value() + model = ZarrV3ArrayMetadata.from_key_value(kv) assert model.shape == (3,) @@ -531,12 +529,12 @@ def test_v3_from_key_value_parses_zarr_json() -> None: FROM_KEY_VALUE_MISSING_PARAMS = [ pytest.param( - ArrayMetadataModelV3, + ZarrV3ArrayMetadata, ExpectFail({}, MetadataValidationError, id="v3-missing-zarr-json", msg="missing store key"), id="v3-missing-zarr-json", ), pytest.param( - ArrayMetadataModelV2, + ZarrV2ArrayMetadata, ExpectFail({}, MetadataValidationError, id="v2-missing-zarray", msg="missing store key"), id="v2-missing-zarray", ), @@ -545,7 +543,7 @@ def test_v3_from_key_value_parses_zarr_json() -> None: @pytest.mark.parametrize(("model_cls", "case"), FROM_KEY_VALUE_MISSING_PARAMS) def test_from_key_value_missing_key_raises( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], case: ExpectFail[dict[str, bytes]], ) -> None: """from_key_value raises MetadataValidationError when the required store key is absent.""" @@ -557,19 +555,19 @@ def test_from_key_value_missing_key_raises( ROUNDTRIP_MODEL_JSON_PARAMS = [ pytest.param( - ArrayMetadataModelV3, - ArrayMetadataModelV3.create_default( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default( shape=(10,), attributes={"a": 1}, dimension_names=("x",), - storage_transformers=(NamedConfigModelV3(name="t", configuration={}),), + storage_transformers=(ZarrV3NamedConfig(name="t", configuration={}),), extra_fields={"ext": {"must_understand": False}}, ), id="v3-full", ), pytest.param( - ArrayMetadataModelV3, - ArrayMetadataModelV3.create_default( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default( attributes={}, dimension_names=UNSET, storage_transformers=(), @@ -578,8 +576,8 @@ def test_from_key_value_missing_key_raises( id="v3-empty-optionals", ), pytest.param( - ArrayMetadataModelV2, - ArrayMetadataModelV2.create_default(attributes={"a": 1}, filters=None, compressor=None), + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, filters=None, compressor=None), id="v2-basic", ), ] @@ -587,8 +585,8 @@ def test_from_key_value_missing_key_raises( @pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_MODEL_JSON_PARAMS) def test_roundtrip_model_json_model( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], - model: ArrayMetadataModelV3 | ArrayMetadataModelV2, + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, ) -> None: """A model round-trips through to_json/from_json back to an equal model.""" assert model_cls.from_json(model.to_json()) == model @@ -598,13 +596,13 @@ def test_roundtrip_model_json_model( ROUNDTRIP_KEY_VALUE_PARAMS = [ pytest.param( - ArrayMetadataModelV3, - ArrayMetadataModelV3.create_default(attributes={"a": 1}), + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default(attributes={"a": 1}), id="v3", ), pytest.param( - ArrayMetadataModelV2, - ArrayMetadataModelV2.create_default(attributes={"a": 1}), + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadata.create_default(attributes={"a": 1}), id="v2", ), ] @@ -612,8 +610,8 @@ def test_roundtrip_model_json_model( @pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_KEY_VALUE_PARAMS) def test_roundtrip_via_key_value( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], - model: ArrayMetadataModelV3 | ArrayMetadataModelV2, + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, ) -> None: """A model round-trips through to_key_value/from_key_value back to an equal model.""" assert model_cls.from_key_value(model.to_key_value()) == model @@ -624,48 +622,46 @@ def test_roundtrip_via_key_value( def test_v3_roundtrip_json_model_json() -> None: """A v3 document round-trips through from_json/to_json back to an equal document.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( shape=(10,), attributes={"a": 1}, dimension_names=("x",) ).to_json() - assert ArrayMetadataModelV3.from_json(doc).to_json() == doc + assert ZarrV3ArrayMetadata.from_json(doc).to_json() == doc def test_v2_roundtrip_json_model_json() -> None: """A v2 document round-trips through from_json/to_json back to an equal document.""" - doc = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() - assert ArrayMetadataModelV2.from_json(doc).to_json() == doc + doc = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json() + assert ZarrV2ArrayMetadata.from_json(doc).to_json() == doc def test_v3_parser_accepts_bare_string_data_type() -> None: """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" - doc = ArrayMetadataModelV3.create_default().to_json() + doc = ZarrV3ArrayMetadata.create_default().to_json() doc["data_type"] = "int32" # bare-string form, not canonical object form - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) # parses correctly, re-serializes to canonical object form - assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) + assert model.data_type == ZarrV3NamedConfig(name="int32", configuration={}) assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} def test_v2_roundtrip_with_compressor_and_filters() -> None: # Non-None compressor/filters must round-trip; extra assertion on .compressor. """A v2 model with non-None compressor and filters round-trips.""" - compressor: CodecMetadataV2 = {"id": "blosc", "clevel": 5} - filters: tuple[CodecMetadataV2, ...] = ({"id": "delta"},) - m = ArrayMetadataModelV2.create_default(compressor=compressor, filters=filters) - restored = ArrayMetadataModelV2.from_json(m.to_json()) + compressor: ZarrV2CodecMetadata = {"id": "blosc", "clevel": 5} + filters: tuple[ZarrV2CodecMetadata, ...] = ({"id": "delta"},) + m = ZarrV2ArrayMetadata.create_default(compressor=compressor, filters=filters) + restored = ZarrV2ArrayMetadata.from_json(m.to_json()) assert restored == m assert restored.compressor == {"id": "blosc", "clevel": 5} -# --- ArrayMetadataModelV2.from_json ---------------------------------------- +# --- ZarrV2ArrayMetadata.from_json ---------------------------------------- def test_v2_from_json_reconstructs_fields() -> None: """V2 from_json reconstructs the fields from a document.""" - doc = ArrayMetadataModelV2.create_default( - shape=(4,), attributes={"a": 1}, dtype=" None: def test_v2_from_json_attributes_absent_is_unset() -> None: """V2 from_json reads an absent attributes key as UNSET, distinct from an explicit empty mapping.""" - absent = ArrayMetadataModelV2.from_json(ArrayMetadataModelV2.create_default().to_json()) - explicit = ArrayMetadataModelV2.from_json( - ArrayMetadataModelV2.create_default(attributes={}).to_json() + absent = ZarrV2ArrayMetadata.from_json(ZarrV2ArrayMetadata.create_default().to_json()) + explicit = ZarrV2ArrayMetadata.from_json( + ZarrV2ArrayMetadata.create_default(attributes={}).to_json() ) assert absent.attributes is UNSET assert explicit.attributes == {} assert absent != explicit -# --- ArrayMetadataModelV2.from_key_value -------------------------------- +# --- ZarrV2ArrayMetadata.from_key_value -------------------------------- def test_v2_from_key_value_remerges_zattrs() -> None: """V2 from_key_value re-merges .zattrs back into attributes.""" - kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() - model = ArrayMetadataModelV2.from_key_value(kv) + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() + model = ZarrV2ArrayMetadata.from_key_value(kv) assert model.attributes == {"a": 1} assert model.shape == (10,) @@ -698,13 +694,13 @@ def test_v2_zattrs_presence_round_trips() -> None: """The .zattrs file's presence is part of the store: an absent file reads as UNSET and emits no .zattrs; an explicit empty file reads as {} and emits .zattrs — the two stores stay distinct through a round-trip.""" - explicit_kv = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) + explicit_kv = dict(ZarrV2ArrayMetadata.create_default(attributes={}).to_key_value()) assert ".zattrs" in explicit_kv absent_kv = dict(explicit_kv) del absent_kv[".zattrs"] - absent = ArrayMetadataModelV2.from_key_value(absent_kv) - explicit = ArrayMetadataModelV2.from_key_value(explicit_kv) + absent = ZarrV2ArrayMetadata.from_key_value(absent_kv) + explicit = ZarrV2ArrayMetadata.from_key_value(explicit_kv) assert absent.attributes is UNSET assert explicit.attributes == {} assert ".zattrs" not in absent.to_key_value() @@ -713,8 +709,8 @@ def test_v2_zattrs_presence_round_trips() -> None: def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: """V2 from_json converts nested arrays in attributes into tuples.""" - doc = ArrayMetadataModelV2.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() - model = ArrayMetadataModelV2.from_json(doc) + doc = ZarrV2ArrayMetadata.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() + model = ZarrV2ArrayMetadata.from_json(doc) assert model.attributes == {"axes": ((0, 1), (2, 3))} @@ -829,12 +825,12 @@ def test_parse_metadata_field_v3( # invalid cases (a subset check, so accumulation of OTHER problems is allowed). -def _build_v3(**overrides: Unpack[ArrayMetadataModelV3Partial]) -> dict[str, object]: - return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) +def _build_v3(**overrides: Unpack[ZarrV3ArrayMetadataPartial]) -> dict[str, object]: + return dict(ZarrV3ArrayMetadata.create_default(**overrides).to_json()) -def _build_v2(**overrides: Unpack[ArrayMetadataModelV2Partial]) -> dict[str, object]: - return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) +def _build_v2(**overrides: Unpack[ZarrV2ArrayMetadataPartial]) -> dict[str, object]: + return dict(ZarrV2ArrayMetadata.create_default(**overrides).to_json()) def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: @@ -971,22 +967,22 @@ def test_array_metadata_guards( FROM_JSON_REJECT_PARAMS = [ pytest.param( - ArrayMetadataModelV3, + ZarrV3ArrayMetadata, ExpectFail(lambda: {"zarr_format": 3}, MetadataValidationError, id="x"), id="v3-missing-required", ), pytest.param( - ArrayMetadataModelV3, + ZarrV3ArrayMetadata, ExpectFail(_mutate(_build_v3, _set("data_type", 5)), MetadataValidationError, id="x"), id="v3-bad-field-type", ), pytest.param( - ArrayMetadataModelV2, + ZarrV2ArrayMetadata, ExpectFail(lambda: {"zarr_format": 2}, MetadataValidationError, id="x"), id="v2-missing-required", ), pytest.param( - NamedConfigModelV3, + ZarrV3NamedConfig, ExpectFail(lambda: 5, MetadataValidationError, id="x"), id="zarr-metadata-bad-input", ), @@ -995,7 +991,7 @@ def test_array_metadata_guards( @pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) def test_from_json_rejects_malformed( - model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | NamedConfigModelV3], + model: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata | ZarrV3NamedConfig], case: ExpectFail[Callable[[], object]], ) -> None: """from_json raises MetadataValidationError on a malformed document.""" @@ -1056,7 +1052,7 @@ def test_prefix_prepends_loc_head() -> None: def test_v2_dtype_must_be_string_or_records() -> None: """A non-string, non-records v2 dtype is rejected with an invalid_type problem.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": 42} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": 42} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")] @@ -1064,34 +1060,34 @@ def test_v2_dtype_must_be_string_or_records() -> None: def test_v2_structured_dtype_records_accepted() -> None: """A structured v2 dtype (field records, optionally nested/shaped) validates.""" dtype = (("a", " None: """A field record with the wrong arity is rejected.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": (("a",),)} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": (("a",),)} problems = validate_array_metadata_v2(doc) assert [p.loc for p in problems] == [("dtype",)] def test_v2_order_literal_enforced() -> None: """An order other than 'C' or 'F' is rejected with an invalid_value problem.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"order": "Q"} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"order": "Q"} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("order",), "invalid_value")] def test_v2_compressor_must_be_codec_or_none() -> None: """A compressor that is not null or a codec config mapping is rejected.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": "zlib"} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": "zlib"} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("compressor",), "invalid_type")] def test_v2_compressor_requires_string_id() -> None: """A compressor mapping without a string id is rejected.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": {"level": 3}} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": {"level": 3}} problems = validate_array_metadata_v2(doc) assert [p.loc for p in problems] == [("compressor",)] @@ -1099,42 +1095,42 @@ def test_v2_compressor_requires_string_id() -> None: def test_v2_filters_must_be_codec_sequence_or_none() -> None: """Filters that are not null or a sequence of codec configs are rejected.""" for bad in (7, (5,), "gzip"): - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"filters": bad} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"filters": bad} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad def test_v2_dimension_separator_literal_enforced() -> None: """A dimension_separator other than '.' or '/' is rejected.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": "-"} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": "-"} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] def test_v2_zarr_format_literal_enforced() -> None: """A v2 document claiming zarr_format 3 is rejected with an invalid_value problem.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"zarr_format": 3} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 3} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] def test_v3_zarr_format_literal_enforced() -> None: """A v3 document claiming zarr_format 2 is rejected with an invalid_value problem.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"zarr_format": 2} + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 2} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] def test_v3_node_type_literal_enforced() -> None: """A v3 array document claiming node_type 'group' is rejected.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"node_type": "group"} + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"node_type": "group"} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("node_type",), "invalid_value")] def test_missing_key_kind_is_machine_readable() -> None: """A missing required key is distinguishable by kind, without message matching.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) del doc["chunk_key_encoding"] problems = validate_array_metadata_v3(doc) assert problems == [ @@ -1148,14 +1144,14 @@ def test_missing_key_kind_is_machine_readable() -> None: def test_from_key_value_invalid_json_raises_metadata_error() -> None: """Undecodable store bytes raise MetadataValidationError (kind invalid_json), not JSONDecodeError.""" with pytest.raises(MetadataValidationError) as exc_info: - ArrayMetadataModelV3.from_key_value({"zarr.json": b"{not json"}) + ZarrV3ArrayMetadata.from_key_value({"zarr.json": b"{not json"}) assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] def test_from_key_value_missing_key_kind() -> None: """A missing store key surfaces as a missing_key problem at the store-key loc.""" with pytest.raises(MetadataValidationError) as exc_info: - ArrayMetadataModelV2.from_key_value({}) + ZarrV2ArrayMetadata.from_key_value({}) assert exc_info.value.problems == [ ValidationProblem((".zarray",), "missing store key", "missing_key") ] @@ -1164,20 +1160,20 @@ def test_from_key_value_missing_key_kind() -> None: def test_extra_fields_overlap_raises_metadata_error() -> None: """The extra-fields overlap invariant raises MetadataValidationError (a ValueError).""" with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: - ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + ZarrV3ArrayMetadata.create_default(extra_fields={"shape": {"must_understand": False}}) assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] def test_extension_point_fields_annotated_with_role_alias() -> None: - """Extension-point fields are annotated with MetadataFieldModelV3 (the - logical role), not NamedConfigModelV3 (the current serialized form), so a + """Extension-point fields are annotated with ZarrV3MetadataField (the + logical role), not ZarrV3NamedConfig (the current serialized form), so a future widening of the field union does not move annotation sites.""" - assert MetadataFieldModelV3 is NamedConfigModelV3 - annotations = ArrayMetadataModelV3.__annotations__ + assert ZarrV3MetadataField is ZarrV3NamedConfig + annotations = ZarrV3ArrayMetadata.__annotations__ for field_name in ("data_type", "chunk_grid", "chunk_key_encoding"): - assert annotations[field_name] == "MetadataFieldModelV3" + assert annotations[field_name] == "ZarrV3MetadataField" for field_name in ("codecs", "storage_transformers"): - assert annotations[field_name] == "tuple[MetadataFieldModelV3, ...]" + assert annotations[field_name] == "tuple[ZarrV3MetadataField, ...]" # --- Adversarial-probe fixes: documents that used to pass validation --------- @@ -1186,19 +1182,19 @@ def test_extension_point_fields_annotated_with_role_alias() -> None: def test_shape_rejects_json_booleans() -> None: """JSON booleans are not integers: shape/chunks containing true/false are rejected (bool is an int subclass in Python, so isinstance alone passes).""" - v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (True, True)} + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (True, True)} assert [p.loc for p in validate_array_metadata_v3(v3)] == [("shape",)] - v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (True,)} + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (True,)} assert [p.loc for p in validate_array_metadata_v2(v2)] == [("chunks",)] def test_shape_rejects_negative_dimensions() -> None: """Dimension lengths must be non-negative; a negative entry is invalid_value.""" - v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (-1,)} + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (-1,)} assert [(p.loc, p.kind) for p in validate_array_metadata_v3(v3)] == [ (("shape",), "invalid_value") ] - v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (-5,)} + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (-5,)} assert [(p.loc, p.kind) for p in validate_array_metadata_v2(v2)] == [ (("chunks",), "invalid_value") ] @@ -1206,7 +1202,7 @@ def test_shape_rejects_negative_dimensions() -> None: def test_dimension_names_length_must_match_shape() -> None: """dimension_names must have one entry per dimension of shape.""" - doc = dict(ArrayMetadataModelV3.create_default(shape=(10,)).to_json()) | { + doc = dict(ZarrV3ArrayMetadata.create_default(shape=(10,)).to_json()) | { "dimension_names": ("x", "y", "z") } assert [(p.loc, p.kind) for p in validate_array_metadata_v3(doc)] == [ @@ -1217,7 +1213,7 @@ def test_dimension_names_length_must_match_shape() -> None: def test_attributes_values_must_be_json() -> None: """Attribute values are JSON-checked recursively (like fill_value), so a non-serializable value is a validation problem, not a later TypeError.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"attributes": {"a": {1, 2}}} + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"attributes": {"a": {1, 2}}} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("attributes", "a"), "invalid_type")] @@ -1225,7 +1221,7 @@ def test_attributes_values_must_be_json() -> None: def test_configuration_values_must_be_json() -> None: """Configuration values are JSON-checked recursively, so an int-keyed dict cannot pass validation and be silently rewritten by json.dumps.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | { + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | { "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": {1: 2}}} } problems = validate_array_metadata_v3(doc) @@ -1242,7 +1238,7 @@ def test_must_understand_fields_partition() -> None: with must_understand: false, including implicitly-true and non-mapping fields, so a reader can discharge the spec's fail-to-open duty by subtracting the extensions it recognizes.""" - model = ArrayMetadataModelV3.create_default( + model = ZarrV3ArrayMetadata.create_default( extra_fields={ "ext_a": {"name": "a", "must_understand": False}, "ext_b": {"name": "b"}, @@ -1257,7 +1253,7 @@ def test_must_understand_fields_partition() -> None: def test_must_understand_fields_empty_when_all_waived() -> None: """must_understand_fields is empty when every extra field is explicitly waived.""" - model = ArrayMetadataModelV3.create_default( + model = ZarrV3ArrayMetadata.create_default( extra_fields={"ext_a": {"name": "a", "must_understand": False}} ) assert model.must_understand_fields == {} @@ -1268,13 +1264,12 @@ def test_dimension_names_null_field_rejected() -> None: null as an element (an unnamed dimension), never as the field value — "not specified" is spelled by omitting the key. Consumers bridging from an in-memory None sentinel must drop the key, not write null.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"dimension_names": None} + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"dimension_names": None} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] # and the model's own None spelling correctly maps to key absence assert ( - "dimension_names" - not in ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() + "dimension_names" not in ZarrV3ArrayMetadata.create_default(dimension_names=UNSET).to_json() ) @@ -1285,28 +1280,28 @@ def test_v3_create_default_chunk_grid_follows_shape() -> None: """Overriding shape without chunk_grid derives a consistent default grid: one chunk covering the array (chunk_shape == shape), instead of silently keeping the scalar default's 0-d grid.""" - model = ArrayMetadataModelV3.create_default(shape=(100, 100)) - assert model.chunk_grid == NamedConfigModelV3( + model = ZarrV3ArrayMetadata.create_default(shape=(100, 100)) + assert model.chunk_grid == ZarrV3NamedConfig( name="regular", configuration={"chunk_shape": (100, 100)} ) def test_v3_create_default_explicit_chunk_grid_respected() -> None: """An explicit chunk_grid override wins over the shape-derived default.""" - grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) - model = ArrayMetadataModelV3.create_default(shape=(100, 100), chunk_grid=grid) + grid = ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (10, 10)}) + model = ZarrV3ArrayMetadata.create_default(shape=(100, 100), chunk_grid=grid) assert model.chunk_grid == grid def test_v2_create_default_chunks_follow_shape() -> None: """Overriding shape without chunks derives chunks == shape.""" - model = ArrayMetadataModelV2.create_default(shape=(100, 100)) + model = ZarrV2ArrayMetadata.create_default(shape=(100, 100)) assert model.chunks == (100, 100) def test_v2_create_default_explicit_chunks_respected() -> None: """An explicit chunks override wins over the shape-derived default.""" - model = ArrayMetadataModelV2.create_default(shape=(100, 100), chunks=(10, 10)) + model = ZarrV2ArrayMetadata.create_default(shape=(100, 100), chunks=(10, 10)) assert model.chunks == (10, 10) @@ -1315,7 +1310,7 @@ def test_v3_create_default_zero_length_dimensions() -> None: 'The chunk shape elements are non-zero when the corresponding dimensions of the arrays have non-zero length' — the constraint is conditional, so a zero chunk length is permitted exactly where the dimension is empty.""" - model = ArrayMetadataModelV3.create_default(shape=(0, 3)) + model = ZarrV3ArrayMetadata.create_default(shape=(0, 3)) assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) @@ -1324,11 +1319,11 @@ def test_create_default_derivation_is_one_way() -> None: scalar default shape=() untouched: a user-supplied chunk_grid is an extension point taken verbatim, and deriving shape from it would require interpreting grid configurations, which the model layer never does.""" - grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) - v3 = ArrayMetadataModelV3.create_default(chunk_grid=grid) + grid = ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (10, 10)}) + v3 = ZarrV3ArrayMetadata.create_default(chunk_grid=grid) assert v3.shape == () assert v3.chunk_grid == grid - v2 = ArrayMetadataModelV2.create_default(chunks=(10, 10)) + v2 = ZarrV2ArrayMetadata.create_default(chunks=(10, 10)) assert v2.shape == () assert v2.chunks == (10, 10) @@ -1341,9 +1336,9 @@ def test_v2_absent_dimension_separator_means_dot() -> None: '.', not '/': chunk keys of real-world default-separator v2 arrays look like '0.0'. The model normalizes the absent key to an explicit '.' — a semantics-preserving spelling normalization.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) del doc["dimension_separator"] - model = ArrayMetadataModelV2.from_json(doc) + model = ZarrV2ArrayMetadata.from_json(doc) assert model.dimension_separator == "." assert model.to_json()["dimension_separator"] == "." @@ -1353,19 +1348,19 @@ def test_v2_from_key_value_without_separator_means_dot() -> None: dimension_separator key.""" doc = { k: v - for k, v in ArrayMetadataModelV2.create_default().to_json().items() + for k, v in ZarrV2ArrayMetadata.create_default().to_json().items() if k not in ("dimension_separator", "attributes") } import json as _json - model = ArrayMetadataModelV2.from_key_value({".zarray": _json.dumps(doc).encode()}) + model = ZarrV2ArrayMetadata.from_key_value({".zarray": _json.dumps(doc).encode()}) assert model.dimension_separator == "." def test_v2_null_dimension_separator_rejected() -> None: """dimension_separator may be absent, '.', or '/' — never null: the document grammar has no null spelling for this field.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": None} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": None} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] @@ -1376,11 +1371,11 @@ def test_dimension_names_absent_and_all_null_are_distinct() -> None: has a name, which is null; absence says there are no dimension names. The model preserves the distinction (UNSET vs a tuple of Nones), and both spellings round-trip faithfully.""" - absent_doc = dict(ArrayMetadataModelV3.create_default(shape=(2, 3)).to_json()) + absent_doc = dict(ZarrV3ArrayMetadata.create_default(shape=(2, 3)).to_json()) explicit_doc = absent_doc | {"dimension_names": (None, None)} - absent = ArrayMetadataModelV3.from_json(absent_doc) - explicit = ArrayMetadataModelV3.from_json(explicit_doc) + absent = ZarrV3ArrayMetadata.from_json(absent_doc) + explicit = ZarrV3ArrayMetadata.from_json(explicit_doc) assert absent.dimension_names is UNSET assert explicit.dimension_names == (None, None) diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 58b92f5d71..2259a3d8ca 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -6,14 +6,14 @@ import pytest from zarr_metadata.model import UNSET -from zarr_metadata.model._array import ArrayMetadataModelV3 +from zarr_metadata.model._array import ZarrV3ArrayMetadata from zarr_metadata.model._group import ( - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV2Partial, - GroupMetadataModelV3, - GroupMetadataModelV3Partial, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, ) from zarr_metadata.model._validation import ( MetadataValidationError, @@ -23,26 +23,26 @@ validate_group_metadata_v3, ) -# --- GroupMetadataModelV3 --------------------------------------------------- +# --- ZarrV3GroupMetadata --------------------------------------------------- def test_group_v3_roundtrip() -> None: """A v3 group document round-trips through the model unchanged.""" doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} - model = GroupMetadataModelV3.from_json(doc) + model = ZarrV3GroupMetadata.from_json(doc) assert model.to_json() == doc def test_group_v3_omits_empty_attributes() -> None: """to_json omits the attributes key when attributes is empty.""" - model = GroupMetadataModelV3.create_default() + model = ZarrV3GroupMetadata.create_default() assert "attributes" not in model.to_json() def test_group_v3_lists_become_tuples() -> None: """from_json converts JSON arrays in attributes to tuples.""" doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2]}} - model = GroupMetadataModelV3.from_json(doc) + model = ZarrV3GroupMetadata.from_json(doc) assert model.attributes == {"a": (1, 2)} @@ -53,7 +53,7 @@ def test_group_v3_extra_fields_roundtrip() -> None: "node_type": "group", "my_extension": {"name": "thing", "must_understand": False}, } - model = GroupMetadataModelV3.from_json(doc) + model = ZarrV3GroupMetadata.from_json(doc) assert model.extra_fields == {"my_extension": {"name": "thing", "must_understand": False}} assert model.to_json() == doc @@ -61,7 +61,7 @@ def test_group_v3_extra_fields_roundtrip() -> None: def test_group_v3_extra_fields_overlap_rejected() -> None: """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" with pytest.raises(ValueError, match="Extra fields"): - GroupMetadataModelV3( + ZarrV3GroupMetadata( attributes={}, consolidated_metadata=UNSET, extra_fields={"node_type": {"name": "x", "must_understand": False}}, @@ -71,7 +71,7 @@ def test_group_v3_extra_fields_overlap_rejected() -> None: def test_group_v3_consolidated_extra_field_rejected() -> None: """extra_fields may not shadow the consolidated_metadata convention key.""" with pytest.raises(ValueError, match="Extra fields"): - GroupMetadataModelV3( + ZarrV3GroupMetadata( attributes={}, consolidated_metadata=UNSET, extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, @@ -92,38 +92,38 @@ def test_group_v3_bad_attributes() -> None: def test_group_v3_key_value_roundtrip() -> None: """from_key_value(to_key_value()) is the identity for v3 groups.""" - model = GroupMetadataModelV3.create_default(attributes={"a": 1}) - assert GroupMetadataModelV3.from_key_value(model.to_key_value()) == model + model = ZarrV3GroupMetadata.create_default(attributes={"a": 1}) + assert ZarrV3GroupMetadata.from_key_value(model.to_key_value()) == model def test_group_v3_update() -> None: """update replaces the given fields and returns a new instance.""" - base = GroupMetadataModelV3.create_default() + base = ZarrV3GroupMetadata.create_default() updated = base.update(attributes={"a": 1}) assert updated.attributes == {"a": 1} assert base.attributes == {} -# --- GroupMetadataModelV2 --------------------------------------------------- +# --- ZarrV2GroupMetadata --------------------------------------------------- def test_group_v2_key_value_split() -> None: """v2 to_key_value writes .zgroup and .zattrs; from_key_value merges them.""" - model = GroupMetadataModelV2.create_default(attributes={"a": 1}) + model = ZarrV2GroupMetadata.create_default(attributes={"a": 1}) kv = model.to_key_value() assert set(kv) == {".zgroup", ".zattrs"} assert json.loads(kv[".zgroup"]) == {"zarr_format": 2} - assert GroupMetadataModelV2.from_key_value(kv) == model + assert ZarrV2GroupMetadata.from_key_value(kv) == model def test_group_v2_zattrs_presence_round_trips() -> None: """A v2 group with no .zattrs file parses with UNSET attributes and emits no .zattrs; an explicit empty .zattrs stays a file — the stores remain distinct through a round-trip.""" - absent = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + absent = ZarrV2GroupMetadata.from_key_value({".zgroup": b'{"zarr_format": 2}'}) assert absent.attributes is UNSET assert ".zattrs" not in absent.to_key_value() - explicit = GroupMetadataModelV2.from_key_value( + explicit = ZarrV2GroupMetadata.from_key_value( {".zgroup": b'{"zarr_format": 2}', ".zattrs": b"{}"} ) assert explicit.attributes == {} @@ -134,13 +134,13 @@ def test_group_v2_zattrs_presence_round_trips() -> None: def test_group_v2_json_roundtrip() -> None: """A merged-form v2 group document round-trips through the model unchanged.""" doc = {"zarr_format": 2, "attributes": {"a": 1}} - model = GroupMetadataModelV2.from_json(doc) + model = ZarrV2GroupMetadata.from_json(doc) assert model.to_json() == doc def test_group_v2_omits_empty_attributes() -> None: """to_json omits the attributes key when attributes is empty.""" - assert "attributes" not in GroupMetadataModelV2.create_default().to_json() + assert "attributes" not in ZarrV2GroupMetadata.create_default().to_json() def test_group_v2_not_a_mapping() -> None: @@ -165,20 +165,20 @@ def test_group_partial_keys_match_settable_model_fields() -> None: without updating its `*Partial` TypedDict fails here. """ for model_cls, partial_cls in ( - (GroupMetadataModelV3, GroupMetadataModelV3Partial), - (GroupMetadataModelV2, GroupMetadataModelV2Partial), + (ZarrV3GroupMetadata, ZarrV3GroupMetadataPartial), + (ZarrV2GroupMetadata, ZarrV2GroupMetadataPartial), ): settable = {f.name for f in dataclasses.fields(model_cls) if f.init} assert set(partial_cls.__annotations__) == settable -# --- ConsolidatedMetadataModelV3 -------------------------------------------- +# --- ZarrV3ConsolidatedMetadata -------------------------------------------- def test_consolidated_v3_roundtrip() -> None: """A v3 group with inline consolidated metadata round-trips, with child entries parsed into array/group models.""" - child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() doc = { "zarr_format": 3, "node_type": "group", @@ -188,23 +188,23 @@ def test_consolidated_v3_roundtrip() -> None: "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, }, } - model = GroupMetadataModelV3.from_json(doc) - assert isinstance(model.consolidated_metadata, ConsolidatedMetadataModelV3) - assert isinstance(model.consolidated_metadata.metadata["a"], ArrayMetadataModelV3) - assert isinstance(model.consolidated_metadata.metadata["g"], GroupMetadataModelV3) + model = ZarrV3GroupMetadata.from_json(doc) + assert isinstance(model.consolidated_metadata, ZarrV3ConsolidatedMetadata) + assert isinstance(model.consolidated_metadata.metadata["a"], ZarrV3ArrayMetadata) + assert isinstance(model.consolidated_metadata.metadata["g"], ZarrV3GroupMetadata) assert model.to_json() == doc def test_consolidated_v3_must_understand_true_rejected() -> None: - """ConsolidatedMetadataModelV3 enforces must_understand=False at runtime.""" + """ZarrV3ConsolidatedMetadata enforces must_understand=False at runtime.""" with pytest.raises(ValueError, match="must_understand"): - ConsolidatedMetadataModelV3(must_understand=True, metadata={}) + ZarrV3ConsolidatedMetadata(must_understand=True, metadata={}) def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: """from_json rejects a consolidated document carrying must_understand=true.""" with pytest.raises(MetadataValidationError, match="must_understand"): - ConsolidatedMetadataModelV3.from_json( + ZarrV3ConsolidatedMetadata.from_json( {"kind": "inline", "must_understand": True, "metadata": {}} ) @@ -212,7 +212,7 @@ def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: def test_consolidated_v3_entry_without_node_type_rejected() -> None: """from_json rejects a consolidated entry lacking a recognizable node_type.""" with pytest.raises(MetadataValidationError, match="node_type"): - ConsolidatedMetadataModelV3.from_json( + ZarrV3ConsolidatedMetadata.from_json( {"kind": "inline", "must_understand": False, "metadata": {"a": {"zarr_format": 3}}} ) @@ -220,10 +220,10 @@ def test_consolidated_v3_entry_without_node_type_rejected() -> None: def test_consolidated_v3_not_a_mapping() -> None: """from_json rejects a non-mapping consolidated document.""" with pytest.raises(MetadataValidationError, match="expected a mapping"): - ConsolidatedMetadataModelV3.from_json(5) + ZarrV3ConsolidatedMetadata.from_json(5) -# --- ConsolidatedMetadataModelV2 -------------------------------------------- +# --- ZarrV2ConsolidatedMetadata -------------------------------------------- def test_consolidated_v2_verbatim_roundtrip() -> None: @@ -245,16 +245,16 @@ def test_consolidated_v2_verbatim_roundtrip() -> None: }, }, } - model = ConsolidatedMetadataModelV2.from_json(doc) + model = ZarrV2ConsolidatedMetadata.from_json(doc) assert model.to_json() == doc def test_consolidated_v2_key_value_roundtrip() -> None: """from_key_value(to_key_value()) is the identity for .zmetadata documents.""" - model = ConsolidatedMetadataModelV2.from_json( + model = ZarrV2ConsolidatedMetadata.from_json( {"zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}} ) - assert ConsolidatedMetadataModelV2.from_key_value(model.to_key_value()) == model + assert ZarrV2ConsolidatedMetadata.from_key_value(model.to_key_value()) == model def test_consolidated_v2_lists_become_tuples() -> None: @@ -263,20 +263,20 @@ def test_consolidated_v2_lists_become_tuples() -> None: "zarr_consolidated_format": 1, "metadata": {"a/.zarray": {"shape": [2, 3]}}, } - model = ConsolidatedMetadataModelV2.from_json(doc) + model = ZarrV2ConsolidatedMetadata.from_json(doc) assert model.metadata == {"a/.zarray": {"shape": (2, 3)}} def test_consolidated_v2_envelope_validation() -> None: """from_json rejects a .zmetadata document missing the metadata key.""" with pytest.raises(MetadataValidationError, match="metadata"): - ConsolidatedMetadataModelV2.from_json({"zarr_consolidated_format": 1}) + ZarrV2ConsolidatedMetadata.from_json({"zarr_consolidated_format": 1}) def test_consolidated_v2_not_a_mapping() -> None: """from_json rejects a non-mapping .zmetadata document.""" with pytest.raises(MetadataValidationError, match="expected a mapping"): - ConsolidatedMetadataModelV2.from_json([1]) + ZarrV2ConsolidatedMetadata.from_json([1]) # --- Literal-value enforcement ----------------------------------------------- @@ -284,7 +284,7 @@ def test_consolidated_v2_not_a_mapping() -> None: def test_group_v3_literals_enforced() -> None: """A v3 group doc with wrong zarr_format or node_type is rejected with invalid_value.""" - base = GroupMetadataModelV3.create_default().to_json() + base = ZarrV3GroupMetadata.create_default().to_json() for key, bad in (("zarr_format", 2), ("node_type", "array")): problems = validate_group_metadata_v3(dict(base) | {key: bad}) assert [(p.loc, p.kind) for p in problems] == [((key,), "invalid_value")], key @@ -329,12 +329,12 @@ def test_group_v3_validator_agrees_with_from_json_on_consolidated() -> None: for doc in bad_docs: assert validate_group_metadata_v3(doc) != [], doc with pytest.raises(MetadataValidationError): - GroupMetadataModelV3.from_json(doc) + ZarrV3GroupMetadata.from_json(doc) def test_group_v3_valid_consolidated_passes_validator() -> None: """A well-formed consolidated group validates cleanly (control case).""" - child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() doc = { "zarr_format": 3, "node_type": "group", @@ -353,7 +353,7 @@ def test_group_v3_valid_consolidated_passes_validator() -> None: def test_group_must_understand_fields_partition() -> None: """The group model partitions extra fields by the spec's implicit-true rule, like the array model.""" - model = GroupMetadataModelV3.create_default( + model = ZarrV3GroupMetadata.create_default( extra_fields={ "waived": {"name": "w", "must_understand": False}, "implicit": {"name": "i"}, @@ -369,7 +369,7 @@ def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: deliberately repairs the document rather than preserving the bug.""" null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} assert validate_group_metadata_v3(null_doc) == [] - model = GroupMetadataModelV3.from_json(null_doc) + model = ZarrV3GroupMetadata.from_json(null_doc) assert model.consolidated_metadata is UNSET assert "consolidated_metadata" not in model.to_json() - assert model == GroupMetadataModelV3.from_json({"zarr_format": 3, "node_type": "group"}) + assert model == ZarrV3GroupMetadata.from_json({"zarr_format": 3, "node_type": "group"}) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 7e80d52ee8..d3e6bc6358 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -3,7 +3,7 @@ Pydantic's native dataclass introspection CAN be made to work (see `test_native_dataclass_introspection_is_possible_but_diverges`): the models keep their annotation-only imports behind `TYPE_CHECKING`, so a bare -`TypeAdapter(ArrayMetadataModelV3)` raises `class-not-fully-defined`, but +`TypeAdapter(ZarrV3ArrayMetadata)` raises `class-not-fully-defined`, but `rebuild(_types_namespace=...)` with the names supplied resolves the schema. It is still the wrong tool: it validates the MODEL SHAPE, not the DOCUMENT — no `from_json` normalization (a bare-string `data_type` is rejected), and @@ -42,24 +42,24 @@ ) from zarr_metadata import JSONValue -from zarr_metadata.model import ArrayMetadataModelV3, NamedConfigModelV3 +from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3NamedConfig # --- the integration (this is the example) ----------------------------------- -def _as_array_metadata_v3(value: object) -> ArrayMetadataModelV3: +def _as_array_metadata_v3(value: object) -> ZarrV3ArrayMetadata: """Accept an existing model instance or a raw metadata document.""" - if isinstance(value, ArrayMetadataModelV3): + if isinstance(value, ZarrV3ArrayMetadata): return value - return ArrayMetadataModelV3.from_json(value) + return ZarrV3ArrayMetadata.from_json(value) -# return_type is explicit because to_json's own annotation (`ArrayMetadataV3`) +# return_type is explicit because to_json's own annotation (`ZarrV3ArrayMetadataJSON`) # is a TYPE_CHECKING-only name pydantic cannot resolve at runtime. ArrayMetadataV3Field = Annotated[ - InstanceOf[ArrayMetadataModelV3], + InstanceOf[ZarrV3ArrayMetadata], BeforeValidator(_as_array_metadata_v3), - PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), + PlainSerializer(ZarrV3ArrayMetadata.to_json, return_type=dict), ] """A pydantic-ready field type for v3 array metadata. @@ -93,14 +93,14 @@ def test_raw_document_is_validated_into_a_model() -> None: """A raw metadata document on a pydantic field is parsed by from_json, with the library's normalization applied (tuples, canonical field form).""" manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) - assert isinstance(manifest.metadata, ArrayMetadataModelV3) + assert isinstance(manifest.metadata, ZarrV3ArrayMetadata) assert manifest.metadata.shape == (10,) assert manifest.metadata.data_type.name == "uint8" def test_model_instance_passes_through() -> None: """An already-constructed model instance is accepted unchanged.""" - model = ArrayMetadataModelV3.from_json(VALID_DOC) + model = ZarrV3ArrayMetadata.from_json(VALID_DOC) manifest = ArrayManifest(path="a/b", metadata=model) assert manifest.metadata is model @@ -137,7 +137,7 @@ def test_type_adapter_standalone() -> None: """The annotated alias also works without a BaseModel, via TypeAdapter.""" adapter = TypeAdapter(ArrayMetadataV3Field) model = adapter.validate_python(VALID_DOC) - assert isinstance(model, ArrayMetadataModelV3) + assert isinstance(model, ZarrV3ArrayMetadata) assert adapter.dump_python(model) == model.to_json() @@ -157,20 +157,20 @@ def test_native_dataclass_introspection_is_not_supported() -> None: path needs its divergences documented again.""" from zarr_metadata._common import JSONValue from zarr_metadata.model import UNSET - from zarr_metadata.v3._common import MetadataV3 - from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField def build_and_use() -> None: - adapter = TypeAdapter(ArrayMetadataModelV3) + adapter = TypeAdapter(ZarrV3ArrayMetadata) adapter.rebuild( force=True, _types_namespace={ "JSONValue": JSONValue, - "ExtensionFieldV3": ExtensionFieldV3, - "MetadataV3": MetadataV3, - "ArrayMetadataV3": ArrayMetadataV3, - "NamedConfigModelV3": NamedConfigModelV3, - "MetadataFieldModelV3": NamedConfigModelV3, + "ZarrV3ExtensionField": ZarrV3ExtensionField, + "ZarrV3MetadataFieldJSON": ZarrV3MetadataFieldJSON, + "ZarrV3ArrayMetadataJSON": ZarrV3ArrayMetadataJSON, + "ZarrV3NamedConfig": ZarrV3NamedConfig, + "ZarrV3MetadataField": ZarrV3NamedConfig, "UNSET": UNSET, }, ) @@ -204,7 +204,7 @@ class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): """A pydantic-native, attribute-typed view of a v3 array metadata document. The library is the engine: every input is canonicalized and structurally - validated by `ArrayMetadataModelV3.from_json` before pydantic sees the + validated by `ZarrV3ArrayMetadata.from_json` before pydantic sees the fields, and `to_document` / `to_metadata_model` emit through the library. """ @@ -228,12 +228,12 @@ def _canonicalize(cls, data: object) -> object: """Route every input document through the library's validation and normalization; pydantic then parses only canonical documents.""" if isinstance(data, Mapping): - doc = dict(ArrayMetadataModelV3.from_json(data).to_json()) + doc = dict(ZarrV3ArrayMetadata.from_json(data).to_json()) doc.setdefault("attributes", {}) return doc return data - def to_metadata_model(self) -> ArrayMetadataModelV3: + def to_metadata_model(self) -> ZarrV3ArrayMetadata: """Bridge back to the canonical model, via the document form. In the document, "no dimension names" is key-absence, not null; the @@ -242,7 +242,7 @@ def to_metadata_model(self) -> ArrayMetadataModelV3: doc = self.model_dump() if doc["dimension_names"] is None: del doc["dimension_names"] - return ArrayMetadataModelV3.from_json(doc) + return ZarrV3ArrayMetadata.from_json(doc) def to_document(self) -> dict[str, object]: """The canonical document (omit-empty conventions applied).""" @@ -280,7 +280,7 @@ def test_spec_bridges_to_canonical_model_and_document() -> None: doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) model = spec.to_metadata_model() - assert isinstance(model, ArrayMetadataModelV3) + assert isinstance(model, ZarrV3ArrayMetadata) assert spec.to_document() == dict(model.to_json()) # and back: the document revalidates to an equal spec assert ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(spec.to_document()) == spec diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 81b9b962af..56be055c77 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -10,17 +10,17 @@ import zarr_metadata.pydantic as zmp from zarr_metadata.model import ( - ArrayMetadataModelV2, - ArrayMetadataModelV3, - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV3, - NamedConfigModelV3, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3NamedConfig, ) -V3_ARRAY_DOC = dict(ArrayMetadataModelV3.create_default(shape=(4,)).to_json()) -V2_ARRAY_DOC = dict(ArrayMetadataModelV2.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_ARRAY_DOC = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} V3_CONSOLIDATED_DOC = { @@ -34,23 +34,23 @@ } FIELD_CASES = [ - pytest.param(zmp.ArrayMetadataV3, ArrayMetadataModelV3, V3_ARRAY_DOC, id="array-v3"), - pytest.param(zmp.ArrayMetadataV2, ArrayMetadataModelV2, V2_ARRAY_DOC, id="array-v2"), - pytest.param(zmp.GroupMetadataV3, GroupMetadataModelV3, V3_GROUP_DOC, id="group-v3"), - pytest.param(zmp.GroupMetadataV2, GroupMetadataModelV2, V2_GROUP_DOC, id="group-v2"), + pytest.param(zmp.ZarrV3ArrayMetadata, ZarrV3ArrayMetadata, V3_ARRAY_DOC, id="array-v3"), + pytest.param(zmp.ZarrV2ArrayMetadata, ZarrV2ArrayMetadata, V2_ARRAY_DOC, id="array-v2"), + pytest.param(zmp.ZarrV3GroupMetadata, ZarrV3GroupMetadata, V3_GROUP_DOC, id="group-v3"), + pytest.param(zmp.ZarrV2GroupMetadata, ZarrV2GroupMetadata, V2_GROUP_DOC, id="group-v2"), pytest.param( - zmp.ConsolidatedMetadataV3, - ConsolidatedMetadataModelV3, + zmp.ZarrV3ConsolidatedMetadata, + ZarrV3ConsolidatedMetadata, V3_CONSOLIDATED_DOC, id="consolidated-v3", ), pytest.param( - zmp.ConsolidatedMetadataV2, - ConsolidatedMetadataModelV2, + zmp.ZarrV2ConsolidatedMetadata, + ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC, id="consolidated-v2", ), - pytest.param(zmp.MetadataFieldV3, NamedConfigModelV3, {"name": "bytes"}, id="field-v3"), + pytest.param(zmp.ZarrV3MetadataField, ZarrV3NamedConfig, {"name": "bytes"}, id="field-v3"), ] @@ -74,9 +74,9 @@ def test_core_instances_interoperate() -> None: the core classes rather than pydantic-aware subclasses.""" class Manifest(BaseModel): - metadata: zmp.ArrayMetadataV3 + metadata: zmp.ZarrV3ArrayMetadata - core = ArrayMetadataModelV3.from_json(V3_ARRAY_DOC) + core = ZarrV3ArrayMetadata.from_json(V3_ARRAY_DOC) manifest = Manifest(metadata=core) assert manifest.metadata is core @@ -85,7 +85,7 @@ def test_validation_error_carries_problems() -> None: """A defective document fails with the library's loc-annotated messages.""" class Manifest(BaseModel): - metadata: zmp.ArrayMetadataV3 + metadata: zmp.ZarrV3ArrayMetadata doc = dict(V3_ARRAY_DOC) del doc["chunk_key_encoding"] @@ -97,11 +97,11 @@ def test_json_schema_generation() -> None: """model_json_schema works, describing the document form each field accepts.""" class Manifest(BaseModel): - metadata: zmp.ArrayMetadataV3 - codec: zmp.MetadataFieldV3 + metadata: zmp.ZarrV3ArrayMetadata + codec: zmp.ZarrV3MetadataField schema = Manifest.model_json_schema() - assert schema["properties"]["metadata"] == {"type": "object", "title": "ArrayMetadataV3"} + assert schema["properties"]["metadata"] == {"type": "object", "title": "ZarrV3ArrayMetadata"} assert schema["properties"]["codec"]["anyOf"] == [{"type": "string"}, {"type": "object"}] @@ -109,7 +109,7 @@ def test_json_roundtrip() -> None: """model_dump_json output re-validates to an equal pydantic model.""" class Manifest(BaseModel): - metadata: zmp.ArrayMetadataV3 + metadata: zmp.ZarrV3ArrayMetadata manifest = Manifest.model_validate({"metadata": V3_ARRAY_DOC}) assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest diff --git a/packages/zarr-metadata/tests/model/test_sentinel.py b/packages/zarr-metadata/tests/model/test_sentinel.py index 78aa60e208..252a4cdd30 100644 --- a/packages/zarr-metadata/tests/model/test_sentinel.py +++ b/packages/zarr-metadata/tests/model/test_sentinel.py @@ -21,11 +21,11 @@ from zarr_metadata.model import ( UNSET, - ArrayMetadataModelV2, - ArrayMetadataModelV3, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV3, + ZarrV2ArrayMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, ) # Whole-model cases covering the states we know are problematic for @@ -33,20 +33,20 @@ # same fields in the present state (including present-but-empty, which must # stay distinct from absent), and UNSET nested inside consolidated metadata. MODEL_CASES = { - "array-v3-dimension-names-unset": ArrayMetadataModelV3.create_default(shape=(4,)), - "array-v3-dimension-names-set": ArrayMetadataModelV3.create_default(shape=(2, 2)).update( + "array-v3-dimension-names-unset": ZarrV3ArrayMetadata.create_default(shape=(4,)), + "array-v3-dimension-names-set": ZarrV3ArrayMetadata.create_default(shape=(2, 2)).update( dimension_names=("x", None) ), - "array-v2-attributes-unset": ArrayMetadataModelV2.create_default(shape=(4,)), - "array-v2-attributes-empty": ArrayMetadataModelV2.create_default(shape=(4,), attributes={}), - "group-v2-attributes-unset": GroupMetadataModelV2.create_default(), - "group-v2-attributes-set": GroupMetadataModelV2.create_default(attributes={"a": 1}), - "group-v3-consolidated-unset": GroupMetadataModelV3.create_default(), - "group-v3-consolidated-with-unset-inside": GroupMetadataModelV3.create_default( - consolidated_metadata=ConsolidatedMetadataModelV3( + "array-v2-attributes-unset": ZarrV2ArrayMetadata.create_default(shape=(4,)), + "array-v2-attributes-empty": ZarrV2ArrayMetadata.create_default(shape=(4,), attributes={}), + "group-v2-attributes-unset": ZarrV2GroupMetadata.create_default(), + "group-v2-attributes-set": ZarrV2GroupMetadata.create_default(attributes={"a": 1}), + "group-v3-consolidated-unset": ZarrV3GroupMetadata.create_default(), + "group-v3-consolidated-with-unset-inside": ZarrV3GroupMetadata.create_default( + consolidated_metadata=ZarrV3ConsolidatedMetadata( metadata={ - "child": ArrayMetadataModelV3.create_default(shape=(4,)), - "subgroup": GroupMetadataModelV3.create_default(), + "child": ZarrV3ArrayMetadata.create_default(shape=(4,)), + "subgroup": ZarrV3GroupMetadata.create_default(), } ) ), @@ -65,10 +65,7 @@ def test_unset_copy_preserves_identity() -> None: @pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) def test_model_pickle_round_trip( - model: ArrayMetadataModelV2 - | ArrayMetadataModelV3 - | GroupMetadataModelV2 - | GroupMetadataModelV3, + model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, ) -> None: restored = pickle.loads(pickle.dumps(model)) assert restored == model @@ -76,10 +73,7 @@ def test_model_pickle_round_trip( @pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) def test_model_deepcopy( - model: ArrayMetadataModelV2 - | ArrayMetadataModelV3 - | GroupMetadataModelV2 - | GroupMetadataModelV3, + model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, ) -> None: assert copy.deepcopy(model) == model diff --git a/packages/zarr-metadata/tests/test_partial_equivalence.py b/packages/zarr-metadata/tests/test_partial_equivalence.py index 995a6a21e1..33492b2356 100644 --- a/packages/zarr-metadata/tests/test_partial_equivalence.py +++ b/packages/zarr-metadata/tests/test_partial_equivalence.py @@ -14,17 +14,17 @@ import pytest -from zarr_metadata.v2.array import ArrayMetadataV2, ArrayMetadataV2Partial -from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial -from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial -from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataJSONPartial +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ArrayMetadataJSONPartial +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial # (full, partial) pairs to check. Add new pairs here as more are introduced. PAIRS: list[tuple[type, type]] = [ - (ArrayMetadataV3, ArrayMetadataV3Partial), - (GroupMetadataV3, GroupMetadataV3Partial), - (ArrayMetadataV2, ArrayMetadataV2Partial), - (GroupMetadataV2, GroupMetadataV2Partial), + (ZarrV3ArrayMetadataJSON, ZarrV3ArrayMetadataJSONPartial), + (ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial), + (ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial), + (ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataJSONPartial), ] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index ee26e80c5a..de8e271d28 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -21,43 +21,43 @@ def _group_rank(s: str) -> int: EXPECTED = [ # Category A — metadata-document types - "ArrayMetadataV2", - "ArrayMetadataV2Partial", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", "ZArrayMetadata", - "GroupMetadataV2", - "GroupMetadataV2Partial", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", "ZGroupMetadata", - "ConsolidatedMetadataV2", + "ZarrV2ConsolidatedMetadataJSON", "ZAttrsMetadata", - "CodecMetadataV2", - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ExtensionFieldV3", - "GroupMetadataV3", - "GroupMetadataV3Partial", - "ConsolidatedMetadataV3", - "NamedConfigV3", - "MetadataV3", + "ZarrV2CodecMetadata", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3NamedConfigJSON", + "ZarrV3MetadataFieldJSON", "JSONValue", # Category A' — metadata models (in-memory dataclasses over the documents) - "ArrayMetadataModelV2", - "ArrayMetadataModelV2Partial", - "ArrayMetadataModelV3", - "ArrayMetadataModelV3Partial", - "GroupMetadataModelV2", - "GroupMetadataModelV2Partial", - "GroupMetadataModelV3", - "GroupMetadataModelV3Partial", - "ConsolidatedMetadataModelV2", - "ConsolidatedMetadataModelV3", - "NamedConfigModelV3", - "MetadataFieldModelV3", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataPartial", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataPartial", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataPartial", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataPartial", + "ZarrV2ConsolidatedMetadata", + "ZarrV3ConsolidatedMetadata", + "ZarrV3NamedConfig", + "ZarrV3MetadataField", "ValidationProblem", "MetadataValidationError", "ProblemKind", "UNSET", # v2 data-type encoding union - "DataTypeMetadataV2", + "ZarrV2DataTypeMetadata", # Category B — codec canonical unions "BloscCodecMetadata", "BytesCodecMetadata", @@ -146,9 +146,9 @@ def _group_rank(s: str) -> int: "RawBytesFillValue", # Category E — constant+Literal pairs "ARRAY_ORDER_V2", - "ArrayOrderV2", + "ZarrV2ArrayOrder", "ARRAY_DIMENSION_SEPARATOR_V2", - "ArrayDimensionSeparatorV2", + "ZarrV2ArrayDimensionSeparator", "ENDIANNESS", "Endianness", "BYTES_CODEC_NAME", @@ -226,7 +226,7 @@ def test_promoted_pairs_drift() -> None: (zm.NUMPY_TIME_UNIT, zm.NumpyTimeUnit), (zm.CAST_ROUNDING_MODE, zm.CastRoundingMode), (zm.CAST_OUT_OF_RANGE_MODE, zm.CastOutOfRangeMode), - (zm.ARRAY_ORDER_V2, zm.ArrayOrderV2), + (zm.ARRAY_ORDER_V2, zm.ZarrV2ArrayOrder), ] for const, lit in pairs: assert set(const) == set(get_args(lit)) diff --git a/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py b/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py index 9dad66d074..e802c5bef8 100644 --- a/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ConsolidatedMetadataV2) +ADAPTER = TypeAdapter(ZarrV2ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/array/test_fixtures.py b/packages/zarr-metadata/tests/v3/array/test_fixtures.py index fccd00d481..c84cc4042b 100644 --- a/packages/zarr-metadata/tests/v3/array/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/array/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v3 array metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`zarr.json` that should validate cleanly as `ArrayMetadataV3`. +`zarr.json` that should validate cleanly as `ZarrV3ArrayMetadataJSON`. Fixtures are named for the variant they exercise (regular vs rectilinear grid, blosc/gzip/zstd/sharding_indexed codecs, named-config dtypes, optional fields, extra fields). @@ -15,11 +15,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ArrayMetadataV3) +ADAPTER = TypeAdapter(ZarrV3ArrayMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/array/with_extra_field.json b/packages/zarr-metadata/tests/v3/array/with_extra_field.json index 46a7f0f235..bd7a9f5b45 100644 --- a/packages/zarr-metadata/tests/v3/array/with_extra_field.json +++ b/packages/zarr-metadata/tests/v3/array/with_extra_field.json @@ -16,6 +16,6 @@ ], "my_custom_extension": { "must_understand": false, - "purpose": "exercise the extra_items=ExtensionFieldV3 path" + "purpose": "exercise the extra_items=ZarrV3ExtensionField path" } } diff --git a/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py b/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py index 4d9e300bae..d052b16986 100644 --- a/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ConsolidatedMetadataV3) +ADAPTER = TypeAdapter(ZarrV3ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/group/test_fixtures.py b/packages/zarr-metadata/tests/v3/group/test_fixtures.py index 2015d5ce96..ffcdedef2b 100644 --- a/packages/zarr-metadata/tests/v3/group/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/group/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(GroupMetadataV3) +ADAPTER = TypeAdapter(ZarrV3GroupMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) From ee2c3d2140ababd8feb8a392bec5ae794f0a30c1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 12:52:15 +0200 Subject: [PATCH 36/48] fix(zarr-metadata): harden model validation Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/model/_array.py | 9 +- .../src/zarr_metadata/model/_group.py | 30 +++- .../src/zarr_metadata/model/_validation.py | 155 ++++++++++++++++-- .../zarr-metadata/tests/model/test_array.py | 86 ++++++++++ .../zarr-metadata/tests/model/test_group.py | 41 +++++ 5 files changed, 297 insertions(+), 24 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index f68c1b0f5e..aee4f80fa8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -447,11 +447,14 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: - zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) + zarray_raw = cast("object", load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2)) + if not isinstance(zarray_raw, Mapping): + return cls.from_json(zarray_raw) + zarray = cast("Mapping[str, object]", zarray_raw) if ATTRIBUTES_STORE_KEY_V2 in mapping: - zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) return cls.from_json({**zarray, "attributes": zattrs}) - return cls.from_json(dict(zarray)) + return cls.from_json(zarray) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zarray` diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 222b0b3bf9..81bbbad270 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -25,6 +25,7 @@ parse_group_metadata_v2, parse_group_metadata_v3, validate_consolidated_metadata_v3, + validate_json, ) if TYPE_CHECKING: @@ -323,11 +324,14 @@ def from_json(cls, data: object) -> GroupMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: - zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) + zgroup_raw = cast("object", load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2)) + if not isinstance(zgroup_raw, Mapping): + return cls.from_json(zgroup_raw) + zgroup = cast("Mapping[str, object]", zgroup_raw) if ATTRIBUTES_STORE_KEY_V2 in mapping: - zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) return cls.from_json({**zgroup, "attributes": zattrs}) - return cls.from_json(dict(zgroup)) + return cls.from_json(zgroup) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zgroup` @@ -374,6 +378,18 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: for key in ("zarr_consolidated_format", "metadata") if key not in doc ] + if "zarr_consolidated_format" in doc and ( + not isinstance(doc["zarr_consolidated_format"], int) + or isinstance(doc["zarr_consolidated_format"], bool) + or doc["zarr_consolidated_format"] != 1 + ): + problems.append( + ValidationProblem( + ("zarr_consolidated_format",), + f"expected 1, got {doc['zarr_consolidated_format']!r}", + "invalid_value", + ) + ) if "metadata" in doc: entries = doc["metadata"] if not isinstance(entries, Mapping) or not all( @@ -384,6 +400,14 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: ("metadata",), "expected a mapping with string keys", "invalid_type" ) ) + else: + for key, value in cast("Mapping[str, object]", entries).items(): + problems.extend( + ValidationProblem( + ("metadata", key, *problem.loc), problem.message, problem.kind + ) + for problem in validate_json(value) + ) if problems: raise MetadataValidationError(problems) entries_tupled = cast( diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index fae09187a4..a05fe7a430 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -161,6 +161,27 @@ def _check_literal( return [] +def _validate_extension_fields_v3( + doc: Mapping[object, object], + standard_keys: frozenset[str], + *, + additional_reserved_keys: frozenset[str] = frozenset(), +) -> list[ValidationProblem]: + """Validate v3 top-level key types and unknown-field JSON payloads.""" + problems: list[ValidationProblem] = [] + reserved_keys = standard_keys | additional_reserved_keys + for key, value in doc.items(): + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string top-level key {key!r}", "invalid_type") + ) + continue + if key in reserved_keys: + continue + problems.extend(_prefix(key, validate_json(value))) + return problems + + def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. @@ -216,7 +237,7 @@ def _is_int_sequence(value: object) -> bool: is not an integer in a metadata document, so booleans are excluded. """ return ( - not isinstance(value, str) + not isinstance(value, (str, bytes, bytearray)) and isinstance(value, Sequence) and all( isinstance(item, int) and not isinstance(item, bool) @@ -267,6 +288,78 @@ def _is_dtype_v2(value: object) -> bool: return True +def _is_canonical_dtype_v2(value: object) -> bool: + """Whether a validated v2 dtype uses the tuple-backed public representation.""" + if isinstance(value, str): + return True + if not isinstance(value, tuple): + return False + for record in cast("tuple[object, ...]", value): + if not isinstance(record, tuple): + return False + fields = cast("tuple[object, ...]", record) + if not _is_canonical_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not isinstance(fields[2], tuple): + return False + return True + + +def _is_canonical_metadata_field_v3(value: object) -> bool: + """Whether a validated v3 metadata field has its declared runtime container type.""" + return isinstance(value, (str, dict)) + + +def _is_canonical_array_metadata_v3(value: object) -> bool: + """Whether a validated v3 array document matches `ArrayMetadataV3` at runtime.""" + if not isinstance(value, dict): + return False + doc = cast("dict[str, object]", value) + if not isinstance(doc["shape"], tuple) or not isinstance(doc["codecs"], tuple): + return False + if "storage_transformers" in doc and not isinstance(doc["storage_transformers"], tuple): + return False + if "dimension_names" in doc and not isinstance(doc["dimension_names"], tuple): + return False + if not all( + _is_canonical_metadata_field_v3(doc[key]) + for key in ("data_type", "chunk_grid", "chunk_key_encoding") + ): + return False + if not all( + _is_canonical_metadata_field_v3(item) for item in cast("tuple[object, ...]", doc["codecs"]) + ): + return False + if "storage_transformers" in doc and not all( + _is_canonical_metadata_field_v3(item) + for item in cast("tuple[object, ...]", doc["storage_transformers"]) + ): + return False + return all( + key in ARRAY_METADATA_STANDARD_KEYS_V3 or isinstance(item, dict) + for key, item in doc.items() + ) + + +def _is_canonical_array_metadata_v2(value: object) -> bool: + """Whether a validated v2 array document matches `ArrayMetadataV2` at runtime.""" + if not isinstance(value, dict): + return False + doc = cast("dict[str, object]", value) + if not isinstance(doc["shape"], tuple) or not isinstance(doc["chunks"], tuple): + return False + if not _is_canonical_dtype_v2(doc["dtype"]): + return False + compressor = doc["compressor"] + if compressor is not None and not isinstance(compressor, dict): + return False + filters = doc["filters"] + return filters is None or ( + isinstance(filters, tuple) + and all(isinstance(item, dict) for item in cast("tuple[object, ...]", filters)) + ) + + def _is_codec_v2(value: object) -> bool: """Whether `value` is shaped like a v2 codec config: a mapping with a string `id`.""" return isinstance(value, Mapping) and isinstance( @@ -274,6 +367,17 @@ def _is_codec_v2(value: object) -> bool: ) +def _validate_codec_v2(value: object) -> list[ValidationProblem]: + """Validate a v2 codec's required shape and JSON-valued configuration.""" + if not _is_codec_v2(value): + return [ + ValidationProblem( + (), "expected a codec configuration with a string 'id'", "invalid_type" + ) + ] + return validate_json(value) + + def _validate_attributes(value: object) -> list[ValidationProblem]: """Validate an `attributes` value: a mapping with string keys. @@ -307,6 +411,11 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend( + _validate_extension_fields_v3( + cast("Mapping[object, object]", value), ARRAY_METADATA_STANDARD_KEYS_V3 + ) + ) problems.extend(_check_literal(doc, "zarr_format", 3)) problems.extend(_check_literal(doc, "node_type", "array")) problems.extend(_validate_dim_sequence(doc, "shape")) @@ -357,15 +466,16 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: def is_array_metadata_v3(value: object) -> TypeIs[ArrayMetadataV3]: """Whether `value` is a structurally-valid v3 array metadata document.""" - return not validate_array_metadata_v3(value) + return not validate_array_metadata_v3(value) and _is_canonical_array_metadata_v3(value) def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: """Return `value` narrowed to `ArrayMetadataV3`, or raise `MetadataValidationError`.""" - problems = validate_array_metadata_v3(value) + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v3(normalized) if problems: raise MetadataValidationError(problems) - return cast(ArrayMetadataV3, value) + return cast(ArrayMetadataV3, normalized) def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: @@ -399,14 +509,8 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: ) if "compressor" in doc: compressor = doc["compressor"] - if compressor is not None and not _is_codec_v2(compressor): - problems.append( - ValidationProblem( - ("compressor",), - "expected null or a codec configuration with a string 'id'", - "invalid_type", - ) - ) + if compressor is not None: + problems.extend(_prefix("compressor", _validate_codec_v2(compressor))) if "filters" in doc: filters = doc["filters"] if filters is not None and ( @@ -421,6 +525,9 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: "invalid_type", ) ) + elif filters is not None: + for index, item in enumerate(cast("Sequence[object]", filters)): + problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): problems.append( ValidationProblem( @@ -438,15 +545,16 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: def is_array_metadata_v2(value: object) -> TypeIs[ArrayMetadataV2]: """Whether `value` is a structurally-valid v2 array metadata document.""" - return not validate_array_metadata_v2(value) + return not validate_array_metadata_v2(value) and _is_canonical_array_metadata_v2(value) def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: """Return `value` narrowed to `ArrayMetadataV2`, or raise `MetadataValidationError`.""" - problems = validate_array_metadata_v2(value) + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v2(normalized) if problems: raise MetadataValidationError(problems) - return cast(ArrayMetadataV2, value) + return cast(ArrayMetadataV2, normalized) def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: @@ -514,6 +622,13 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend( + _validate_extension_fields_v3( + cast("Mapping[object, object]", value), + GROUP_METADATA_STANDARD_KEYS_V3, + additional_reserved_keys=frozenset({"consolidated_metadata"}), + ) + ) problems.extend(_check_literal(doc, "zarr_format", 3)) problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: @@ -587,7 +702,7 @@ def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: ) try: return json.loads(mapping[key]) - except json.JSONDecodeError as exc: + except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise MetadataValidationError( [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] ) from exc @@ -598,7 +713,11 @@ def arrays_to_tuples(obj: object) -> object: if isinstance(obj, list): return tuple(arrays_to_tuples(item) for item in cast("list[object]", obj)) if isinstance(obj, dict): - return { - key: arrays_to_tuples(value) for key, value in cast("dict[object, object]", obj).items() + mapping = cast("dict[object, object]", obj) + converted: dict[object, object] = { + key: arrays_to_tuples(value) for key, value in mapping.items() } + if all(converted[key] is value for key, value in mapping.items()): + return cast("object", obj) + return converted return obj diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index ff27923635..a17548634b 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1152,6 +1152,22 @@ def test_from_key_value_invalid_json_raises_metadata_error() -> None: assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] +def test_from_key_value_invalid_utf8_raises_metadata_error() -> None: + """Invalid UTF-8 store bytes use the same invalid_json error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_key_value({"zarr.json": b"\x80"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_v2_from_key_value_scalar_root_raises_metadata_error() -> None: + """A scalar .zarray document fails through the unified metadata error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV2.from_key_value({".zarray": b"null"}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((), "invalid_type") + ] + + def test_from_key_value_missing_key_kind() -> None: """A missing store key surfaces as a missing_key problem at the store-key loc.""" with pytest.raises(MetadataValidationError) as exc_info: @@ -1234,6 +1250,76 @@ def test_configuration_values_must_be_json() -> None: ] +def test_v3_extension_keys_must_be_strings() -> None: + """A non-string top-level key cannot be represented by a v3 document type.""" + doc: dict[object, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[1] = {"must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((), "invalid_type") + ] + + +def test_v3_extension_values_must_be_json() -> None: + """Extension payloads are JSON-checked before a model is constructed.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc["ext"] = {"must_understand": False, "payload": object()} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("ext", "payload"), "invalid_type") + ] + + +def test_v3_json_extension_without_waiver_is_preserved_as_must_understand() -> None: + """A JSON extension without an explicit false waiver remains must-understand.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc["ext"] = 1 + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields["ext"] == 1 + assert model.must_understand_fields == {"ext": 1} + + +def test_v2_codec_configuration_values_must_be_json() -> None: + """Non-JSON codec parameters are rejected for compressors and filters.""" + for field, value, expected_loc in ( + ("compressor", {"id": "x", "payload": object()}, ("compressor", "payload")), + ("filters", ({"id": "x", "payload": object()},), ("filters", 0, "payload")), + ): + doc = dict(ArrayMetadataModelV2.create_default().to_json()) + doc[field] = value + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v2(doc)] == [ + (expected_loc, "invalid_type") + ] + + +def test_dimension_sequences_reject_binary_values() -> None: + """Binary buffers are not JSON arrays even though they are integer sequences.""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": b"\x02"} + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": b"\x02"} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_type") + ] + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_type") + ] + + +def test_array_parsers_normalize_json_lists_before_narrowing() -> None: + """Parsers return tuple-backed document types while guards reject raw list forms.""" + v3_raw = json.loads(json.dumps(ArrayMetadataModelV3.create_default(shape=(2,)).to_json())) + v2_raw = json.loads(json.dumps(ArrayMetadataModelV2.create_default(shape=(2,)).to_json())) + + assert validate_array_metadata_v3(v3_raw) == [] + assert validate_array_metadata_v2(v2_raw) == [] + assert not is_array_metadata_v3(v3_raw) + assert not is_array_metadata_v2(v2_raw) + + v3_parsed = parse_array_metadata_v3(v3_raw) + v2_parsed = parse_array_metadata_v2(v2_raw) + assert isinstance(v3_parsed["shape"], tuple) + assert isinstance(v3_parsed["codecs"], tuple) + assert isinstance(v2_parsed["shape"], tuple) + assert isinstance(v2_parsed["chunks"], tuple) + + # --- must_understand partition (spec: MUST fail to open unrecognized fields) -- diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 58b92f5d71..08adb19d40 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -90,6 +90,18 @@ def test_group_v3_bad_attributes() -> None: parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) +def test_group_v3_extension_fields_are_validated() -> None: + """Group extension payloads must be JSON values with a must-understand flag.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "ext": {"must_understand": False, "payload": object()}, + } + assert [(problem.loc, problem.kind) for problem in validate_group_metadata_v3(doc)] == [ + (("ext", "payload"), "invalid_type") + ] + + def test_group_v3_key_value_roundtrip() -> None: """from_key_value(to_key_value()) is the identity for v3 groups.""" model = GroupMetadataModelV3.create_default(attributes={"a": 1}) @@ -279,6 +291,35 @@ def test_consolidated_v2_not_a_mapping() -> None: ConsolidatedMetadataModelV2.from_json([1]) +def test_consolidated_v2_format_literal_enforced() -> None: + """A .zmetadata document must declare consolidated format 1.""" + with pytest.raises(MetadataValidationError) as exc_info: + ConsolidatedMetadataModelV2.from_json({"zarr_consolidated_format": 2, "metadata": {}}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("zarr_consolidated_format",), "invalid_value") + ] + + +def test_consolidated_v2_metadata_values_must_be_json() -> None: + """Non-JSON values in the flat metadata map are rejected during ingestion.""" + with pytest.raises(MetadataValidationError) as exc_info: + ConsolidatedMetadataModelV2.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": object()}} + ) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("metadata", ".zgroup"), "invalid_type") + ] + + +def test_group_v2_from_key_value_scalar_root_raises_metadata_error() -> None: + """A scalar .zgroup document fails through the unified metadata error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + GroupMetadataModelV2.from_key_value({".zgroup": b"null"}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((), "invalid_type") + ] + + # --- Literal-value enforcement ----------------------------------------------- From a3a4f1961785f667d94a3ede663b05a7061daeb8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 13:06:30 +0200 Subject: [PATCH 37/48] docs(zarr-metadata): define v3 conformance boundary Assisted-by: Codex:gpt-5 --- ...rr-v3-metadata-model-conformance-design.md | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md diff --git a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md new file mode 100644 index 0000000000..de352a374d --- /dev/null +++ b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md @@ -0,0 +1,234 @@ +# Design: Zarr v3 metadata model conformance + +**Status:** Approved design; implementation pending. +**Scope:** `packages/zarr-metadata`, principally the raw v3 metadata types, +the immutable model layer, and their validators and tests. +**Normative authority:** Zarr core protocol v3.1. The behavior of zarrs and +zarr-python is interoperability evidence where the specification leaves an +implementation boundary or where a documented legacy-read exception is +required. + +## Goal + +Make the v3 array and group metadata models conform to the core Zarr v3.1 +document grammar without turning the generic models into interpreters for +individual extensions. + +The generic layer will validate required and optional core fields, JSON value +and container types, fixed literals and core cross-field constraints, the +common extension envelope, the core rules governing `must_understand`, and the +forward-compatibility obligations for unknown top-level fields. + +It will not validate the configuration semantics of a data type, chunk grid, +chunk key encoding, codec, or storage transformer. Those belong to typed +extension models and readers that resolve extension names. + +## Sources and precedence + +The implementation will be checked against: + +1. [Zarr core protocol v3.1](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html), + especially Array metadata, Group metadata, Codecs, and Extensions. +2. [zarrs](https://github.com/zarrs/zarrs), especially its `MetadataV3`, + `AdditionalFieldV3`, array-opening validation, and consolidated-metadata + extension handling. +3. Existing zarr-python metadata emitted in the wild, but only for explicit + tolerant-read paths. Legacy data does not redefine what this package emits + or calls conformant. + +If these disagree, the current v3.1 specification wins. An interoperability +exception must be narrow, documented as non-conformant input, normalized on +read, and never emitted. + +## Chosen approach + +Evolve the existing normalized `NamedConfigModelV3` instead of introducing a +new hierarchy of role-specific extension classes or preserving every source +JSON spelling. + +This keeps the public shape small and minimizes compatibility churn. +Validation remains context-aware: the same extension envelope is used at all +extension points, while the array-document validator supplies the few rules +that vary by field. + +Two alternatives were rejected: + +- A spelling-preserving model would retain shorthand versus object form and + absent versus empty configuration. That fidelity is not needed by this + semantic model and would substantially change its API. +- Separate required/optional extension classes would encode + `must_understand` restrictions in types, but would duplicate behavior and + force callers to use different classes for otherwise identical envelopes. + +## Extension envelope + +### Raw type + +`NamedConfigV3` will describe the v3.1 object form: + +- required `name: str`; +- optional `configuration: Mapping[str, JSONValue]`; and +- optional `must_understand: bool`. + +`MetadataV3` remains the union of a shorthand name string and this object +form. + +The object form accepts only those three members. Extension-owned fields must +be placed inside `configuration`. Rejecting other envelope members matches +zarrs' `deny_unknown_fields` behavior and prevents silently discarding data +during normalization. + +### Normalized model + +`NamedConfigModelV3` will hold: + +- `name: str`; +- `configuration: dict[str, JSONValue]`, normalized to an empty mapping when + absent; and +- `must_understand: bool`, normalized to `True` when absent or when the input + is a shorthand string. + +`to_json()` will continue emitting the canonical object form with a +configuration mapping. It will omit `must_understand` when true and emit +`"must_understand": false` when false. Thus the model is semantically +lossless, but deliberately not spelling-preserving. + +### Name validation + +Names will be checked syntactically without attempting to query or freeze the +external extension registry: + +- registered-name syntax is `^[a-z][a-z0-9-_.]+$`; and +- legacy URI names remain accepted for v3.0 compatibility. + +Registry membership is not a structural property that an extension-agnostic +offline model can determine. Extension resolution remains a reader concern. + +## Context-sensitive `must_understand` rules + +An omitted `must_understand` value and every shorthand name mean `True`. + +The array validator will reject `must_understand=False` for `data_type`, +`chunk_grid`, and `chunk_key_encoding`. It will permit false for individual +codecs and storage transformers. + +This validation is structural only. The model does not decide whether an +implementation recognizes a name and does not remove or skip optional +extensions. A reader that resolves extensions owns that decision. + +## Core array and group constraints + +The generic array model will enforce the core constraints that do not require +interpreting an extension: + +- `zarr_format` is exactly `3` and `node_type` is exactly `"array"`; +- `shape` contains non-negative JSON integers (booleans excluded); +- all mandatory fields are present; +- `fill_value` is JSON, while its data-type-dependent meaning remains opaque; +- `codecs` is a non-empty sequence of valid extension envelopes; +- `storage_transformers`, when present, is a sequence of valid envelopes; +- `attributes`, when present, is a string-keyed JSON object; and +- `dimension_names`, when present, contains strings or null and has the same + length as `shape`. + +The model will not enforce regular-grid rank or positive chunk lengths, +data-type-specific fill values, codec kinds or ordering, or storage-transformer +behavior. These rules require resolving an extension name and therefore live +outside the generic model. + +The generic group model will enforce the corresponding core document rules: +fixed literals, required fields, JSON attributes, string top-level keys, and +unknown-field handling. + +## Unknown top-level fields + +Unknown array and group members will be represented as arbitrary `JSONValue`, +not as a TypedDict that requires an object. + +For reader obligations: + +- an object whose `must_understand` member is the literal JSON boolean `false` + is explicitly waivable; and +- every other value, including scalars, arrays, objects with no such member, + and objects with a non-boolean value, implicitly requires understanding. + +This matches the v3.1 implicit-true rule and zarrs' `AdditionalFieldV3` +behavior. The raw and model annotations, `is_*` guards, parsers, and +`must_understand_fields` property must all agree on this representation. + +The model itself will not fail merely because such a field requires +understanding. It has no extension registry. It exposes the partition so the +reader can fail when a required field is unrecognized. + +## Consolidated metadata + +Inline consolidated metadata is a known interoperability extension rather +than a core v3.1 group field. Its dedicated raw and model types will remain, +and group parsing may continue recognizing it explicitly. + +The standard emitted form is an object containing `kind`, `metadata`, and a +`must_understand` marker. The extension model validates its own payload and +recursively validates embedded v3 nodes. This special handling must not cause +the generic unknown-field type to become object-only. + +Historical zarr-python versions wrote `"consolidated_metadata": null`. +zarrs also accepts this input as a compatibility hotfix. `from_json()` will +continue repairing that exact value to absence, and `to_json()` will never +write it. Documentation and tests will identify this as a tolerant-read +exception, not valid core metadata. + +## Errors and normalization + +All newly enforced constraints use the existing aggregated +`MetadataValidationError`/`ValidationProblem` mechanism. Locations must point +to the envelope member or array entry that failed, and problem kinds remain +machine-readable. + +Parsing still recursively converts JSON arrays to the tuple-backed public +representation. Validation and type guards must agree on canonical runtime +containers after normalization. + +No constructor-wide semantic validation will be added to `update()`; +`from_json()` remains the validated ingestion boundary. Direct construction +and `update()` continue to support building or repairing intermediate models. + +## Public API compatibility + +The existing class names remain. The intended public changes are additive or +corrections to annotations that did not match accepted runtime data: + +- `NamedConfigV3` gains optional `must_understand`; +- `NamedConfigModelV3` gains a defaulted `must_understand` field; and +- v3 additional-field annotations widen to arbitrary JSON. + +Documents currently accepted only because an extension envelope contains +unknown members, an invalid name, a forbidden false `must_understand`, or an +empty codec list will become validation errors. Those documents violate the +agreed core grammar; accepting them is not a compatibility contract to retain. + +## Test strategy + +Implementation will be test-first and cover: + +1. shorthand and object normalization, including implicit true; +2. explicit true/false round trips; +3. invalid `must_understand` types and unknown envelope members; +4. registered and legacy URI name forms plus invalid names; +5. false `must_understand` rejected at mandatory extension points; +6. false accepted for codecs and storage transformers; +7. empty codecs rejected without interpreting non-empty pipelines; +8. arbitrary JSON unknown top-level fields and their obligation partition; +9. raw type, parser, type-guard, model, and serializer agreement; +10. consolidated-metadata parsing and the legacy-null read repair; and +11. representative metadata serialized by zarrs. + +The package test suite, formatting, linting, and static type checking will run +before the implementation commit. + +## Completion criteria + +The work is complete when every normative core-document rule in scope has an +explicit validator or a documented extension-owned boundary; raw annotations +and runtime behavior agree; tests demonstrate `must_understand` at every +extension point; zarrs-compatible metadata round-trips semantically; and the +package tests and quality checks pass. From 55d329769827e453230b8849669b043bb8867b5e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 14:54:22 +0200 Subject: [PATCH 38/48] fix(zarr-metadata): preserve v3 extension obligations Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/_common.py | 1 + .../src/zarr_metadata/model/_array.py | 28 +++++++--- .../src/zarr_metadata/model/_validation.py | 28 +++++++++- .../src/zarr_metadata/pydantic.py | 2 +- .../zarr-metadata/tests/model/test_array.py | 54 +++++++++++++++---- .../tests/model/test_pydantic_module.py | 11 +++- 6 files changed, 102 insertions(+), 22 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index b335cd0bd6..5c530fcebf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -34,3 +34,4 @@ class NamedConfigV3(TypedDict): name: str configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index aee4f80fa8..ed2767c851 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -23,7 +23,7 @@ ) if TYPE_CHECKING: - from zarr_metadata._common import JSONValue + from zarr_metadata._common import JSONValue, NamedConfigV3 from zarr_metadata.v2.array import ( ArrayDimensionSeparatorV2, ArrayMetadataV2, @@ -46,31 +46,43 @@ @dataclass(frozen=True, slots=True, kw_only=True) class NamedConfigModelV3: - """A v3 metadata field in normalized form: a name plus a configuration. + """A normalized v3 metadata field with its reader obligation. - This is the in-memory model of `MetadataV3` (a bare name string or a - `{name, configuration}` mapping): the bare-name and missing-configuration - forms normalize to an empty configuration. + Bare names and missing configurations normalize to an empty configuration. + Bare names and missing `must_understand` members normalize to the spec's + implicit `True` value. """ name: str configuration: dict[str, JSONValue] + must_understand: bool = True def to_json(self) -> MetadataV3: - return {"name": self.name, "configuration": self.configuration} + if not self.configuration and self.must_understand: + return self.name + out: NamedConfigV3 = {"name": self.name} + if self.configuration: + out["configuration"] = self.configuration + if not self.must_understand: + out["must_understand"] = False + return out @classmethod def from_json(cls, data: object) -> NamedConfigModelV3: field = parse_metadata_field_v3(data) if isinstance(field, str): - return cls(name=field, configuration={}) + return cls(name=field, configuration={}, must_understand=True) # Sound cast: parse_metadata_field_v3 checked the configuration is a # string-keyed mapping of JSON values; arrays_to_tuples only converts # lists to tuples within that shape. configuration = cast( "dict[str, JSONValue]", arrays_to_tuples(dict(field.get("configuration", {}))) ) - return cls(name=field["name"], configuration=configuration) + return cls( + name=field["name"], + configuration=configuration, + must_understand=field.get("must_understand", True), + ) MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index a05fe7a430..1c8834d3f2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -182,7 +182,9 @@ def _validate_extension_fields_v3( return problems -def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: +def validate_metadata_field_v3( + value: object, *, allow_must_understand_false: bool = True +) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. A metadata field is a bare name string or a `{name, configuration}` mapping. @@ -199,6 +201,16 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: ] field = cast("Mapping[object, object]", value) problems: list[ValidationProblem] = [] + allowed_keys = frozenset({"name", "configuration", "must_understand"}) + for key in field: + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string metadata field key {key!r}", "invalid_type") + ) + elif key not in allowed_keys: + problems.append( + ValidationProblem((key,), "unexpected metadata field member", "invalid_value") + ) if not isinstance(field.get("name"), str): problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) if "configuration" in field: @@ -214,6 +226,20 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: else: for key, item in cast("Mapping[str, object]", configuration).items(): problems.extend(_prefix("configuration", _prefix(key, validate_json(item)))) + if "must_understand" in field: + must_understand = field["must_understand"] + if not isinstance(must_understand, bool): + problems.append( + ValidationProblem(("must_understand",), "expected a boolean", "invalid_type") + ) + elif not allow_must_understand_false and not must_understand: + problems.append( + ValidationProblem( + ("must_understand",), + "false is not supported at this extension point", + "invalid_value", + ) + ) return problems diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 3eada76316..a19e192eb4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -115,7 +115,7 @@ def coerce(value: object) -> _M: MetadataFieldV3 = Annotated[ InstanceOf[NamedConfigModelV3], BeforeValidator(_coerce_to(NamedConfigModelV3, NamedConfigModelV3.from_json)), - PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), + PlainSerializer(NamedConfigModelV3.to_json, return_type=str | dict), WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), ] """Field type for one v3 metadata field (bare name string or name + configuration).""" diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index a17548634b..dc235fb2dd 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -132,18 +132,24 @@ def test_string_nan_fill_value_roundtrips() -> None: ), Expect( NamedConfigModelV3(name="bytes", configuration={}), - {"name": "bytes", "configuration": {}}, - id="without-configuration", + "bytes", + id="empty-configuration-shorthand", ), ] @pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, dict[str, object]]) -> None: - """NamedConfigModelV3.to_json emits the canonical object form.""" +def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, object]) -> None: + """NamedConfigModelV3.to_json emits the canonical extension form.""" assert case.input.to_json() == case.output +def test_zarr_metadata_v3_to_json_preserves_false_obligation() -> None: + """An empty optional extension stays an object so false is not lost.""" + model = NamedConfigModelV3(name="optional", configuration={}, must_understand=False) + assert model.to_json() == {"name": "optional", "must_understand": False} + + # --- NamedConfigModelV3.from_json ----------------------------------------------- ZARR_FROM_JSON_CASES = [ @@ -167,6 +173,12 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> assert NamedConfigModelV3.from_json(case.input) == case.output +def test_zarr_metadata_v3_from_json_preserves_false_obligation() -> None: + """Explicit false is represented on the normalized model.""" + model = NamedConfigModelV3.from_json({"name": "optional", "must_understand": False}) + assert model.must_understand is False + + # --- V3 baseline ----------------------------------------------------------- @@ -181,10 +193,10 @@ def test_v3_to_json_emits_canonical_document() -> None: "node_type": "array", "shape": (10,), "fill_value": 0, - "data_type": {"name": "int32", "configuration": {}}, + "data_type": "int32", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, - "codecs": ({"name": "bytes", "configuration": {}},), - "chunk_key_encoding": {"name": "default", "configuration": {}}, + "codecs": ("bytes",), + "chunk_key_encoding": "default", } @@ -233,7 +245,7 @@ def test_v3_single_storage_transformer_included() -> None: out: dict[str, object] = dict( ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() ) - assert out["storage_transformers"] == ({"name": "some_transformer", "configuration": {}},) + assert out["storage_transformers"] == ("some_transformer",) def test_v3_no_storage_transformers_omitted() -> None: @@ -639,11 +651,31 @@ def test_v2_roundtrip_json_model_json() -> None: def test_v3_parser_accepts_bare_string_data_type() -> None: """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" doc = ArrayMetadataModelV3.create_default().to_json() - doc["data_type"] = "int32" # bare-string form, not canonical object form + doc["data_type"] = "int32" model = ArrayMetadataModelV3.from_json(doc) - # parses correctly, re-serializes to canonical object form assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) - assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} + assert model.to_json()["data_type"] == "int32" + + +@pytest.mark.parametrize("name", ["bytes", "ANY string", "urn:example:codec"]) +def test_metadata_field_accepts_any_string_name(name: str) -> None: + """The structural layer checks the name type, not syntax or registration.""" + assert validate_metadata_field_v3({"name": name}) == [] + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: + """must_understand is a JSON boolean, not a truthy scalar.""" + problems = validate_metadata_field_v3({"name": "x", "must_understand": value}) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("must_understand",), "invalid_type") + ] + + +def test_metadata_field_rejects_unknown_envelope_member() -> None: + """Unknown envelope keys cannot be silently discarded during normalization.""" + problems = validate_metadata_field_v3({"name": "x", "typo": 1}) + assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "invalid_value")] def test_v2_roundtrip_with_compressor_and_filters() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 81b9b962af..9ddbc86ded 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -65,7 +65,7 @@ def test_field_type_validates_and_dumps_canonically( model = adapter.validate_python(doc) assert type(model) is model_cls assert adapter.validate_python(model) is model - assert adapter.dump_python(model) == dict(model.to_json()) + assert adapter.dump_python(model) == model.to_json() def test_core_instances_interoperate() -> None: @@ -115,6 +115,15 @@ class Manifest(BaseModel): assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest +def test_metadata_field_serializes_shorthand_and_false_object() -> None: + """The optional integration exposes the core model's canonical extension form.""" + adapter = TypeAdapter(zmp.MetadataFieldV3) + assert adapter.dump_python(adapter.validate_python({"name": "bytes"})) == "bytes" + assert adapter.dump_python( + adapter.validate_python({"name": "optional", "must_understand": False}) + ) == {"name": "optional", "must_understand": False} + + def test_core_package_does_not_import_pydantic() -> None: """Importing zarr_metadata (in a fresh interpreter) must not import pydantic: the integration is opt-in via zarr_metadata.pydantic.""" From a2167f1b219a51c7a662b5a23f634fdfa1701faf Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 14:55:43 +0200 Subject: [PATCH 39/48] fix(zarr-metadata): enforce v3 core extension rules Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/model/_validation.py | 13 ++++++++- .../zarr-metadata/tests/model/test_array.py | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 1c8834d3f2..faaaa19511 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -449,13 +449,24 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): if key in doc: - problems.extend(_prefix(key, validate_metadata_field_v3(doc[key]))) + problems.extend( + _prefix( + key, + validate_metadata_field_v3(doc[key], allow_must_understand_false=False), + ) + ) for key in ("codecs", "storage_transformers"): if key in doc: entries = doc[key] if isinstance(entries, str) or not isinstance(entries, Sequence): problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) else: + if key == "codecs" and len(entries) == 0: + problems.append( + ValidationProblem( + ("codecs",), "expected at least one codec", "invalid_value" + ) + ) for index, entry in enumerate(cast("Sequence[object]", entries)): problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) if "attributes" in doc: diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index dc235fb2dd..fa732c6786 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -678,6 +678,33 @@ def test_metadata_field_rejects_unknown_envelope_member() -> None: assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "invalid_value")] +@pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) +def test_optional_extension_points_allow_must_understand_false(field: str) -> None: + """Codecs and storage transformers may be explicitly ignorable.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[field] = ({"name": "optional", "must_understand": False},) + assert validate_array_metadata_v3(doc) == [] + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_required_extension_points_reject_must_understand_false(field: str) -> None: + """Core extension points needed to locate or decode chunks cannot be ignored.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[field] = {"name": "optional", "must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((field, "must_understand"), "invalid_value") + ] + + +def test_v3_codecs_cannot_be_empty() -> None: + """The core document requires at least one array-to-bytes codec.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc["codecs"] = () + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("codecs",), "invalid_value") + ] + + def test_v2_roundtrip_with_compressor_and_filters() -> None: # Non-None compressor/filters must round-trip; extra assertion on .compressor. """A v2 model with non-None compressor and filters round-trips.""" From 5bc85320cfe1cb7619de6e0f48b71979de19daaf Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 14:57:47 +0200 Subject: [PATCH 40/48] fix(zarr-metadata): align v3 additional field types Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/model/_validation.py | 7 +--- .../src/zarr_metadata/v3/array.py | 40 ++++--------------- .../src/zarr_metadata/v3/group.py | 2 +- .../zarr-metadata/tests/model/test_array.py | 2 + .../zarr-metadata/tests/model/test_group.py | 8 ++++ 5 files changed, 20 insertions(+), 39 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index faaaa19511..491eb30aca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -356,14 +356,9 @@ def _is_canonical_array_metadata_v3(value: object) -> bool: _is_canonical_metadata_field_v3(item) for item in cast("tuple[object, ...]", doc["codecs"]) ): return False - if "storage_transformers" in doc and not all( + return "storage_transformers" not in doc or all( _is_canonical_metadata_field_v3(item) for item in cast("tuple[object, ...]", doc["storage_transformers"]) - ): - return False - return all( - key in ARRAY_METADATA_STANDARD_KEYS_V3 or isinstance(item, dict) - for key, item in doc.items() ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index a3fec24a6c..3226707665 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -1,51 +1,27 @@ """Zarr v3 array metadata types.""" from collections.abc import Mapping -from typing import Literal, NotRequired +from typing import Literal, NotRequired, TypeAlias from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.v3._common import MetadataV3 +ExtensionFieldV3: TypeAlias = JSONValue +"""The JSON value of an unknown top-level v3 metadata field. -class ExtensionFieldV3(TypedDict, extra_items=JSONValue): - """ - Required shape of any extension field on a v3 metadata document. - - The Zarr v3 spec permits extra keys on array and group metadata - documents, provided each value is an object with a `must_understand` - boolean key. This TypedDict captures that constraint and is used as - the `extra_items=` parameter on `ArrayMetadataV3` and `GroupMetadataV3`. - - `must_understand` is typed as `bool` rather than `Literal[False]` so - that applications which understand a particular extension can produce - or consume it with `must_understand: true` (signalling that readers - that don't recognize the extension MUST refuse to open the document). - The common case is still `false`, signalling that unknown readers may - safely ignore the field. - - Spec interpretation: this type follows the original Zarr v3.0 reading - of the spec, under which any object with a `must_understand` key is a - valid extension field. The v3.1 spec rewrite added language requiring - extension fields to also include a `name: str` key (the "Extension - definition" form). Under the strict v3.1 reading, real-world extension - fields written by zarr-python and zarrs (notably `consolidated_metadata`, - which has no `name` field) are out of spec. The community consensus at - the time of writing is that this is a regression to be reverted; this - package models the v3.0 / pre-revert interpretation. See - https://github.com/zarr-developers/zarr-specs/issues/371 for the - ongoing discussion. - """ - - must_understand: bool +An object carrying the literal member `must_understand: false` may be ignored. +Every other JSON shape implicitly requires understanding; recognition itself +belongs to the reader rather than this structural type. +""" class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 array metadata document (the `zarr.json` content for an array). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys may contain arbitrary JSON values. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#array-metadata """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index be990b1ae7..0593896f37 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -16,7 +16,7 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 group metadata document (the `zarr.json` content for a group). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys may contain arbitrary JSON values. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#group-metadata """ diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index fa732c6786..f6b08fdcf1 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1331,6 +1331,8 @@ def test_v3_json_extension_without_waiver_is_preserved_as_must_understand() -> N """A JSON extension without an explicit false waiver remains must-understand.""" doc = dict(ArrayMetadataModelV3.create_default().to_json()) doc["ext"] = 1 + parsed = parse_array_metadata_v3(doc) + assert is_array_metadata_v3(parsed) model = ArrayMetadataModelV3.from_json(doc) assert model.extra_fields["ext"] == 1 assert model.must_understand_fields == {"ext": 1} diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 08adb19d40..fbbcbcb875 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -58,6 +58,14 @@ def test_group_v3_extra_fields_roundtrip() -> None: assert model.to_json() == doc +def test_group_v3_json_extra_field_roundtrips_as_must_understand() -> None: + """A non-object extra field is preserved and implicitly requires understanding.""" + doc = {"zarr_format": 3, "node_type": "group", "ext": [1, 2]} + model = GroupMetadataModelV3.from_json(doc) + assert model.to_json()["ext"] == (1, 2) + assert model.must_understand_fields == {"ext": (1, 2)} + + def test_group_v3_extra_fields_overlap_rejected() -> None: """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" with pytest.raises(ValueError, match="Extra fields"): From fbaf9ad959a0f6de40efe71e27be447deee623f5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 15:04:08 +0200 Subject: [PATCH 41/48] chore(zarr-metadata): finalize v3 model conformance Assisted-by: Codex:gpt-5 --- ...7-22-zarr-v3-metadata-model-conformance.md | 398 ++++++++++++++++++ ...rr-v3-metadata-model-conformance-design.md | 30 +- .../src/zarr_metadata/_common.py | 4 +- .../src/zarr_metadata/model/_array.py | 14 +- .../src/zarr_metadata/model/_group.py | 8 +- .../src/zarr_metadata/model/_validation.py | 7 +- .../src/zarr_metadata/pydantic.py | 2 +- .../src/zarr_metadata/v3/_common.py | 2 +- .../src/zarr_metadata/v3/consolidated.py | 12 +- .../tests/model/test_pydantic.py | 14 +- 10 files changed, 446 insertions(+), 45 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md diff --git a/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md b/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md new file mode 100644 index 0000000000..98d1cd896c --- /dev/null +++ b/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md @@ -0,0 +1,398 @@ +# Zarr v3 Metadata Model Conformance Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the extension-agnostic v3 metadata models preserve Zarr 3.1 extension envelopes, enforce core envelope/cardinality rules, and keep raw types, parsers, guards, and serializers consistent. + +**Architecture:** Keep one normalized `NamedConfigModelV3` for every extension point. It parses shorthand and object forms, stores opaque configuration plus `must_understand`, and applies one field-independent shorthand heuristic. The array validator supplies only core rules that vary by extension point; it never resolves an extension name or configuration. + +**Tech Stack:** Python 3.11+, frozen dataclasses, `TypedDict`/PEP 728, pytest, Ruff, Pyright, and optional Pydantic v2 integration. + +## Global Constraints + +- Zarr v3.1 is normative; zarrs is interoperability evidence. +- Validate extension names only as strings; do not check syntax or registry membership. +- Do not interpret extension configurations, fill values, or codec kinds/order. +- Reject `must_understand=False` for data types, chunk grids, and chunk-key encodings. +- Require a non-empty codec list. +- Preserve the `consolidated_metadata: null` tolerant-read repair and never emit null. +- Observe each new behavioral test fail before changing production code. +- Commits must be conventional and include `Assisted-by: Codex:gpt-5`. + +--- + +### Task 1: Preserve and canonically serialize extension envelopes + +**Files:** +- Modify: `packages/zarr-metadata/tests/model/test_array.py` +- Modify: `packages/zarr-metadata/tests/model/test_pydantic_module.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/_common.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/pydantic.py` + +**Interfaces:** +- Produces: `NamedConfigV3` with optional `must_understand: bool`. +- Produces: `NamedConfigModelV3(name: str, configuration: dict[str, JSONValue], must_understand: bool = True)`. +- Produces: `validate_metadata_field_v3(value: object, *, allow_must_understand_false: bool = True) -> list[ValidationProblem]`. + +- [x] **Step 1: Write failing normalized-model tests** + +Update the existing case tables and canonical-document expectation: + +```python +ZARR_TO_JSON_CASES = [ + Expect( + NamedConfigModelV3(name="regular", configuration={"chunk_shape": [1]}), + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + id="with-configuration", + ), + Expect( + NamedConfigModelV3(name="bytes", configuration={}), + "bytes", + id="empty-configuration-shorthand", + ), + Expect( + NamedConfigModelV3(name="optional", configuration={}, must_understand=False), + {"name": "optional", "must_understand": False}, + id="false-obligation-needs-object", + ), +] +``` + +Add `from_json` cases for implicit true and explicit false. Expect the default +array document to contain `"uint8"`, `("bytes",)`, and `"default"`, while its +configured regular grid remains an object. + +- [x] **Step 2: Run the focused tests and verify RED** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + -k 'zarr_metadata_v3 or canonical_document or explicit_false' -q +``` + +Expected: empty configurations remain objects, the constructor lacks +`must_understand`, or explicit false is lost. + +- [x] **Step 3: Write failing envelope-validation tests** + +```python +@pytest.mark.parametrize("name", ["bytes", "ANY string", "urn:example:codec"]) +def test_metadata_field_accepts_any_string_name(name: str) -> None: + assert validate_metadata_field_v3({"name": name}) == [] + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: + problems = validate_metadata_field_v3({"name": "x", "must_understand": value}) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("must_understand",), "invalid_type") + ] + + +def test_metadata_field_rejects_unknown_envelope_member() -> None: + problems = validate_metadata_field_v3({"name": "x", "typo": 1}) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("typo",), "invalid_value") + ] +``` + +- [x] **Step 4: Run the envelope tests and verify RED** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + -k 'metadata_field_accepts or must_understand_must_be or unknown_envelope' -q +``` + +Expected: invalid obligation values and the unknown member are accepted before the fix. + +- [x] **Step 5: Implement the raw type, validator, and normalized model** + +```python +class NamedConfigV3(TypedDict): + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] +``` + +Reject envelope keys outside `name`, `configuration`, and `must_understand`; +require a real boolean for the last member; keep all string names valid. + +```python +@dataclass(frozen=True, slots=True, kw_only=True) +class NamedConfigModelV3: + name: str + configuration: dict[str, JSONValue] + must_understand: bool = True + + def to_json(self) -> MetadataV3: + if not self.configuration and self.must_understand: + return self.name + out: NamedConfigV3 = {"name": self.name} + if self.configuration: + out["configuration"] = self.configuration + if not self.must_understand: + out["must_understand"] = False + return out +``` + +`from_json` sets true for shorthand/absence and preserves explicit false. + +- [x] **Step 6: Update and test the Pydantic serializer** + +Change `MetadataFieldV3`'s serializer return type from `dict` to `str | dict`. + +```python +def test_metadata_field_serializes_shorthand_and_false_object() -> None: + adapter = TypeAdapter(zmp.MetadataFieldV3) + assert adapter.dump_python(adapter.validate_python({"name": "bytes"})) == "bytes" + assert adapter.dump_python( + adapter.validate_python({"name": "optional", "must_understand": False}) + ) == {"name": "optional", "must_understand": False} +``` + +- [x] **Step 7: Run Task 1 tests and verify GREEN** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_pydantic_module.py -q +``` + +Expected: both files pass. + +- [x] **Step 8: Commit Task 1** + +```bash +git add packages/zarr-metadata/src/zarr_metadata/_common.py \ + packages/zarr-metadata/src/zarr_metadata/model/_array.py \ + packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ + packages/zarr-metadata/src/zarr_metadata/pydantic.py \ + packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_pydantic_module.py +git commit -m "fix(zarr-metadata): preserve v3 extension obligations" \ + -m "Assisted-by: Codex:gpt-5" +``` + +### Task 2: Enforce context-sensitive core array rules + +**Files:** +- Modify: `packages/zarr-metadata/tests/model/test_array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` + +**Interfaces:** +- Consumes: `validate_metadata_field_v3(..., allow_must_understand_false=...)`. +- Produces: rejection of forbidden false obligations and empty codecs. + +- [x] **Step 1: Write failing context and cardinality tests** + +```python +@pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) +def test_optional_extension_points_allow_must_understand_false(field: str) -> None: + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[field] = ({"name": "optional", "must_understand": False},) + assert validate_array_metadata_v3(doc) == [] + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_required_extension_points_reject_must_understand_false(field: str) -> None: + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[field] = {"name": "optional", "must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((field, "must_understand"), "invalid_value") + ] + + +def test_v3_codecs_cannot_be_empty() -> None: + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc["codecs"] = () + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("codecs",), "invalid_value") + ] +``` + +- [x] **Step 2: Run Task 2 tests and verify RED** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + -k 'optional_extension_points or required_extension_points or codecs_cannot' -q +``` + +Expected: required points and empty codecs are accepted before the fix. + +- [x] **Step 3: Implement the context rules** + +Call `validate_metadata_field_v3(..., allow_must_understand_false=False)` for +`data_type`, `chunk_grid`, and `chunk_key_encoding`. Leave codecs and storage +transformers at the default. After establishing that `codecs` is a sequence, +add one field-level `invalid_value` problem when it is empty. + +- [x] **Step 4: Run the complete array model tests and verify GREEN** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py -q +``` + +Expected: the file passes. + +- [x] **Step 5: Commit Task 2** + +```bash +git add packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ + packages/zarr-metadata/tests/model/test_array.py +git commit -m "fix(zarr-metadata): enforce v3 core extension rules" \ + -m "Assisted-by: Codex:gpt-5" +``` + +### Task 3: Align additional-field types, guards, and models + +**Files:** +- Modify: `packages/zarr-metadata/tests/model/test_array.py` +- Modify: `packages/zarr-metadata/tests/model/test_group.py` +- Modify: `packages/zarr-metadata/tests/test_partial_equivalence.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/group.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_group.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` + +**Interfaces:** +- Produces: public `ExtensionFieldV3` alias for arbitrary `JSONValue`. +- Produces: `extra_fields: dict[str, JSONValue]` and matching guards/parsers. + +- [x] **Step 1: Write failing parser/guard agreement tests** + +```python +def test_v3_scalar_extra_field_agrees_across_parser_guard_and_model() -> None: + raw = dict(ArrayMetadataModelV3.create_default().to_json()) | {"ext": 1} + parsed = parse_array_metadata_v3(raw) + assert is_array_metadata_v3(parsed) + model = ArrayMetadataModelV3.from_json(raw) + assert model.extra_fields == {"ext": 1} + assert model.must_understand_fields == {"ext": 1} + + +def test_group_v3_scalar_extra_field_roundtrips_as_must_understand() -> None: + raw = {"zarr_format": 3, "node_type": "group", "ext": [1, 2]} + model = GroupMetadataModelV3.from_json(raw) + assert model.to_json()["ext"] == (1, 2) + assert model.must_understand_fields == {"ext": (1, 2)} +``` + +- [x] **Step 2: Run the agreement tests and verify RED** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_group.py -k 'scalar_extra_field' -q +``` + +Expected: the canonical array guard rejects the parsed scalar extra field. + +- [x] **Step 3: Correct raw/model annotations and the guard** + +Replace the object-shaped `ExtensionFieldV3` TypedDict with a public alias to +`JSONValue`; use it as `extra_items` on full and partial array/group types. +Update model partials, dataclass fields, helper signatures, and casts to +`dict[str, JSONValue]`. Remove the canonical array guard's requirement that +every non-standard value be a dict; JSON validation already establishes the +domain and `arrays_to_tuples` establishes canonical sequences. + +- [x] **Step 4: Run selected integration tests and verify GREEN** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_group.py \ + packages/zarr-metadata/tests/test_partial_equivalence.py \ + packages/zarr-metadata/tests/test_public_api.py -q +``` + +Expected: all selected files pass and `ExtensionFieldV3` remains exported. + +- [x] **Step 5: Commit Task 3** + +```bash +git add packages/zarr-metadata/src/zarr_metadata/v3/array.py \ + packages/zarr-metadata/src/zarr_metadata/v3/group.py \ + packages/zarr-metadata/src/zarr_metadata/model/_array.py \ + packages/zarr-metadata/src/zarr_metadata/model/_group.py \ + packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ + packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_group.py \ + packages/zarr-metadata/tests/test_partial_equivalence.py +git commit -m "fix(zarr-metadata): align v3 additional field types" \ + -m "Assisted-by: Codex:gpt-5" +``` + +### Task 4: Documentation reconciliation and full verification + +**Files:** +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/_common.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/__init__.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_group.py` +- Modify: `docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md` +- Create: `docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md` + +**Interfaces:** +- Consumes: all prior tasks. +- Produces: documentation matching semantic normalization and the extension-agnostic boundary. + +- [x] **Step 1: Reconcile documentation** + +Remove object-only and v3.0-over-v3.1 claims. Document the exact heuristic: + +```text +empty configuration + must_understand true -> shorthand string +otherwise -> object containing name and non-default members +``` + +Document arbitrary JSON additional fields, type-only name validation, and the +known non-core consolidated extension. Retain the historical null repair. + +- [x] **Step 2: Run the complete package suite** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests -q +``` + +Expected: zero failures. + +- [x] **Step 3: Run format, lint, typing, and whitespace checks** + +```bash +.venv/bin/ruff format --check packages/zarr-metadata +.venv/bin/ruff check packages/zarr-metadata +cd packages/zarr-metadata +uvx --from pyright==1.1.404 pyright --pythonpath .venv/bin/python src +git diff --check +``` + +Expected: every command exits zero with no diagnostics. + +- [x] **Step 4: Audit scope against the final diff** + +Confirm there is no registry lookup, name-syntax validation, extension +configuration interpretation, fill-value semantics, or codec-kind/order +validation. Confirm `consolidated_metadata: null` is still accepted on read +and never emitted. + +- [x] **Step 5: Commit docs and final integration adjustments** + +```bash +git add -f docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md \ + docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md +git add packages/zarr-metadata +git commit -m "docs(zarr-metadata): document v3 model conformance" \ + -m "Assisted-by: Codex:gpt-5" +``` + +- [x] **Step 6: Verify committed state** + +```bash +git status --short --branch +git log -4 --format='%h %s%n%(trailers:key=Assisted-by,valueonly)' +``` + +Expected: only the user's pre-existing untracked files remain; every new +commit has a conventional subject and `Assisted-by: Codex:gpt-5`. diff --git a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md index de352a374d..49e54b2888 100644 --- a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md +++ b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md @@ -1,6 +1,6 @@ # Design: Zarr v3 metadata model conformance -**Status:** Approved design; implementation pending. +**Status:** Approved and implemented. **Scope:** `packages/zarr-metadata`, principally the raw v3 metadata types, the immutable model layer, and their validators and tests. **Normative authority:** Zarr core protocol v3.1. The behavior of zarrs and @@ -88,21 +88,21 @@ during normalization. - `must_understand: bool`, normalized to `True` when absent or when the input is a shorthand string. -`to_json()` will continue emitting the canonical object form with a -configuration mapping. It will omit `must_understand` when true and emit -`"must_understand": false` when false. Thus the model is semantically -lossless, but deliberately not spelling-preserving. +`to_json()` will use one field-independent canonicalization rule. When the +configuration is empty and `must_understand` is true, it emits the shorthand +name string. Otherwise it emits an object, omitting `configuration` when empty +and omitting `must_understand` when true. Thus the model is semantically +lossless, but deliberately not spelling-preserving. The same rule naturally +emits core data types as strings without giving the data-type field a separate +model or serializer. ### Name validation -Names will be checked syntactically without attempting to query or freeze the -external extension registry: - -- registered-name syntax is `^[a-z][a-z0-9-_.]+$`; and -- legacy URI names remain accepted for v3.0 compatibility. - -Registry membership is not a structural property that an extension-agnostic -offline model can determine. Extension resolution remains a reader concern. +Names will be checked only to ensure that they are strings. The model will not +validate registered-name syntax, URI syntax, or membership in the external +extension registry. Those properties are not part of the extension-agnostic +structural boundary and are left to applications that resolve extension +names. ## Context-sensitive `must_understand` rules @@ -202,7 +202,7 @@ corrections to annotations that did not match accepted runtime data: - v3 additional-field annotations widen to arbitrary JSON. Documents currently accepted only because an extension envelope contains -unknown members, an invalid name, a forbidden false `must_understand`, or an +unknown members, a non-string name, a forbidden false `must_understand`, or an empty codec list will become validation errors. Those documents violate the agreed core grammar; accepting them is not a compatibility contract to retain. @@ -213,7 +213,7 @@ Implementation will be test-first and cover: 1. shorthand and object normalization, including implicit true; 2. explicit true/false round trips; 3. invalid `must_understand` types and unknown envelope members; -4. registered and legacy URI name forms plus invalid names; +4. arbitrary string names accepted and non-string names rejected; 5. false `must_understand` rejected at mandatory extension points; 6. false accepted for codecs and storage transformers; 7. empty codecs rejected without interpreting non-empty pipelines; diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 5c530fcebf..4e8d974ab9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -28,8 +28,8 @@ class NamedConfigV3(TypedDict): """ Externally-tagged union member for a metadata field. - The `configuration` mapping holds arbitrary JSON-encodable values; - it is typed as `Mapping[str, JSONValue]`. + The optional `configuration` mapping holds arbitrary JSON-encodable + values. `must_understand` is implicitly true when absent. """ name: str diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index ed2767c851..d1e151053a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -91,9 +91,10 @@ def from_json(cls, data: object) -> NamedConfigModelV3: This is the role-named alias for annotation positions: model fields and consumer signatures should say `MetadataFieldModelV3` (the logical meaning) rather than `NamedConfigModelV3` (the serialized form the field currently -takes). Today every metadata field normalizes to a named configuration, so +takes). Today every metadata field normalizes to a named configuration plus +its reader obligation, so the alias is exactly `NamedConfigModelV3`; if a future spec revision adds a -field form that cannot be normalized to name + configuration, this alias +field form that cannot be normalized to those values, this alias widens to a union and annotation sites do not change. Mirrors the raw-layer split between `NamedConfigV3` (shape) and `MetadataV3` (field union). """ @@ -156,12 +157,13 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): class ArrayMetadataModelV3: """In-memory model of a v3 array metadata document. - A canonical, lossless representation of the `zarr.json` content for an + A canonical, semantically lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, `codecs`, `storage_transformers`) are held as `MetadataFieldModelV3` - values (currently always `NamedConfigModelV3` name + configuration pairs) - and are never interpreted; `fill_value` is held - verbatim in its JSON form. + values (currently `NamedConfigModelV3` name, configuration, and obligation + records) and are never interpreted; `fill_value` is held verbatim in its + JSON form. Equivalent extension spellings normalize to shorthand strings + when configuration is empty and understanding is required. """ zarr_format: Literal[3] = field(default=3, init=False) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 81bbbad270..5a545a23bf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -131,10 +131,7 @@ def to_json(self) -> GroupMetadataV3: if len(self.attributes) > 0: out["attributes"] = self.attributes if self.consolidated_metadata is not UNSET: - # The consolidated-metadata shape ({kind, must_understand, metadata}, - # no `name`) predates the strict v3.1 extension-field rules, so it is - # not assignable to `ExtensionFieldV3`; see the discussion on - # `zarr_metadata.v3.consolidated`. + # Consolidated metadata is a known non-core top-level JSON field. out[CONSOLIDATED_METADATA_KEY_V3] = cast( "ExtensionFieldV3", self.consolidated_metadata.to_json() ) @@ -145,8 +142,7 @@ def to_json(self) -> GroupMetadataV3: @classmethod def from_json(cls, data: object) -> GroupMetadataModelV3: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) - # Cast to object: the TypedDict's extra_items type does not admit null, - # but wild documents (historical zarr-python) contain it. + # Cast for narrowing across standard and arbitrary extra TypedDict items. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) consolidated: ConsolidatedMetadataModelV3 | UNSET if consolidated_raw is UNSET or consolidated_raw is None: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 491eb30aca..f087de6e90 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -187,7 +187,8 @@ def validate_metadata_field_v3( ) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. - A metadata field is a bare name string or a `{name, configuration}` mapping. + A metadata field is a bare name string or a mapping containing `name` and + optional `configuration` and `must_understand` members. """ if isinstance(value, str): return [] @@ -195,7 +196,7 @@ def validate_metadata_field_v3( return [ ValidationProblem( (), - "expected a metadata field (string or {name, configuration})", + "expected a metadata field (string or extension object)", "invalid_type", ) ] @@ -456,7 +457,7 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: if isinstance(entries, str) or not isinstance(entries, Sequence): problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) else: - if key == "codecs" and len(entries) == 0: + if key == "codecs" and len(cast("Sequence[object]", entries)) == 0: problems.append( ValidationProblem( ("codecs",), "expected at least one codec", "invalid_value" diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index a19e192eb4..2c15cf51ce 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -118,7 +118,7 @@ def coerce(value: object) -> _M: PlainSerializer(NamedConfigModelV3.to_json, return_type=str | dict), WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), ] -"""Field type for one v3 metadata field (bare name string or name + configuration).""" +"""Field type for one normalized v3 metadata extension envelope.""" __all__ = [ "ArrayMetadataV2", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py index 3424587a43..3e0d2b054c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py @@ -9,7 +9,7 @@ MetadataV3 = str | NamedConfigV3 """The JSON shape of any v3 metadata extension-point entry: either a bare -short-hand name string or a `{name, configuration}` envelope. +short-hand name string or a `{name, configuration, must_understand}` envelope. Used for `data_type`, `chunk_grid`, `chunk_key_encoding`, individual codec entries, and `storage_transformers` in v3 array metadata, and for diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index 486a0897a5..79d4b9692d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -5,14 +5,10 @@ implementation (and zarrs), where consolidated metadata is embedded as an extension field on a group's `zarr.json`. -The shape modeled here (`{kind, must_understand, metadata}` with no `name` -field) reflects the original Zarr v3.0 reading of the extension-field -rules. Under the strict Zarr v3.1 reading, every extension field must -also include a `name: str` key, which would make this shape — and every -real-world consolidated metadata document in the wild — out of spec. -See `ExtensionFieldV3` and -https://github.com/zarr-developers/zarr-specs/issues/371 for the -ongoing discussion. +This is a known non-core interoperability extension. Its +`{kind, must_understand, metadata}` payload is an unknown top-level JSON value +to the core document model; implementations that recognize the convention may +interpret it through this dedicated type. """ from collections.abc import Mapping diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 7e80d52ee8..6d9b82b3e3 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -121,8 +121,8 @@ def test_dump_emits_canonical_document() -> None: manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) dumped = manifest.model_dump() assert dumped["metadata"] == manifest.metadata.to_json() - # the bare-string data_type was normalized to the canonical object form - assert dumped["metadata"]["data_type"] == {"name": "uint8", "configuration": {}} + # Empty configurations use the extension-definition shorthand form. + assert dumped["metadata"]["data_type"] == "uint8" def test_json_roundtrip_through_pydantic() -> None: @@ -194,10 +194,11 @@ def build_and_use() -> None: class NamedConfig(BaseModel): - """Pydantic mirror of a normalized metadata field (name + configuration).""" + """Pydantic mirror of a normalized metadata extension envelope.""" name: str configuration: dict[str, JSONValue] = {} + must_understand: bool = True class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): @@ -229,6 +230,13 @@ def _canonicalize(cls, data: object) -> object: normalization; pydantic then parses only canonical documents.""" if isinstance(data, Mapping): doc = dict(ArrayMetadataModelV3.from_json(data).to_json()) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if isinstance(doc[key], str): + doc[key] = {"name": doc[key]} + for key in ("codecs", "storage_transformers"): + doc[key] = tuple( + {"name": item} if isinstance(item, str) else item for item in doc.get(key, ()) + ) doc.setdefault("attributes", {}) return doc return data From 7a88406d42ad877ec3ad89cc6d417457c2309d9d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 15:35:33 +0200 Subject: [PATCH 42/48] docs: remove llm docs --- ...7-22-zarr-v3-metadata-model-conformance.md | 398 ------------------ ...rr-v3-metadata-model-conformance-design.md | 234 ---------- 2 files changed, 632 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md delete mode 100644 docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md diff --git a/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md b/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md deleted file mode 100644 index 1bf8fec1cf..0000000000 --- a/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md +++ /dev/null @@ -1,398 +0,0 @@ -# Zarr v3 Metadata Model Conformance Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the extension-agnostic v3 metadata models preserve Zarr 3.1 extension envelopes, enforce core envelope/cardinality rules, and keep raw types, parsers, guards, and serializers consistent. - -**Architecture:** Keep one normalized `ZarrV3NamedConfig` for every extension point. It parses shorthand and object forms, stores opaque configuration plus `must_understand`, and applies one field-independent shorthand heuristic. The array validator supplies only core rules that vary by extension point; it never resolves an extension name or configuration. - -**Tech Stack:** Python 3.11+, frozen dataclasses, `TypedDict`/PEP 728, pytest, Ruff, Pyright, and optional Pydantic v2 integration. - -## Global Constraints - -- Zarr v3.1 is normative; zarrs is interoperability evidence. -- Validate extension names only as strings; do not check syntax or registry membership. -- Do not interpret extension configurations, fill values, or codec kinds/order. -- Reject `must_understand=False` for data types, chunk grids, and chunk-key encodings. -- Require a non-empty codec list. -- Preserve the `consolidated_metadata: null` tolerant-read repair and never emit null. -- Observe each new behavioral test fail before changing production code. -- Commits must be conventional and include `Assisted-by: Codex:gpt-5`. - ---- - -### Task 1: Preserve and canonically serialize extension envelopes - -**Files:** -- Modify: `packages/zarr-metadata/tests/model/test_array.py` -- Modify: `packages/zarr-metadata/tests/model/test_pydantic_module.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/_common.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/pydantic.py` - -**Interfaces:** -- Produces: `ZarrV3NamedConfigJSON` with optional `must_understand: bool`. -- Produces: `ZarrV3NamedConfig(name: str, configuration: dict[str, JSONValue], must_understand: bool = True)`. -- Produces: `validate_metadata_field_v3(value: object, *, allow_must_understand_false: bool = True) -> list[ValidationProblem]`. - -- [x] **Step 1: Write failing normalized-model tests** - -Update the existing case tables and canonical-document expectation: - -```python -ZARR_TO_JSON_CASES = [ - Expect( - ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": [1]}), - {"name": "regular", "configuration": {"chunk_shape": [1]}}, - id="with-configuration", - ), - Expect( - ZarrV3NamedConfig(name="bytes", configuration={}), - "bytes", - id="empty-configuration-shorthand", - ), - Expect( - ZarrV3NamedConfig(name="optional", configuration={}, must_understand=False), - {"name": "optional", "must_understand": False}, - id="false-obligation-needs-object", - ), -] -``` - -Add `from_json` cases for implicit true and explicit false. Expect the default -array document to contain `"uint8"`, `("bytes",)`, and `"default"`, while its -configured regular grid remains an object. - -- [x] **Step 2: Run the focused tests and verify RED** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - -k 'zarr_metadata_v3 or canonical_document or explicit_false' -q -``` - -Expected: empty configurations remain objects, the constructor lacks -`must_understand`, or explicit false is lost. - -- [x] **Step 3: Write failing envelope-validation tests** - -```python -@pytest.mark.parametrize("name", ["bytes", "ANY string", "urn:example:codec"]) -def test_metadata_field_accepts_any_string_name(name: str) -> None: - assert validate_metadata_field_v3({"name": name}) == [] - - -@pytest.mark.parametrize("value", [0, 1, "false", None]) -def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: - problems = validate_metadata_field_v3({"name": "x", "must_understand": value}) - assert [(problem.loc, problem.kind) for problem in problems] == [ - (("must_understand",), "invalid_type") - ] - - -def test_metadata_field_rejects_unknown_envelope_member() -> None: - problems = validate_metadata_field_v3({"name": "x", "typo": 1}) - assert [(problem.loc, problem.kind) for problem in problems] == [ - (("typo",), "invalid_value") - ] -``` - -- [x] **Step 4: Run the envelope tests and verify RED** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - -k 'metadata_field_accepts or must_understand_must_be or unknown_envelope' -q -``` - -Expected: invalid obligation values and the unknown member are accepted before the fix. - -- [x] **Step 5: Implement the raw type, validator, and normalized model** - -```python -class ZarrV3NamedConfigJSON(TypedDict): - name: str - configuration: NotRequired[Mapping[str, JSONValue]] - must_understand: NotRequired[bool] -``` - -Reject envelope keys outside `name`, `configuration`, and `must_understand`; -require a real boolean for the last member; keep all string names valid. - -```python -@dataclass(frozen=True, slots=True, kw_only=True) -class ZarrV3NamedConfig: - name: str - configuration: dict[str, JSONValue] - must_understand: bool = True - - def to_json(self) -> ZarrV3MetadataFieldJSON: - if not self.configuration and self.must_understand: - return self.name - out: ZarrV3NamedConfigJSON = {"name": self.name} - if self.configuration: - out["configuration"] = self.configuration - if not self.must_understand: - out["must_understand"] = False - return out -``` - -`from_json` sets true for shorthand/absence and preserves explicit false. - -- [x] **Step 6: Update and test the Pydantic serializer** - -Change `ZarrV3MetadataField`'s serializer return type from `dict` to `str | dict`. - -```python -def test_metadata_field_serializes_shorthand_and_false_object() -> None: - adapter = TypeAdapter(zmp.ZarrV3MetadataField) - assert adapter.dump_python(adapter.validate_python({"name": "bytes"})) == "bytes" - assert adapter.dump_python( - adapter.validate_python({"name": "optional", "must_understand": False}) - ) == {"name": "optional", "must_understand": False} -``` - -- [x] **Step 7: Run Task 1 tests and verify GREEN** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_pydantic_module.py -q -``` - -Expected: both files pass. - -- [x] **Step 8: Commit Task 1** - -```bash -git add packages/zarr-metadata/src/zarr_metadata/_common.py \ - packages/zarr-metadata/src/zarr_metadata/model/_array.py \ - packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ - packages/zarr-metadata/src/zarr_metadata/pydantic.py \ - packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_pydantic_module.py -git commit -m "fix(zarr-metadata): preserve v3 extension obligations" \ - -m "Assisted-by: Codex:gpt-5" -``` - -### Task 2: Enforce context-sensitive core array rules - -**Files:** -- Modify: `packages/zarr-metadata/tests/model/test_array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` - -**Interfaces:** -- Consumes: `validate_metadata_field_v3(..., allow_must_understand_false=...)`. -- Produces: rejection of forbidden false obligations and empty codecs. - -- [x] **Step 1: Write failing context and cardinality tests** - -```python -@pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) -def test_optional_extension_points_allow_must_understand_false(field: str) -> None: - doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) - doc[field] = ({"name": "optional", "must_understand": False},) - assert validate_array_metadata_v3(doc) == [] - - -@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) -def test_required_extension_points_reject_must_understand_false(field: str) -> None: - doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) - doc[field] = {"name": "optional", "must_understand": False} - assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ - ((field, "must_understand"), "invalid_value") - ] - - -def test_v3_codecs_cannot_be_empty() -> None: - doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) - doc["codecs"] = () - assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ - (("codecs",), "invalid_value") - ] -``` - -- [x] **Step 2: Run Task 2 tests and verify RED** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - -k 'optional_extension_points or required_extension_points or codecs_cannot' -q -``` - -Expected: required points and empty codecs are accepted before the fix. - -- [x] **Step 3: Implement the context rules** - -Call `validate_metadata_field_v3(..., allow_must_understand_false=False)` for -`data_type`, `chunk_grid`, and `chunk_key_encoding`. Leave codecs and storage -transformers at the default. After establishing that `codecs` is a sequence, -add one field-level `invalid_value` problem when it is empty. - -- [x] **Step 4: Run the complete array model tests and verify GREEN** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py -q -``` - -Expected: the file passes. - -- [x] **Step 5: Commit Task 2** - -```bash -git add packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ - packages/zarr-metadata/tests/model/test_array.py -git commit -m "fix(zarr-metadata): enforce v3 core extension rules" \ - -m "Assisted-by: Codex:gpt-5" -``` - -### Task 3: Align additional-field types, guards, and models - -**Files:** -- Modify: `packages/zarr-metadata/tests/model/test_array.py` -- Modify: `packages/zarr-metadata/tests/model/test_group.py` -- Modify: `packages/zarr-metadata/tests/test_partial_equivalence.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/group.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_group.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` - -**Interfaces:** -- Produces: public `ZarrV3ExtensionField` alias for arbitrary `JSONValue`. -- Produces: `extra_fields: dict[str, JSONValue]` and matching guards/parsers. - -- [x] **Step 1: Write failing parser/guard agreement tests** - -```python -def test_v3_scalar_extra_field_agrees_across_parser_guard_and_model() -> None: - raw = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"ext": 1} - parsed = parse_array_metadata_v3(raw) - assert is_array_metadata_v3(parsed) - model = ZarrV3ArrayMetadata.from_json(raw) - assert model.extra_fields == {"ext": 1} - assert model.must_understand_fields == {"ext": 1} - - -def test_group_v3_scalar_extra_field_roundtrips_as_must_understand() -> None: - raw = {"zarr_format": 3, "node_type": "group", "ext": [1, 2]} - model = ZarrV3GroupMetadata.from_json(raw) - assert model.to_json()["ext"] == (1, 2) - assert model.must_understand_fields == {"ext": (1, 2)} -``` - -- [x] **Step 2: Run the agreement tests and verify RED** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_group.py -k 'scalar_extra_field' -q -``` - -Expected: the canonical array guard rejects the parsed scalar extra field. - -- [x] **Step 3: Correct raw/model annotations and the guard** - -Replace the object-shaped `ZarrV3ExtensionField` TypedDict with a public alias to -`JSONValue`; use it as `extra_items` on full and partial array/group types. -Update model partials, dataclass fields, helper signatures, and casts to -`dict[str, JSONValue]`. Remove the canonical array guard's requirement that -every non-standard value be a dict; JSON validation already establishes the -domain and `arrays_to_tuples` establishes canonical sequences. - -- [x] **Step 4: Run selected integration tests and verify GREEN** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_group.py \ - packages/zarr-metadata/tests/test_partial_equivalence.py \ - packages/zarr-metadata/tests/test_public_api.py -q -``` - -Expected: all selected files pass and `ZarrV3ExtensionField` remains exported. - -- [x] **Step 5: Commit Task 3** - -```bash -git add packages/zarr-metadata/src/zarr_metadata/v3/array.py \ - packages/zarr-metadata/src/zarr_metadata/v3/group.py \ - packages/zarr-metadata/src/zarr_metadata/model/_array.py \ - packages/zarr-metadata/src/zarr_metadata/model/_group.py \ - packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ - packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_group.py \ - packages/zarr-metadata/tests/test_partial_equivalence.py -git commit -m "fix(zarr-metadata): align v3 additional field types" \ - -m "Assisted-by: Codex:gpt-5" -``` - -### Task 4: Documentation reconciliation and full verification - -**Files:** -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/_common.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/__init__.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_group.py` -- Modify: `docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md` -- Create: `docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md` - -**Interfaces:** -- Consumes: all prior tasks. -- Produces: documentation matching semantic normalization and the extension-agnostic boundary. - -- [x] **Step 1: Reconcile documentation** - -Remove object-only and v3.0-over-v3.1 claims. Document the exact heuristic: - -```text -empty configuration + must_understand true -> shorthand string -otherwise -> object containing name and non-default members -``` - -Document arbitrary JSON additional fields, type-only name validation, and the -known non-core consolidated extension. Retain the historical null repair. - -- [x] **Step 2: Run the complete package suite** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests -q -``` - -Expected: zero failures. - -- [x] **Step 3: Run format, lint, typing, and whitespace checks** - -```bash -.venv/bin/ruff format --check packages/zarr-metadata -.venv/bin/ruff check packages/zarr-metadata -cd packages/zarr-metadata -uvx --from pyright==1.1.404 pyright --pythonpath .venv/bin/python src -git diff --check -``` - -Expected: every command exits zero with no diagnostics. - -- [x] **Step 4: Audit scope against the final diff** - -Confirm there is no registry lookup, name-syntax validation, extension -configuration interpretation, fill-value semantics, or codec-kind/order -validation. Confirm `consolidated_metadata: null` is still accepted on read -and never emitted. - -- [x] **Step 5: Commit docs and final integration adjustments** - -```bash -git add -f docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md \ - docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md -git add packages/zarr-metadata -git commit -m "docs(zarr-metadata): document v3 model conformance" \ - -m "Assisted-by: Codex:gpt-5" -``` - -- [x] **Step 6: Verify committed state** - -```bash -git status --short --branch -git log -4 --format='%h %s%n%(trailers:key=Assisted-by,valueonly)' -``` - -Expected: only the user's pre-existing untracked files remain; every new -commit has a conventional subject and `Assisted-by: Codex:gpt-5`. diff --git a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md deleted file mode 100644 index 504ff10a6a..0000000000 --- a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md +++ /dev/null @@ -1,234 +0,0 @@ -# Design: Zarr v3 metadata model conformance - -**Status:** Approved and implemented. -**Scope:** `packages/zarr-metadata`, principally the raw v3 metadata types, -the immutable model layer, and their validators and tests. -**Normative authority:** Zarr core protocol v3.1. The behavior of zarrs and -zarr-python is interoperability evidence where the specification leaves an -implementation boundary or where a documented legacy-read exception is -required. - -## Goal - -Make the v3 array and group metadata models conform to the core Zarr v3.1 -document grammar without turning the generic models into interpreters for -individual extensions. - -The generic layer will validate required and optional core fields, JSON value -and container types, fixed literals and core cross-field constraints, the -common extension envelope, the core rules governing `must_understand`, and the -forward-compatibility obligations for unknown top-level fields. - -It will not validate the configuration semantics of a data type, chunk grid, -chunk key encoding, codec, or storage transformer. Those belong to typed -extension models and readers that resolve extension names. - -## Sources and precedence - -The implementation will be checked against: - -1. [Zarr core protocol v3.1](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html), - especially Array metadata, Group metadata, Codecs, and Extensions. -2. [zarrs](https://github.com/zarrs/zarrs), especially its `ZarrV3MetadataFieldJSON`, - `AdditionalFieldV3`, array-opening validation, and consolidated-metadata - extension handling. -3. Existing zarr-python metadata emitted in the wild, but only for explicit - tolerant-read paths. Legacy data does not redefine what this package emits - or calls conformant. - -If these disagree, the current v3.1 specification wins. An interoperability -exception must be narrow, documented as non-conformant input, normalized on -read, and never emitted. - -## Chosen approach - -Evolve the existing normalized `ZarrV3NamedConfig` instead of introducing a -new hierarchy of role-specific extension classes or preserving every source -JSON spelling. - -This keeps the public shape small and minimizes compatibility churn. -Validation remains context-aware: the same extension envelope is used at all -extension points, while the array-document validator supplies the few rules -that vary by field. - -Two alternatives were rejected: - -- A spelling-preserving model would retain shorthand versus object form and - absent versus empty configuration. That fidelity is not needed by this - semantic model and would substantially change its API. -- Separate required/optional extension classes would encode - `must_understand` restrictions in types, but would duplicate behavior and - force callers to use different classes for otherwise identical envelopes. - -## Extension envelope - -### Raw type - -`ZarrV3NamedConfigJSON` will describe the v3.1 object form: - -- required `name: str`; -- optional `configuration: Mapping[str, JSONValue]`; and -- optional `must_understand: bool`. - -`ZarrV3MetadataFieldJSON` remains the union of a shorthand name string and this object -form. - -The object form accepts only those three members. Extension-owned fields must -be placed inside `configuration`. Rejecting other envelope members matches -zarrs' `deny_unknown_fields` behavior and prevents silently discarding data -during normalization. - -### Normalized model - -`ZarrV3NamedConfig` will hold: - -- `name: str`; -- `configuration: dict[str, JSONValue]`, normalized to an empty mapping when - absent; and -- `must_understand: bool`, normalized to `True` when absent or when the input - is a shorthand string. - -`to_json()` will use one field-independent canonicalization rule. When the -configuration is empty and `must_understand` is true, it emits the shorthand -name string. Otherwise it emits an object, omitting `configuration` when empty -and omitting `must_understand` when true. Thus the model is semantically -lossless, but deliberately not spelling-preserving. The same rule naturally -emits core data types as strings without giving the data-type field a separate -model or serializer. - -### Name validation - -Names will be checked only to ensure that they are strings. The model will not -validate registered-name syntax, URI syntax, or membership in the external -extension registry. Those properties are not part of the extension-agnostic -structural boundary and are left to applications that resolve extension -names. - -## Context-sensitive `must_understand` rules - -An omitted `must_understand` value and every shorthand name mean `True`. - -The array validator will reject `must_understand=False` for `data_type`, -`chunk_grid`, and `chunk_key_encoding`. It will permit false for individual -codecs and storage transformers. - -This validation is structural only. The model does not decide whether an -implementation recognizes a name and does not remove or skip optional -extensions. A reader that resolves extensions owns that decision. - -## Core array and group constraints - -The generic array model will enforce the core constraints that do not require -interpreting an extension: - -- `zarr_format` is exactly `3` and `node_type` is exactly `"array"`; -- `shape` contains non-negative JSON integers (booleans excluded); -- all mandatory fields are present; -- `fill_value` is JSON, while its data-type-dependent meaning remains opaque; -- `codecs` is a non-empty sequence of valid extension envelopes; -- `storage_transformers`, when present, is a sequence of valid envelopes; -- `attributes`, when present, is a string-keyed JSON object; and -- `dimension_names`, when present, contains strings or null and has the same - length as `shape`. - -The model will not enforce regular-grid rank or positive chunk lengths, -data-type-specific fill values, codec kinds or ordering, or storage-transformer -behavior. These rules require resolving an extension name and therefore live -outside the generic model. - -The generic group model will enforce the corresponding core document rules: -fixed literals, required fields, JSON attributes, string top-level keys, and -unknown-field handling. - -## Unknown top-level fields - -Unknown array and group members will be represented as arbitrary `JSONValue`, -not as a TypedDict that requires an object. - -For reader obligations: - -- an object whose `must_understand` member is the literal JSON boolean `false` - is explicitly waivable; and -- every other value, including scalars, arrays, objects with no such member, - and objects with a non-boolean value, implicitly requires understanding. - -This matches the v3.1 implicit-true rule and zarrs' `AdditionalFieldV3` -behavior. The raw and model annotations, `is_*` guards, parsers, and -`must_understand_fields` property must all agree on this representation. - -The model itself will not fail merely because such a field requires -understanding. It has no extension registry. It exposes the partition so the -reader can fail when a required field is unrecognized. - -## Consolidated metadata - -Inline consolidated metadata is a known interoperability extension rather -than a core v3.1 group field. Its dedicated raw and model types will remain, -and group parsing may continue recognizing it explicitly. - -The standard emitted form is an object containing `kind`, `metadata`, and a -`must_understand` marker. The extension model validates its own payload and -recursively validates embedded v3 nodes. This special handling must not cause -the generic unknown-field type to become object-only. - -Historical zarr-python versions wrote `"consolidated_metadata": null`. -zarrs also accepts this input as a compatibility hotfix. `from_json()` will -continue repairing that exact value to absence, and `to_json()` will never -write it. Documentation and tests will identify this as a tolerant-read -exception, not valid core metadata. - -## Errors and normalization - -All newly enforced constraints use the existing aggregated -`MetadataValidationError`/`ValidationProblem` mechanism. Locations must point -to the envelope member or array entry that failed, and problem kinds remain -machine-readable. - -Parsing still recursively converts JSON arrays to the tuple-backed public -representation. Validation and type guards must agree on canonical runtime -containers after normalization. - -No constructor-wide semantic validation will be added to `update()`; -`from_json()` remains the validated ingestion boundary. Direct construction -and `update()` continue to support building or repairing intermediate models. - -## Public API compatibility - -The existing class names remain. The intended public changes are additive or -corrections to annotations that did not match accepted runtime data: - -- `ZarrV3NamedConfigJSON` gains optional `must_understand`; -- `ZarrV3NamedConfig` gains a defaulted `must_understand` field; and -- v3 additional-field annotations widen to arbitrary JSON. - -Documents currently accepted only because an extension envelope contains -unknown members, a non-string name, a forbidden false `must_understand`, or an -empty codec list will become validation errors. Those documents violate the -agreed core grammar; accepting them is not a compatibility contract to retain. - -## Test strategy - -Implementation will be test-first and cover: - -1. shorthand and object normalization, including implicit true; -2. explicit true/false round trips; -3. invalid `must_understand` types and unknown envelope members; -4. arbitrary string names accepted and non-string names rejected; -5. false `must_understand` rejected at mandatory extension points; -6. false accepted for codecs and storage transformers; -7. empty codecs rejected without interpreting non-empty pipelines; -8. arbitrary JSON unknown top-level fields and their obligation partition; -9. raw type, parser, type-guard, model, and serializer agreement; -10. consolidated-metadata parsing and the legacy-null read repair; and -11. representative metadata serialized by zarrs. - -The package test suite, formatting, linting, and static type checking will run -before the implementation commit. - -## Completion criteria - -The work is complete when every normative core-document rule in scope has an -explicit validator or a documented extension-owned boundary; raw annotations -and runtime behavior agree; tests demonstrate `must_understand` at every -extension point; zarrs-compatible metadata round-trips semantically; and the -package tests and quality checks pass. From 989c3c1dea0561ab6ea5714f2d636b54eaa8f2b3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 15:44:30 +0200 Subject: [PATCH 43/48] chore(zarr-metadata): correct changelog PR number Assisted-by: Codex:gpt-5 --- .../zarr-metadata/changes/{210.feature.md => 4119.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{210.feature.md => 4119.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/4119.feature.md similarity index 100% rename from packages/zarr-metadata/changes/210.feature.md rename to packages/zarr-metadata/changes/4119.feature.md From 2586faa1fc2ad19b80e21559aa5e262f3c29b47a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 17:00:20 +0200 Subject: [PATCH 44/48] fix(metadata): enforce canonical document boundaries Assisted-by: Codex:GPT-5 --- packages/zarr-metadata/README.md | 28 +++-- .../zarr-metadata/changes/4119.feature.md | 28 +++-- packages/zarr-metadata/pyproject.toml | 2 +- .../src/zarr_metadata/model/_array.py | 12 +- .../src/zarr_metadata/model/_group.py | 32 +++-- .../src/zarr_metadata/model/_validation.py | 115 ++++++++++++++---- .../src/zarr_metadata/pydantic.py | 78 ++++++++---- .../zarr-metadata/tests/model/test_array.py | 92 +++++++++++++- .../zarr-metadata/tests/model/test_group.py | 74 +++++++++++ .../tests/model/test_pydantic_module.py | 23 +++- 10 files changed, 384 insertions(+), 100 deletions(-) diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index f2884782fd..bcd30b742d 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -9,6 +9,9 @@ JSON shapes specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/late and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/) and a few widely-used-but-unspecified entities (e.g. consolidated metadata). +It also provides canonical frozen-dataclass models, structural validators, +parsers, store-key serialization, and optional Pydantic field integrations. +The optional integration requires Pydantic 2.13 or newer. ## What this is for @@ -28,17 +31,24 @@ with open("zarr.json", "rb") as f: metadata = TypeAdapter(ZarrV3ArrayMetadataJSON).validate_python(raw) ``` -## What this is *not* +For a normalized model with loc-aware validation and serialization: -- Not a parser or builder. There are no `make_array_metadata(...)` factories — - that surface belongs to consumer libraries. -- Not a runtime validator on its own. Pair with `pydantic`, `msgspec`, or - similar to enforce shapes at decode time. +```python +from zarr_metadata.model import ZarrV3ArrayMetadata + +model = ZarrV3ArrayMetadata.from_json(raw) +encoded = model.to_key_value()["zarr.json"] +``` + +## Validation boundary -Even with a runtime validator, these types only describe **structural** -shape — they will not flag *semantically* invalid metadata, like a 3D v3 -array whose `dimension_names` has 4 entries instead of 3. That's a job -for downstream validator routines. +The model validators enforce the declared document structure and a small set +of context-free consistency rules, including fixed format literals, finite +JSON numbers, non-negative dimensions, non-empty v3 codec pipelines, and one +`dimension_names` entry per array dimension. They do not interpret extension +names or configurations, resolve codec pipelines, or decide whether a data +type, chunk grid, codec, or storage transformer is supported. Those decisions +belong to consumer implementations. ## Scope diff --git a/packages/zarr-metadata/changes/4119.feature.md b/packages/zarr-metadata/changes/4119.feature.md index fda1a88cf7..b32caf7b1a 100644 --- a/packages/zarr-metadata/changes/4119.feature.md +++ b/packages/zarr-metadata/changes/4119.feature.md @@ -1,13 +1,15 @@ Added `zarr_metadata.model`: frozen-dataclass models (`ZarrV2ArrayMetadata`, `ZarrV3ArrayMetadata`, `ZarrV2GroupMetadata`, `ZarrV3GroupMetadata`, `ZarrV2ConsolidatedMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV3NamedConfig`) -that are canonical, lossless representations of Zarr metadata documents, plus -structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension -point (data type, chunk grid, chunk key encoding, codecs, storage transformers) -is held as a name + configuration pair; nothing is interpreted. Model fields -are annotated with the role alias `ZarrV3MetadataField` (today exactly -`ZarrV3NamedConfig`), so the annotations convey the logical meaning of the -field and stay put if the spec ever adds a new field form. +that are canonical, semantically lossless representations of Zarr metadata +documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). +Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, +storage transformers) is held as `ZarrV3NamedConfig`: a name, configuration, +and `must_understand` obligation; nothing is interpreted. On the wire, an +empty configuration with the default obligation uses the spec's plain-string +shorthand. Model fields are annotated with the role alias +`ZarrV3MetadataField` (today exactly `ZarrV3NamedConfig`), so annotations +convey the logical meaning and stay put if the spec adds another field form. Validation is strict about what the types declare: v2 `dtype` / `order` / `compressor` / `filters` / `dimension_separator` shapes and the fixed @@ -21,6 +23,8 @@ adversarial review added further structural checks: JSON booleans are not accepted as dimension lengths, dimensions are non-negative, `dimension_names` must have one entry per dimension of `shape`, `attributes` and `configuration` values are JSON-checked recursively (like `fill_value`), +non-finite floats and non-standard JSON constants are rejected, abstract +mappings and sequences normalize to encoder-safe canonical containers, and the inline consolidated-metadata envelope and entries are deep-validated so the group validator's verdict always agrees with the model constructor. @@ -31,11 +35,13 @@ duty by subtracting the extension names they recognize; the model only partitions by obligation, since recognition is reader-specific. Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it -requires pydantic v2; the core package does not depend on it): one `Annotated` +requires pydantic 2.13 or newer; the core package does not depend on it): one +`Annotated` field type per model, validating raw documents through `from_json`, passing -core-model instances through unchanged, and serializing via `to_json`. The -instances are the core model classes, so values interoperate freely with -non-pydantic code. +core-model instances through unchanged, serializing via `to_json`, and +publishing JSON Schemas derived from the raw document TypedDicts. The instances +are the core model classes, so values interoperate freely with non-pydantic +code. `create_default` keeps its output self-consistent: overriding `shape` without a chunk grid derives one regular chunk covering the array (v3 diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 60f5c4aa95..23905a5bb7 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -47,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/README.md" [dependency-groups] -test = ["pytest", "pydantic>=2"] +test = ["pytest", "pydantic>=2.13"] [tool.hatch.version] source = "vcs" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index fea75d6e96..14ca92e8fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -3,7 +3,6 @@ from __future__ import annotations import dataclasses -import json from collections.abc import Mapping from dataclasses import dataclass, field from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast @@ -16,6 +15,7 @@ MetadataValidationError, ValidationProblem, arrays_to_tuples, + dump_store_json, load_store_json, parse_array_metadata_v2, parse_array_metadata_v3, @@ -318,9 +318,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: - return { - ARRAY_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") - } + return {ARRAY_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} class ZarrV2ArrayMetadataPartial(TypedDict, total=False): @@ -472,9 +470,7 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} - out = {ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8")} + out = {ARRAY_METADATA_STORE_KEY_V2: dump_store_json(zarray, indent=indent)} if self.attributes is not UNSET: - out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( - "utf-8" - ) + out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index ae43955d91..39c4da5b1e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -3,7 +3,6 @@ from __future__ import annotations import dataclasses -import json from collections.abc import Mapping from dataclasses import dataclass, field from typing import TYPE_CHECKING, Final, Literal, cast @@ -21,6 +20,7 @@ MetadataValidationError, ValidationProblem, arrays_to_tuples, + dump_store_json, load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, @@ -184,9 +184,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: - return { - GROUP_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") - } + return {GROUP_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} @dataclass(frozen=True, slots=True, kw_only=True) @@ -228,10 +226,11 @@ def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: @classmethod def from_json(cls, data: object) -> ZarrV3ConsolidatedMetadata: - problems = validate_consolidated_metadata_v3(data) + normalized = arrays_to_tuples(data) + problems = validate_consolidated_metadata_v3(normalized) if problems: raise MetadataValidationError(problems) - env = cast("Mapping[str, object]", data) + env = cast("Mapping[str, object]", normalized) entries: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] = {} for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): node_type = cast("Mapping[str, object]", entry).get("node_type") @@ -330,11 +329,9 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} - out = {GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8")} + out = {GROUP_METADATA_STORE_KEY_V2: dump_store_json(zgroup, indent=indent)} if self.attributes is not UNSET: - out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( - "utf-8" - ) + out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) return out @@ -360,16 +357,21 @@ def to_json(self) -> dict[str, JSONValue]: @classmethod def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: - if not isinstance(data, Mapping): + normalized = arrays_to_tuples(data) + if not isinstance(normalized, Mapping): raise MetadataValidationError( [ValidationProblem((), "expected a mapping", "invalid_type")] ) - doc = cast("Mapping[str, object]", data) + doc = cast("Mapping[str, object]", normalized) problems: list[ValidationProblem] = [ ValidationProblem((key,), "missing required key", "missing_key") for key in ("zarr_consolidated_format", "metadata") if key not in doc ] + problems.extend( + ValidationProblem((key,), "unexpected document member", "invalid_value") + for key in doc.keys() - {"zarr_consolidated_format", "metadata"} + ) if "zarr_consolidated_format" in doc and ( not isinstance(doc["zarr_consolidated_format"], int) or isinstance(doc["zarr_consolidated_format"], bool) @@ -413,8 +415,4 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetad return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: - return { - CONSOLIDATED_METADATA_STORE_KEY_V2: json.dumps(self.to_json(), indent=indent).encode( - "utf-8" - ) - } + return {CONSOLIDATED_METADATA_STORE_KEY_V2: dump_store_json(self.to_json(), indent=indent)} diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 049a399685..5e8f669a1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -14,9 +14,10 @@ from __future__ import annotations import json +import math from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Any, Final, Literal, cast +from typing import Any, Final, Literal, NoReturn, cast from typing_extensions import TypeIs @@ -76,7 +77,11 @@ def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[Vali def validate_json(value: object) -> list[ValidationProblem]: """Return every reason `value` is not JSON-serializable (recursively).""" - if isinstance(value, (str, int, float, bool)) or value is None: + if isinstance(value, float): + if math.isfinite(value): + return [] + return [ValidationProblem((), f"non-finite float {value!r} is not JSON", "invalid_value")] + if isinstance(value, (str, int, bool)) or value is None: return [] problems: list[ValidationProblem] = [] if isinstance(value, Mapping): @@ -101,11 +106,12 @@ def is_json(value: object) -> TypeIs[JSONValue]: def parse_json(value: object) -> JSONValue: - """Return `value` narrowed to `JSONValue`, or raise `MetadataValidationError`.""" - problems = validate_json(value) + """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_json(normalized) if problems: raise MetadataValidationError(problems) - return cast(JSONValue, value) + return cast(JSONValue, normalized) # The standard top-level keys of a v3 array metadata document. Anything outside @@ -124,6 +130,12 @@ def parse_json(value: object) -> JSONValue: ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2ArrayMetadataJSON.__required_keys__ ) +ARRAY_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2ArrayMetadataJSON.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V2 | ARRAY_METADATA_OPTIONAL_KEYS_V2 +) # The standard top-level keys of a v3 group metadata document. Anything outside # this set is an extension field. @@ -140,6 +152,12 @@ def parse_json(value: object) -> JSONValue: GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2GroupMetadataJSON.__required_keys__ ) +GROUP_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2GroupMetadataJSON.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V2 | GROUP_METADATA_OPTIONAL_KEYS_V2 +) def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[ValidationProblem]: @@ -150,11 +168,28 @@ def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[V ] +def _unexpected_keys( + allowed: frozenset[str], doc: Mapping[object, object] +) -> list[ValidationProblem]: + """One problem per member outside a closed document's declared shape.""" + problems: list[ValidationProblem] = [] + for key in doc: + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string document key {key!r}", "invalid_type") + ) + elif key not in allowed: + problems.append( + ValidationProblem((key,), "unexpected document member", "invalid_value") + ) + return problems + + def _check_literal( doc: Mapping[str, object], key: str, expected: object ) -> list[ValidationProblem]: """One `invalid_value` problem if `doc[key]` is present but not `expected`.""" - if key in doc and doc[key] != expected: + if key in doc and (type(doc[key]) is not type(expected) or doc[key] != expected): return [ ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value") ] @@ -251,10 +286,11 @@ def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: """Return `value` narrowed to `ZarrV3MetadataFieldJSON`, or raise `MetadataValidationError`.""" - problems = validate_metadata_field_v3(value) + normalized = arrays_to_tuples(value) + problems = validate_metadata_field_v3(normalized) if problems: raise MetadataValidationError(problems) - return cast(ZarrV3MetadataFieldJSON, value) + return cast(ZarrV3MetadataFieldJSON, normalized) def _is_int_sequence(value: object) -> bool: @@ -523,6 +559,9 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend( + _unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) + ) problems.extend(_check_literal(doc, "zarr_format", 2)) problems.extend(_validate_dim_sequence(doc, "shape")) problems.extend(_validate_dim_sequence(doc, "chunks")) @@ -606,6 +645,12 @@ def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: for key in ("kind", "must_understand", "metadata") if key not in env ] + problems.extend( + _unexpected_keys( + frozenset({"kind", "must_understand", "metadata"}), + cast("Mapping[object, object]", value), + ) + ) problems.extend(_check_literal(env, "kind", "inline")) if "must_understand" in env and env["must_understand"] is not False: problems.append(ValidationProblem(("must_understand",), "expected False", "invalid_value")) @@ -681,15 +726,18 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: """Whether `value` is a structurally-valid v3 group metadata document.""" - return not validate_group_metadata_v3(value) + return isinstance(value, dict) and not validate_group_metadata_v3( + cast("dict[object, object]", value) + ) def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: """Return `value` narrowed to `ZarrV3GroupMetadataJSON`, or raise `MetadataValidationError`.""" - problems = validate_group_metadata_v3(value) + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v3(normalized) if problems: raise MetadataValidationError(problems) - return cast(ZarrV3GroupMetadataJSON, value) + return cast(ZarrV3GroupMetadataJSON, normalized) def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: @@ -702,6 +750,9 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend( + _unexpected_keys(GROUP_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) + ) problems.extend(_check_literal(doc, "zarr_format", 2)) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) @@ -710,15 +761,23 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: """Whether `value` is a structurally-valid v2 group metadata document.""" - return not validate_group_metadata_v2(value) + return isinstance(value, dict) and not validate_group_metadata_v2( + cast("dict[object, object]", value) + ) def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: """Return `value` narrowed to `ZarrV2GroupMetadataJSON`, or raise `MetadataValidationError`.""" - problems = validate_group_metadata_v2(value) + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v2(normalized) if problems: raise MetadataValidationError(problems) - return cast(ZarrV2GroupMetadataJSON, value) + return cast(ZarrV2GroupMetadataJSON, normalized) + + +def _reject_json_constant(constant: str) -> NoReturn: + """Reject the JavaScript constants accepted by Python's JSON decoder.""" + raise ValueError(f"non-standard JSON constant {constant!r}") def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: @@ -734,23 +793,35 @@ def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: [ValidationProblem((key,), "missing store key", "missing_key")] ) try: - return json.loads(mapping[key]) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return json.loads(mapping[key], parse_constant=_reject_json_constant) + except (UnicodeDecodeError, ValueError) as exc: raise MetadataValidationError( [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] ) from exc +def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: + """Encode a metadata document as strict RFC 8259 JSON bytes.""" + return json.dumps(value, indent=indent, allow_nan=False).encode("utf-8") + + def arrays_to_tuples(obj: object) -> object: - """Recursively convert every list in a JSON-decoded structure to a tuple.""" - if isinstance(obj, list): - return tuple(arrays_to_tuples(item) for item in cast("list[object]", obj)) - if isinstance(obj, dict): - mapping = cast("dict[object, object]", obj) + """Recursively materialize mappings and convert array-like values to tuples.""" + if isinstance(obj, Sequence) and not isinstance(obj, (str, bytes, bytearray)): + sequence = cast("Sequence[object]", obj) + converted_sequence = tuple(arrays_to_tuples(item) for item in sequence) + if isinstance(obj, tuple) and all( + converted is original + for converted, original in zip(converted_sequence, sequence, strict=True) + ): + return cast("tuple[object, ...]", obj) + return converted_sequence + if isinstance(obj, Mapping): + mapping = cast("Mapping[object, object]", obj) converted: dict[object, object] = { key: arrays_to_tuples(value) for key, value in mapping.items() } - if all(converted[key] is value for key, value in mapping.items()): + if isinstance(obj, dict) and all(converted[key] is value for key, value in mapping.items()): return cast("object", obj) return converted return obj diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index f3bd4e75db..048bbe766f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -30,9 +30,16 @@ class ArrayManifest(BaseModel): from typing import TYPE_CHECKING, Annotated, TypeVar -from pydantic import BeforeValidator, InstanceOf, PlainSerializer, WithJsonSchema +from pydantic import BeforeValidator, InstanceOf, PlainSerializer from zarr_metadata import model as _model +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON if TYPE_CHECKING: from collections.abc import Callable @@ -51,68 +58,85 @@ def coerce(value: object) -> _M: return coerce -# JSON schemas describe the DOCUMENT form each field accepts (the validation -# input), not the in-memory model shape. -_DOCUMENT_SCHEMA = {"type": "object"} -_FIELD_SCHEMA = {"anyOf": [{"type": "string"}, {"type": "object"}]} - ZarrV3ArrayMetadata = Annotated[ InstanceOf[_model.ZarrV3ArrayMetadata], - BeforeValidator(_coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json)), - PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3ArrayMetadata"}), + BeforeValidator( + _coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), + json_schema_input_type=ZarrV3ArrayMetadataJSON, + ), + PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=ZarrV3ArrayMetadataJSON), ] """Field type for a v3 array metadata document (`zarr.json` content).""" ZarrV2ArrayMetadata = Annotated[ InstanceOf[_model.ZarrV2ArrayMetadata], - BeforeValidator(_coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json)), - PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2ArrayMetadata"}), + BeforeValidator( + _coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json), + json_schema_input_type=ZarrV2ArrayMetadataJSON, + ), + PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=ZarrV2ArrayMetadataJSON), ] """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" ZarrV3GroupMetadata = Annotated[ InstanceOf[_model.ZarrV3GroupMetadata], - BeforeValidator(_coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json)), - PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3GroupMetadata"}), + BeforeValidator( + _coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json), + json_schema_input_type=ZarrV3GroupMetadataJSON, + ), + PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=ZarrV3GroupMetadataJSON), ] """Field type for a v3 group metadata document (`zarr.json` content).""" ZarrV2GroupMetadata = Annotated[ InstanceOf[_model.ZarrV2GroupMetadata], - BeforeValidator(_coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json)), - PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2GroupMetadata"}), + BeforeValidator( + _coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json), + json_schema_input_type=ZarrV2GroupMetadataJSON, + ), + PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=ZarrV2GroupMetadataJSON), ] """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" ZarrV3ConsolidatedMetadata = Annotated[ InstanceOf[_model.ZarrV3ConsolidatedMetadata], BeforeValidator( - _coerce_to(_model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json) + _coerce_to( + _model.ZarrV3ConsolidatedMetadata, + _model.ZarrV3ConsolidatedMetadata.from_json, + ), + json_schema_input_type=ZarrV3ConsolidatedMetadataJSON, + ), + PlainSerializer( + _model.ZarrV3ConsolidatedMetadata.to_json, + return_type=ZarrV3ConsolidatedMetadataJSON, ), - PlainSerializer(_model.ZarrV3ConsolidatedMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3ConsolidatedMetadata"}), ] """Field type for v3 inline consolidated metadata.""" ZarrV2ConsolidatedMetadata = Annotated[ InstanceOf[_model.ZarrV2ConsolidatedMetadata], BeforeValidator( - _coerce_to(_model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json) + _coerce_to( + _model.ZarrV2ConsolidatedMetadata, + _model.ZarrV2ConsolidatedMetadata.from_json, + ), + json_schema_input_type=ZarrV2ConsolidatedMetadataJSON, + ), + PlainSerializer( + _model.ZarrV2ConsolidatedMetadata.to_json, + return_type=ZarrV2ConsolidatedMetadataJSON, ), - PlainSerializer(_model.ZarrV2ConsolidatedMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2ConsolidatedMetadata"}), ] """Field type for a v2 `.zmetadata` document.""" ZarrV3MetadataField = Annotated[ InstanceOf[_model.ZarrV3NamedConfig], - BeforeValidator(_coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json)), - PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=str | dict), - WithJsonSchema(_FIELD_SCHEMA | {"title": "ZarrV3MetadataField"}), + BeforeValidator( + _coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json), + json_schema_input_type=ZarrV3MetadataFieldJSON, + ), + PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=ZarrV3MetadataFieldJSON), ] """Field type for one normalized v3 metadata extension envelope.""" diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index f84dac40c9..571e4beebf 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -2,6 +2,7 @@ import dataclasses import json +from collections import UserDict from collections.abc import Callable from typing import TYPE_CHECKING @@ -787,8 +788,9 @@ def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: Expect(None, frozenset(), id="none"), Expect({"a": [1, {"b": None}], "c": "x"}, frozenset(), id="nested-containers"), Expect((1, 2, 3), frozenset(), id="tuple-array"), - Expect(float("nan"), frozenset(), id="nan"), - Expect(float("inf"), frozenset(), id="inf"), + Expect(float("nan"), frozenset({()}), id="nan"), + Expect(float("inf"), frozenset({()}), id="inf"), + Expect(float("-inf"), frozenset({()}), id="negative-inf"), Expect(object(), frozenset({()}), id="object"), Expect(b"abc", frozenset({()}), id="bytes"), Expect(bytearray(b"abc"), frozenset({()}), id="bytearray"), @@ -816,12 +818,36 @@ def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) - def test_parse_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: """parse_json returns valid JSON values and raises on invalid ones.""" if case.output == frozenset(): - assert parse_json(case.input) is case.input + parsed = parse_json(case.input) + assert arrays_to_tuples(parsed) == arrays_to_tuples(case.input) else: with pytest.raises(MetadataValidationError): parse_json(case.input) +def test_parse_json_materializes_abstract_containers() -> None: + """Accepted Mapping and Sequence values normalize to JSON encoder containers.""" + value = UserDict({"values": range(3)}) + + parsed = parse_json(value) + + assert parsed == {"values": (0, 1, 2)} + assert type(parsed) is dict + assert type(parsed["values"]) is tuple + json.dumps(parsed, allow_nan=False) + + +def test_parse_metadata_field_materializes_abstract_containers() -> None: + """Named-config parsing produces canonical containers at every nesting level.""" + value = UserDict({"name": "example", "configuration": UserDict({"values": range(2)})}) + + parsed = parse_metadata_field_v3(value) + + assert isinstance(parsed, dict) + assert parsed == {"name": "example", "configuration": {"values": (0, 1)}} + assert type(parsed["configuration"]) is dict + + def test_validate_json_reports_json_in_message() -> None: """validate_json's message for a non-JSON value mentions JSON.""" problems = validate_json(object()) @@ -1180,6 +1206,66 @@ def test_v3_zarr_format_literal_enforced() -> None: assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] +@pytest.mark.parametrize( + ("document", "validate"), + [ + pytest.param( + dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 2.0}, + validate_array_metadata_v2, + id="v2", + ), + pytest.param( + dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 3.0}, + validate_array_metadata_v3, + id="v3", + ), + ], +) +def test_array_zarr_format_rejects_float( + document: object, validate: Callable[[object], list[ValidationProblem]] +) -> None: + """Integer-valued floats do not satisfy integer format literals.""" + assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] + + +def test_array_v2_rejects_unknown_document_member() -> None: + """The closed v2 merged-document shape rejects undeclared members.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("unexpected",), "invalid_value") + ] + + +def test_array_v3_from_json_materializes_abstract_containers() -> None: + """A flexible input mapping becomes the canonical dict/tuple model shape.""" + doc = UserDict(dict(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json())) + doc["shape"] = range(2) + + model = ZarrV3ArrayMetadata.from_json(doc) + + assert model.shape == (0, 1) + assert type(model.shape) is tuple + + +def test_from_key_value_rejects_non_standard_json_constant() -> None: + """Store JSON decoding rejects JavaScript NaN/Infinity constants.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["fill_value"] = float("nan") + raw = json.dumps(doc) + + with pytest.raises(MetadataValidationError, match="invalid JSON"): + ZarrV3ArrayMetadata.from_key_value({"zarr.json": raw.encode()}) + + +def test_to_key_value_rejects_non_finite_model_value() -> None: + """Strict encoding prevents directly-constructed models from writing invalid JSON.""" + model = ZarrV3ArrayMetadata.create_default(fill_value=float("nan")) + + with pytest.raises(ValueError, match="JSON compliant"): + model.to_key_value() + + def test_v3_node_type_literal_enforced() -> None: """A v3 array document claiming node_type 'group' is rejected.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"node_type": "group"} diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 0ce89180c9..52b27ac131 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -2,6 +2,8 @@ import dataclasses import json +from collections import UserDict +from collections.abc import Callable import pytest @@ -17,6 +19,7 @@ ) from zarr_metadata.model._validation import ( MetadataValidationError, + ValidationProblem, parse_group_metadata_v2, parse_group_metadata_v3, validate_group_metadata_v2, @@ -98,6 +101,56 @@ def test_group_v3_bad_attributes() -> None: parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) +@pytest.mark.parametrize( + ("document", "validate"), + [ + pytest.param( + {"zarr_format": 2.0}, + validate_group_metadata_v2, + id="v2", + ), + pytest.param( + {"zarr_format": 3.0, "node_type": "group"}, + validate_group_metadata_v3, + id="v3", + ), + ], +) +def test_group_zarr_format_rejects_float( + document: object, validate: Callable[[object], list[ValidationProblem]] +) -> None: + """Integer-valued floats do not satisfy integer format literals.""" + assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] + + +def test_group_v2_rejects_unknown_document_member() -> None: + """The closed v2 merged-document shape rejects undeclared members.""" + assert [(p.loc, p.kind) for p in validate_group_metadata_v2({"zarr_format": 2, "x": 1})] == [ + (("x",), "invalid_value") + ] + + +@pytest.mark.parametrize( + ("parse", "document"), + [ + pytest.param(parse_group_metadata_v2, {"zarr_format": 2}, id="v2"), + pytest.param( + parse_group_metadata_v3, + {"zarr_format": 3, "node_type": "group"}, + id="v3", + ), + ], +) +def test_group_parser_materializes_abstract_mapping( + parse: Callable[[object], object], document: dict[str, object] +) -> None: + """A successful group parser always returns the declared concrete TypedDict shape.""" + parsed = parse(UserDict(document)) + + assert type(parsed) is dict + assert parsed == document + + def test_group_v3_extension_fields_are_validated() -> None: """Group extension payloads must be JSON values with a must-understand flag.""" doc = { @@ -396,6 +449,27 @@ def test_group_v3_valid_consolidated_passes_validator() -> None: assert validate_group_metadata_v3(doc) == [] +def test_v3_consolidated_rejects_unknown_envelope_member() -> None: + """The inline consolidated envelope is closed and never drops accepted members.""" + doc = { + "kind": "inline", + "must_understand": False, + "metadata": {}, + "unexpected": 1, + } + + with pytest.raises(MetadataValidationError, match="unexpected"): + ZarrV3ConsolidatedMetadata.from_json(doc) + + +def test_v2_consolidated_rejects_unknown_document_member() -> None: + """The v2 consolidated document is closed and never drops accepted members.""" + doc = {"zarr_consolidated_format": 1, "metadata": {}, "unexpected": 1} + + with pytest.raises(MetadataValidationError, match="unexpected"): + ZarrV2ConsolidatedMetadata.from_json(doc) + + # --- must_understand partition ------------------------------------------------ diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 4ee72da16d..9010344d5a 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -101,8 +101,27 @@ class Manifest(BaseModel): codec: zmp.ZarrV3MetadataField schema = Manifest.model_json_schema() - assert schema["properties"]["metadata"] == {"type": "object", "title": "ZarrV3ArrayMetadata"} - assert schema["properties"]["codec"]["anyOf"] == [{"type": "string"}, {"type": "object"}] + metadata_schema = schema["$defs"]["ZarrV3ArrayMetadataJSON"] + assert schema["properties"]["metadata"]["$ref"] == "#/$defs/ZarrV3ArrayMetadataJSON" + assert metadata_schema["required"] == [ + "zarr_format", + "node_type", + "data_type", + "shape", + "chunk_grid", + "chunk_key_encoding", + "fill_value", + "codecs", + ] + assert metadata_schema["properties"]["zarr_format"] == { + "const": 3, + "title": "Zarr Format", + "type": "integer", + } + assert schema["properties"]["codec"]["anyOf"] == [ + {"type": "string"}, + {"$ref": "#/$defs/ZarrV3NamedConfigJSON"}, + ] def test_json_roundtrip() -> None: From 15f9b5807ca445a5f813dccc6c2b07005066edba Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 17:32:14 +0200 Subject: [PATCH 45/48] docs(metadata): record review fix design Assisted-by: Codex:gpt-5 --- ...07-22-zarr-metadata-review-fixes-design.md | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md diff --git a/docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md b/docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md new file mode 100644 index 0000000000..c169a822d9 --- /dev/null +++ b/docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md @@ -0,0 +1,118 @@ +# Zarr Metadata Review Fixes Design + +## Goal + +Correct the significant conformance and public-contract defects found by the +independent branch review without changing the documented ability to construct +temporarily invalid model instances for metadata repair workflows. + +## Scope + +This change fixes four areas: + +1. Validation of raw Zarr v2 `.zarray` and `.zgroup` documents must be distinct + from validation of the library's merged in-memory document representation. +2. Public `TypeIs` predicates must narrow only values that actually inhabit the + type named in their return annotation. +3. The public Zarr v2 dtype type must represent nested structured dtypes. +4. Pydantic-generated JSON schemas must encode the structural constraints that + the corresponding runtime parsers enforce. + +The inaccurate consolidated-metadata docstring is minor and outside this fix. +Existing model-instance pass-through in the Pydantic integration remains +unchanged because it is part of the documented repair-state design. + +## Raw V2 Documents and Merged Models + +The public `ZarrV2ArrayMetadataJSON` and `ZarrV2GroupMetadataJSON` types describe +an in-memory representation in which an optional `attributes` member has been +merged from `.zattrs`. They do not describe the raw `.zarray` and `.zgroup` +objects stored on disk. + +Private raw-document parsers will enforce the storage specification before any +merge occurs: + +- `.zarray` accepts exactly the required array metadata members plus the + optional `dimension_separator` member. Every other member, including + `attributes`, is an error. +- `.zgroup` accepts exactly `zarr_format`. Every other member, including + `attributes`, is an error. +- `.zattrs`, when present, must be a JSON object with string keys and JSON + values. It is then added as the merged `attributes` member. +- An absent `.zattrs` remains distinguishable from an explicitly empty object, + preserving the existing `UNSET` behavior. + +The existing public merged-document validators remain strict. This avoids +silently discarding data and keeps their declared `TypedDict` contracts sound. + +## Sound Type Guards + +Validation and parsing intentionally accept abstract `Mapping` and `Sequence` +inputs. Parsers materialize those inputs into canonical dictionaries and +tuples. Type guards cannot make the same promise because `TypeIs[T]` asserts +that the original object already inhabits `T`. + +`is_json` will therefore require recursively canonical JSON container types: +`dict`, `list`, and `tuple`, with string dictionary keys and canonical children. +It will reject abstract containers such as `UserDict` and `range`, even though +`parse_json` continues to accept and normalize them. + +`is_metadata_field_v3` will likewise require either a string or a concrete +dictionary whose nested values are already canonical. Parsing continues to +accept abstract mappings and materialize them. + +## Recursive V2 Dtype Type + +`ZarrV2DataTypeMetadata` will be a named recursive alias. A structured field's +datatype may be either a dtype string or another structured dtype tuple. The +optional subarray shape remains a tuple of integers. Runtime validation already +accepts this structure, so this change aligns static typing and generated schema +with existing spec-compliant behavior. + +## Pydantic JSON Schema Parity + +The Pydantic field aliases will retain their current runtime behavior: raw +documents are parsed through the core model constructors, and existing core +model instances pass through unchanged. + +Their JSON schema input definitions will be replaced or annotated with +constraints that express the runtime document rules, including: + +- non-negative shape and chunk dimensions; +- at least one v3 codec; +- closed named-configuration objects; +- closed v2 merged documents and consolidated payloads where runtime parsing is + closed; +- `must_understand` constraints at mandatory v3 extension points; and +- the recursive v2 structured-dtype representation. + +Constraints that depend on semantic interpretation outside this package's +structural validators will not be added. The generated schema should describe +accepted raw document inputs, not the intentionally permissive state of an +already-constructed core model instance. + +## Error Handling + +All new raw-v2 failures use `MetadataValidationError` and preserve precise +locations. An illegal `.zarray` member named `attributes`, for example, reports +the location `("attributes",)` and the existing `invalid_value` kind. +Malformed `.zattrs` values report locations beneath `attributes` after the +merge boundary, consistent with the public merged representation. + +## Testing + +Implementation follows red-green-refactor cycles. Regression coverage will +prove: + +- raw `.zarray` and `.zgroup` reject `attributes` and other extra members; +- valid sibling `.zattrs` still merges, including absent-versus-empty behavior; +- `is_json(range(3))` and `is_metadata_field_v3(UserDict(...))` are false while + the corresponding parsers still normalize those inputs; +- nested structured v2 dtypes type-check through Pydantic and validate at + runtime; and +- generated schemas reject representative inputs that runtime parsing rejects: + empty codecs, negative dimensions, forbidden extra members, and unsupported + `must_understand: false` values. + +Focused package tests, formatting, linting, type checking where configured, and +the full repository test suite will run before the implementation is committed. From d1c6dda3ea7094b318a79443210145f71667bc99 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 18:10:42 +0200 Subject: [PATCH 46/48] fix(metadata): align validation and schemas Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/_pydantic_schema.py | 119 ++++++++++++++++++ .../src/zarr_metadata/model/_array.py | 10 ++ .../src/zarr_metadata/model/_group.py | 10 ++ .../src/zarr_metadata/model/_validation.py | 23 +++- .../src/zarr_metadata/pydantic.py | 67 ++++++---- .../src/zarr_metadata/v2/array.py | 11 +- .../zarr-metadata/tests/model/test_array.py | 29 +++++ .../zarr-metadata/tests/model/test_group.py | 13 ++ .../tests/model/test_pydantic_module.py | 84 +++++++++++++ 9 files changed, 339 insertions(+), 27 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py new file mode 100644 index 0000000000..e7f94262c9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -0,0 +1,119 @@ +"""Private input types used only to generate accurate Pydantic JSON schemas.""" + +from __future__ import annotations + +from collections.abc import Mapping # noqa: TC003 # resolved by Pydantic at runtime +from typing import Annotated, Literal, NotRequired + +from pydantic import ConfigDict, Field, with_config +from typing_extensions import TypedDict + +from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ( # noqa: TC001 # resolved by Pydantic at runtime + ZarrV2DataTypeMetadata, +) +from zarr_metadata.v2.codec import ( # noqa: TC001 # resolved by Pydantic at runtime + ZarrV2CodecMetadata, +) + +NonNegativeInt = Annotated[int, Field(ge=0)] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV3NamedConfigJSON(TypedDict): + """Closed v3 named configuration accepted at optional extension points.""" + + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV3MandatoryNamedConfigJSON(TypedDict): + """Closed named configuration accepted where understanding is mandatory.""" + + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[Literal[True]] + + +ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON +ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON +ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] + + +class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for a v3 array document, including arbitrary extensions.""" + + zarr_format: Literal[3] + node_type: Literal["array"] + data_type: ZarrV3MandatoryMetadataFieldJSON + shape: tuple[NonNegativeInt, ...] + chunk_grid: ZarrV3MandatoryMetadataFieldJSON + chunk_key_encoding: ZarrV3MandatoryMetadataFieldJSON + fill_value: JSONValue + codecs: ZarrV3CodecPipelineJSON + attributes: NotRequired[Mapping[str, JSONValue]] + storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] + dimension_names: NotRequired[tuple[str | None, ...]] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV3ConsolidatedMetadataJSON(TypedDict): + """Schema input for the closed inline consolidated-metadata envelope.""" + + kind: Literal["inline"] + must_understand: Literal[False] + metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] + + +class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for a v3 group document, including arbitrary extensions.""" + + zarr_format: Literal[3] + node_type: Literal["group"] + attributes: NotRequired[Mapping[str, JSONValue]] + consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV2ArrayMetadataJSON(TypedDict): + """Schema input for the closed, merged v2 array representation.""" + + zarr_format: Literal[2] + shape: tuple[NonNegativeInt, ...] + chunks: tuple[NonNegativeInt, ...] + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None + fill_value: JSONValue + order: Literal["C", "F"] + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: NotRequired[Literal[".", "/"]] + attributes: NotRequired[Mapping[str, JSONValue]] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV2GroupMetadataJSON(TypedDict): + """Schema input for the closed, merged v2 group representation.""" + + zarr_format: Literal[2] + attributes: NotRequired[Mapping[str, JSONValue]] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV2ConsolidatedMetadataJSON(TypedDict): + """Schema input matching the v2 consolidated model's structural parser.""" + + zarr_consolidated_format: Literal[1] + metadata: Mapping[str, JSONValue] + + +__all__ = [ + "ZarrV2ArrayMetadataJSON", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2GroupMetadataJSON", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3GroupMetadataJSON", + "ZarrV3MetadataFieldJSON", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 14ca92e8fc..bc6f2592f1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -460,6 +460,16 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: if not isinstance(zarray_raw, Mapping): return cls.from_json(zarray_raw) zarray = cast("Mapping[str, object]", zarray_raw) + if "attributes" in zarray: + raise MetadataValidationError( + [ + ValidationProblem( + ("attributes",), + "unexpected document member", + "invalid_value", + ) + ] + ) if ATTRIBUTES_STORE_KEY_V2 in mapping: zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) return cls.from_json({**zarray, "attributes": zattrs}) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 39c4da5b1e..811cc7b81e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -319,6 +319,16 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: if not isinstance(zgroup_raw, Mapping): return cls.from_json(zgroup_raw) zgroup = cast("Mapping[str, object]", zgroup_raw) + if "attributes" in zgroup: + raise MetadataValidationError( + [ + ValidationProblem( + ("attributes",), + "unexpected document member", + "invalid_value", + ) + ] + ) if ATTRIBUTES_STORE_KEY_V2 in mapping: zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) return cls.from_json({**zgroup, "attributes": zattrs}) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 5e8f669a1c..3c3e4a1fea 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -100,9 +100,22 @@ def validate_json(value: object) -> list[ValidationProblem]: return [ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type")] +def _is_canonical_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` already uses the concrete containers in `JSONValue`.""" + if isinstance(value, float): + return math.isfinite(value) + if isinstance(value, (str, int, bool)) or value is None: + return True + if isinstance(value, (list, tuple)): + return all(_is_canonical_json(item) for item in value) + if isinstance(value, dict): + return all(isinstance(key, str) and _is_canonical_json(item) for key, item in value.items()) + return False + + def is_json(value: object) -> TypeIs[JSONValue]: - """Whether `value` is a JSON-serializable structure (recursively).""" - return not validate_json(value) + """Whether `value` is a canonical JSON structure (recursively).""" + return _is_canonical_json(value) def parse_json(value: object) -> JSONValue: @@ -281,7 +294,11 @@ def validate_metadata_field_v3( def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: """Whether `value` is a v3 metadata field: a bare name or a named config.""" - return not validate_metadata_field_v3(value) + return isinstance(value, str) or ( + isinstance(value, dict) + and _is_canonical_json(value) + and not validate_metadata_field_v3(value) + ) def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 048bbe766f..8584efa570 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -33,13 +33,38 @@ class ArrayManifest(BaseModel): from pydantic import BeforeValidator, InstanceOf, PlainSerializer from zarr_metadata import model as _model -from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON -from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON -from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON -from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON -from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON -from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON +from zarr_metadata._pydantic_schema import ( + ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldSchema, +) +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.consolidated import ( + ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataJSON, +) +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.consolidated import ( + ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataJSON, +) +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataJSON if TYPE_CHECKING: from collections.abc import Callable @@ -62,9 +87,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV3ArrayMetadata], BeforeValidator( _coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), - json_schema_input_type=ZarrV3ArrayMetadataJSON, + json_schema_input_type=_ZarrV3ArrayMetadataSchema, ), - PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=ZarrV3ArrayMetadataJSON), + PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=_ZarrV3ArrayMetadataJSON), ] """Field type for a v3 array metadata document (`zarr.json` content).""" @@ -72,9 +97,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV2ArrayMetadata], BeforeValidator( _coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json), - json_schema_input_type=ZarrV2ArrayMetadataJSON, + json_schema_input_type=_ZarrV2ArrayMetadataSchema, ), - PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=ZarrV2ArrayMetadataJSON), + PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=_ZarrV2ArrayMetadataJSON), ] """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" @@ -82,9 +107,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV3GroupMetadata], BeforeValidator( _coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json), - json_schema_input_type=ZarrV3GroupMetadataJSON, + json_schema_input_type=_ZarrV3GroupMetadataSchema, ), - PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=ZarrV3GroupMetadataJSON), + PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=_ZarrV3GroupMetadataJSON), ] """Field type for a v3 group metadata document (`zarr.json` content).""" @@ -92,9 +117,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV2GroupMetadata], BeforeValidator( _coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json), - json_schema_input_type=ZarrV2GroupMetadataJSON, + json_schema_input_type=_ZarrV2GroupMetadataSchema, ), - PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=ZarrV2GroupMetadataJSON), + PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=_ZarrV2GroupMetadataJSON), ] """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" @@ -105,11 +130,11 @@ def coerce(value: object) -> _M: _model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json, ), - json_schema_input_type=ZarrV3ConsolidatedMetadataJSON, + json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, ), PlainSerializer( _model.ZarrV3ConsolidatedMetadata.to_json, - return_type=ZarrV3ConsolidatedMetadataJSON, + return_type=_ZarrV3ConsolidatedMetadataJSON, ), ] """Field type for v3 inline consolidated metadata.""" @@ -121,11 +146,11 @@ def coerce(value: object) -> _M: _model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json, ), - json_schema_input_type=ZarrV2ConsolidatedMetadataJSON, + json_schema_input_type=_ZarrV2ConsolidatedMetadataSchema, ), PlainSerializer( _model.ZarrV2ConsolidatedMetadata.to_json, - return_type=ZarrV2ConsolidatedMetadataJSON, + return_type=_ZarrV2ConsolidatedMetadataJSON, ), ] """Field type for a v2 `.zmetadata` document.""" @@ -134,9 +159,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV3NamedConfig], BeforeValidator( _coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json), - json_schema_input_type=ZarrV3MetadataFieldJSON, + json_schema_input_type=_ZarrV3MetadataFieldSchema, ), - PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=ZarrV3MetadataFieldJSON), + PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=_ZarrV3MetadataFieldJSON), ] """Field type for one normalized v3 metadata extension envelope.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 741f37acba..28085e4fbb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -3,18 +3,23 @@ from collections.abc import Mapping from typing import Final, Literal, NotRequired -from typing_extensions import TypedDict +from typing_extensions import TypeAliasType, TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.v2.codec import ZarrV2CodecMetadata -ZarrV2DataTypeMetadata = str | tuple[tuple[str, str] | tuple[str, str, tuple[int, ...]], ...] +ZarrV2DataTypeMetadata = TypeAliasType( + "ZarrV2DataTypeMetadata", + "str | tuple[tuple[str, ZarrV2DataTypeMetadata] | " + "tuple[str, ZarrV2DataTypeMetadata, tuple[int, ...]], ...]", +) """The v2 dtype representation. Either a numpy-style dtype string (e.g. `" None: assert model.shape == (10,) +@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) +def test_v2_from_key_value_rejects_zarray_extra_members(extra_key: str) -> None: + """Raw `.zarray` documents reject every non-spec member.""" + doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc.pop("attributes", None) + doc[extra_key] = {} + + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((extra_key,), "invalid_value") + ] + + def test_v2_zattrs_presence_round_trips() -> None: """The .zattrs file's presence is part of the store: an absent file reads as UNSET and emits no .zattrs; an explicit empty file reads as {} and @@ -837,6 +852,12 @@ def test_parse_json_materializes_abstract_containers() -> None: json.dumps(parsed, allow_nan=False) +def test_json_type_guard_rejects_abstract_sequence() -> None: + """A guard cannot narrow an abstract sequence that only the parser materializes.""" + assert not is_json(range(3)) + assert parse_json(range(3)) == (0, 1, 2) + + def test_parse_metadata_field_materializes_abstract_containers() -> None: """Named-config parsing produces canonical containers at every nesting level.""" value = UserDict({"name": "example", "configuration": UserDict({"values": range(2)})}) @@ -848,6 +869,14 @@ def test_parse_metadata_field_materializes_abstract_containers() -> None: assert type(parsed["configuration"]) is dict +def test_metadata_field_type_guard_rejects_abstract_mapping() -> None: + """A metadata-field guard only narrows concrete TypedDict-shaped objects.""" + value = UserDict({"name": "bytes"}) + + assert not is_metadata_field_v3(value) + assert parse_metadata_field_v3(value) == {"name": "bytes"} + + def test_validate_json_reports_json_in_message() -> None: """validate_json's message for a non-JSON value mentions JSON.""" problems = validate_json(object()) diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 52b27ac131..99f99de935 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -189,6 +189,19 @@ def test_group_v2_key_value_split() -> None: assert ZarrV2GroupMetadata.from_key_value(kv) == model +@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) +def test_v2_group_from_key_value_rejects_zgroup_extra_members(extra_key: str) -> None: + """Raw `.zgroup` documents reject every non-spec member.""" + doc: dict[str, object] = {"zarr_format": 2, extra_key: {}} + + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2GroupMetadata.from_key_value({".zgroup": json.dumps(doc).encode()}) + + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((extra_key,), "invalid_value") + ] + + def test_group_v2_zattrs_presence_round_trips() -> None: """A v2 group with no .zattrs file parses with UNSET attributes and emits no .zattrs; an explicit empty .zattrs stays a file — the stores remain diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 9010344d5a..0b096b4b68 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -5,7 +5,10 @@ parallel hierarchy), so values interoperate freely with non-pydantic code. """ +import json + import pytest +from jsonschema import Draft202012Validator from pydantic import BaseModel, TypeAdapter, ValidationError import zarr_metadata.pydantic as zmp @@ -124,6 +127,87 @@ class Manifest(BaseModel): ] +def test_v2_recursive_structured_dtype_is_in_pydantic_schema() -> None: + """The schema accepts nested structured dtypes supported by the v2 specification.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["dtype"] = [["outer", [["inner", " None: + adapter = TypeAdapter(field_type) + with pytest.raises(ValidationError): + adapter.validate_python(document) + assert list(Draft202012Validator(adapter.json_schema()).iter_errors(document)) + + +def test_v3_array_schema_rejects_empty_codecs() -> None: + """The generated schema mirrors the runtime non-empty codec pipeline rule.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc["codecs"] = [] + + _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadata, doc) + + +def test_array_schemas_reject_negative_dimensions() -> None: + """Both array schemas mirror the runtime non-negative dimension rule.""" + for field_type, source in ( + (zmp.ZarrV3ArrayMetadata, V3_ARRAY_DOC), + (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), + ): + doc = json.loads(json.dumps(source)) + doc["shape"] = [-1] + _assert_runtime_and_schema_reject(field_type, doc) + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_v3_array_schema_rejects_false_at_mandatory_extension_points(field: str) -> None: + """Mandatory v3 extension points cannot opt out of understanding.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc[field] = {"name": "example", "must_understand": False} + + _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadata, doc) + + +def test_metadata_field_schema_rejects_unknown_members() -> None: + """Named-configuration envelopes are closed in both runtime and schema validation.""" + _assert_runtime_and_schema_reject( + zmp.ZarrV3MetadataField, + {"name": "example", "unexpected": 1}, + ) + + +@pytest.mark.parametrize( + ("field_type", "source"), + [ + (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), + (zmp.ZarrV2GroupMetadata, V2_GROUP_DOC), + (zmp.ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC), + ], +) +def test_v2_schema_rejects_unknown_document_members( + field_type: object, source: dict[str, object] +) -> None: + """Closed v2 merged documents expose their runtime boundary in JSON Schema.""" + doc = json.loads(json.dumps(source)) + doc["unexpected"] = 1 + + _assert_runtime_and_schema_reject(field_type, doc) + + +def test_v3_array_schema_allows_unknown_extension_fields() -> None: + """Schema constraints do not close the v3 top-level extension namespace.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc["vendor_extension"] = {"anything": [1, 2]} + adapter = TypeAdapter(zmp.ZarrV3ArrayMetadata) + + assert adapter.validate_python(doc).extra_fields == {"vendor_extension": {"anything": (1, 2)}} + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) + + def test_json_roundtrip() -> None: """model_dump_json output re-validates to an equal pydantic model.""" From f4db45b576cdd04d56571c865f70022c2aa1d793 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 20:58:16 +0200 Subject: [PATCH 47/48] fix(metadata): tighten validation boundaries Assisted-by: Codex:gpt-5 --- packages/zarr-metadata/README.md | 34 +++++++++----- .../zarr-metadata/changes/4119.feature.md | 10 ++-- packages/zarr-metadata/pyproject.toml | 6 +-- .../src/zarr_metadata/_pydantic_schema.py | 25 ++++------ .../src/zarr_metadata/model/_group.py | 2 +- .../src/zarr_metadata/model/_validation.py | 46 +++++++++++++++---- .../src/zarr_metadata/v3/consolidated.py | 6 +-- .../zarr-metadata/tests/model/test_array.py | 37 +++++++++++++++ .../zarr-metadata/tests/model/test_group.py | 13 ++++++ .../tests/model/test_pydantic_module.py | 27 +++++++++++ 10 files changed, 160 insertions(+), 46 deletions(-) diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index bcd30b742d..810d0001f0 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -15,31 +15,35 @@ The optional integration requires Pydantic 2.13 or newer. ## What this is for -These types describe the JSON shape of Zarr metadata. They are -intended for libraries that **read, write, validate, or transform** -Zarr metadata. Pair them with a runtime validator like -[pydantic](https://docs.pydantic.dev/) to check JSON loaded from disk: +The public `TypedDict` definitions describe the static JSON shape of Zarr +metadata. For strict, loc-aware validation of JSON loaded from disk, use the +model parser: ```python import json -from pydantic import TypeAdapter -from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.model import ZarrV3ArrayMetadata with open("zarr.json", "rb") as f: raw = json.load(f) -metadata = TypeAdapter(ZarrV3ArrayMetadataJSON).validate_python(raw) +metadata = ZarrV3ArrayMetadata.from_json(raw) ``` -For a normalized model with loc-aware validation and serialization: +The optional Pydantic integration delegates raw input to the same strict +parser and returns the same normalized model class: ```python -from zarr_metadata.model import ZarrV3ArrayMetadata +from pydantic import TypeAdapter +import zarr_metadata.pydantic as zmp -model = ZarrV3ArrayMetadata.from_json(raw) -encoded = model.to_key_value()["zarr.json"] +metadata = TypeAdapter(zmp.ZarrV3ArrayMetadata).validate_python(raw) +encoded = metadata.to_key_value()["zarr.json"] ``` +A bare `TypeAdapter` over a public document `TypedDict` is a coercive shape +adapter, not a Zarr conformance validator; it may coerce values or discard +members that the strict model parser rejects. + ## Validation boundary The model validators enforce the declared document structure and a small set @@ -50,6 +54,14 @@ names or configurations, resolve codec pipelines, or decide whether a data type, chunk grid, codec, or storage transformer is supported. Those decisions belong to consumer implementations. +The Pydantic integration's generated JSON Schemas express independently +checkable document structure and field constraints, but they are not a +replacement for runtime model validation. Standard JSON Schema treats a +mathematically integral number such as `1.0` as an integer, while the runtime +boundary requires Python `int` values, and it cannot express arbitrary +same-length relations such as `dimension_names` versus `shape` or v2 `chunks` +versus `shape`. Consumers should run the model parser after schema validation. + ## Scope At minimum, this library supports what Zarr-Python needs: the complete diff --git a/packages/zarr-metadata/changes/4119.feature.md b/packages/zarr-metadata/changes/4119.feature.md index b32caf7b1a..7da9b728bf 100644 --- a/packages/zarr-metadata/changes/4119.feature.md +++ b/packages/zarr-metadata/changes/4119.feature.md @@ -25,6 +25,9 @@ accepted as dimension lengths, dimensions are non-negative, and `configuration` values are JSON-checked recursively (like `fill_value`), non-finite floats and non-standard JSON constants are rejected, abstract mappings and sequences normalize to encoder-safe canonical containers, +v2 `shape` and `chunks` must have the same rank, non-null v2 filter pipelines +contain at least one filter, document `TypeIs` guards only narrow values that +already use the declared canonical containers, and the inline consolidated-metadata envelope and entries are deep-validated so the group validator's verdict always agrees with the model constructor. @@ -39,9 +42,10 @@ requires pydantic 2.13 or newer; the core package does not depend on it): one `Annotated` field type per model, validating raw documents through `from_json`, passing core-model instances through unchanged, serializing via `to_json`, and -publishing JSON Schemas derived from the raw document TypedDicts. The instances -are the core model classes, so values interoperate freely with non-pydantic -code. +publishing JSON Schemas derived from private constrained document types that +mirror the independently expressible runtime rules. Cross-field cardinality +relations still require runtime validation. The instances are the core model +classes, so values interoperate freely with non-pydantic code. `create_default` keeps its output self-consistent: overriding `shape` without a chunk grid derives one regular chunk covering the array (v3 diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 23905a5bb7..8dcf0a905a 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -71,9 +71,9 @@ xfail_strict = true addopts = ["-ra", "--strict-config", "--strict-markers"] filterwarnings = [ "error", - # pydantic warns about ReadOnly TypedDict items not being enforced at runtime. - # That's expected here — we rely on type-checker enforcement, not pydantic mutation guards. - "ignore::UserWarning:pydantic._internal._generate_schema", + # Pydantic validates these public immutable-shape TypedDicts correctly but + # cannot enforce the type checker's ReadOnly mutation restriction. + "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema", ] [tool.numpydoc_validation] diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index e7f94262c9..e9792d6931 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -5,22 +5,21 @@ from collections.abc import Mapping # noqa: TC003 # resolved by Pydantic at runtime from typing import Annotated, Literal, NotRequired -from pydantic import ConfigDict, Field, with_config +from pydantic import Field from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.v2.array import ( # noqa: TC001 # resolved by Pydantic at runtime ZarrV2DataTypeMetadata, ) -from zarr_metadata.v2.codec import ( # noqa: TC001 # resolved by Pydantic at runtime +from zarr_metadata.v2.codec import ( # resolved by Pydantic at runtime ZarrV2CodecMetadata, ) NonNegativeInt = Annotated[int, Field(ge=0)] -@with_config(ConfigDict(extra="forbid")) -class ZarrV3NamedConfigJSON(TypedDict): +class ZarrV3NamedConfigJSON(TypedDict, closed=True): """Closed v3 named configuration accepted at optional extension points.""" name: str @@ -28,8 +27,7 @@ class ZarrV3NamedConfigJSON(TypedDict): must_understand: NotRequired[bool] -@with_config(ConfigDict(extra="forbid")) -class ZarrV3MandatoryNamedConfigJSON(TypedDict): +class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): """Closed named configuration accepted where understanding is mandatory.""" name: str @@ -40,6 +38,7 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict): ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] +ZarrV2FilterPipelineJSON = Annotated[tuple[ZarrV2CodecMetadata, ...], Field(min_length=1)] class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): @@ -58,8 +57,7 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): dimension_names: NotRequired[tuple[str | None, ...]] -@with_config(ConfigDict(extra="forbid")) -class ZarrV3ConsolidatedMetadataJSON(TypedDict): +class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): """Schema input for the closed inline consolidated-metadata envelope.""" kind: Literal["inline"] @@ -76,8 +74,7 @@ class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None] -@with_config(ConfigDict(extra="forbid")) -class ZarrV2ArrayMetadataJSON(TypedDict): +class ZarrV2ArrayMetadataJSON(TypedDict, closed=True): """Schema input for the closed, merged v2 array representation.""" zarr_format: Literal[2] @@ -87,21 +84,19 @@ class ZarrV2ArrayMetadataJSON(TypedDict): compressor: ZarrV2CodecMetadata | None fill_value: JSONValue order: Literal["C", "F"] - filters: tuple[ZarrV2CodecMetadata, ...] | None + filters: ZarrV2FilterPipelineJSON | None dimension_separator: NotRequired[Literal[".", "/"]] attributes: NotRequired[Mapping[str, JSONValue]] -@with_config(ConfigDict(extra="forbid")) -class ZarrV2GroupMetadataJSON(TypedDict): +class ZarrV2GroupMetadataJSON(TypedDict, closed=True): """Schema input for the closed, merged v2 group representation.""" zarr_format: Literal[2] attributes: NotRequired[Mapping[str, JSONValue]] -@with_config(ConfigDict(extra="forbid")) -class ZarrV2ConsolidatedMetadataJSON(TypedDict): +class ZarrV2ConsolidatedMetadataJSON(TypedDict, closed=True): """Schema input matching the v2 consolidated model's structural parser.""" zarr_consolidated_format: Literal[1] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 811cc7b81e..f1a76e2508 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -350,7 +350,7 @@ class ZarrV2ConsolidatedMetadata: """In-memory model of a v2 `.zmetadata` document. The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, - `"path/.zattrs"`, ...) verbatim, preserving byte-faithful round-tripping. + `"path/.zattrs"`, ...) verbatim, preserving the normalized JSON tree. Entries are deliberately NOT merged into per-node models: which nodes had a `.zattrs` file at all is information the canonical representation must keep. Interpreting entries into node models is consumer work. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 3c3e4a1fea..35e6904edf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -552,7 +552,11 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: def is_array_metadata_v3(value: object) -> TypeIs[ZarrV3ArrayMetadataJSON]: """Whether `value` is a structurally-valid v3 array metadata document.""" - return not validate_array_metadata_v3(value) and _is_canonical_array_metadata_v3(value) + return ( + _is_canonical_json(value) + and not validate_array_metadata_v3(value) + and _is_canonical_array_metadata_v3(value) + ) def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: @@ -580,8 +584,26 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: _unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) ) problems.extend(_check_literal(doc, "zarr_format", 2)) - problems.extend(_validate_dim_sequence(doc, "shape")) - problems.extend(_validate_dim_sequence(doc, "chunks")) + shape_problems = _validate_dim_sequence(doc, "shape") + chunks_problems = _validate_dim_sequence(doc, "chunks") + problems.extend(shape_problems) + problems.extend(chunks_problems) + if ( + not shape_problems + and not chunks_problems + and _is_int_sequence(doc.get("shape")) + and _is_int_sequence(doc.get("chunks")) + ): + shape = cast("Sequence[int]", doc["shape"]) + chunks = cast("Sequence[int]", doc["chunks"]) + if len(shape) != len(chunks): + problems.append( + ValidationProblem( + ("chunks",), + "expected the same number of dimensions as shape", + "invalid_value", + ) + ) if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): problems.append( ValidationProblem( @@ -615,6 +637,10 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: ) ) elif filters is not None: + if len(cast("Sequence[object]", filters)) == 0: + problems.append( + ValidationProblem(("filters",), "expected at least one filter", "invalid_value") + ) for index, item in enumerate(cast("Sequence[object]", filters)): problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): @@ -634,7 +660,11 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: def is_array_metadata_v2(value: object) -> TypeIs[ZarrV2ArrayMetadataJSON]: """Whether `value` is a structurally-valid v2 array metadata document.""" - return not validate_array_metadata_v2(value) and _is_canonical_array_metadata_v2(value) + return ( + _is_canonical_json(value) + and not validate_array_metadata_v2(value) + and _is_canonical_array_metadata_v2(value) + ) def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: @@ -743,9 +773,7 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: """Whether `value` is a structurally-valid v3 group metadata document.""" - return isinstance(value, dict) and not validate_group_metadata_v3( - cast("dict[object, object]", value) - ) + return _is_canonical_json(value) and not validate_group_metadata_v3(value) def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: @@ -778,9 +806,7 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: """Whether `value` is a structurally-valid v2 group metadata document.""" - return isinstance(value, dict) and not validate_group_metadata_v2( - cast("dict[object, object]", value) - ) + return _is_canonical_json(value) and not validate_group_metadata_v2(value) def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index bc29cd145e..bcbe675947 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -24,9 +24,9 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict): """ Inline consolidated metadata embedded in a v3 group. - The `metadata` map contains only v3 array and group entries - v2 - entries are excluded by design. Mixing v2 entries into a v3 - consolidated metadata document is invalid per spec. + The `metadata` map contains only v3 array and group entries. V2 entries + are excluded from this interoperability convention by design; the v3 core + specification does not define consolidated metadata. """ kind: Literal["inline"] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 20764a85eb..f0ff473df1 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1214,6 +1214,30 @@ def test_v2_filters_must_be_codec_sequence_or_none() -> None: assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad +def test_v2_shape_and_chunks_must_have_equal_rank() -> None: + """Raw v2 metadata requires one chunk length per array dimension.""" + doc = dict(ZarrV2ArrayMetadata.create_default(shape=(2, 3)).to_json()) + doc["chunks"] = (1,) + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("chunks",), "invalid_value") + ] + with pytest.raises(MetadataValidationError, match="same number of dimensions"): + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + +def test_v2_filters_must_be_nonempty_when_present() -> None: + """A non-null v2 filter sequence contains one or more codec configurations.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc["filters"] = () + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("filters",), "invalid_value") + ] + with pytest.raises(MetadataValidationError, match="at least one filter"): + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + def test_v2_dimension_separator_literal_enforced() -> None: """A dimension_separator other than '.' or '/' is rejected.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": "-"} @@ -1492,6 +1516,19 @@ def test_array_parsers_normalize_json_lists_before_narrowing() -> None: assert isinstance(v2_parsed["chunks"], tuple) +def test_array_guards_reject_noncanonical_nested_json() -> None: + """Document guards cannot narrow values that only parsers can materialize.""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) + v3["fill_value"] = range(2) + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) + v2["fill_value"] = range(2) + + assert not is_array_metadata_v3(v3) + assert not is_array_metadata_v2(v2) + assert parse_array_metadata_v3(v3)["fill_value"] == (0, 1) + assert parse_array_metadata_v2(v2)["fill_value"] == (0, 1) + + # --- must_understand partition (spec: MUST fail to open unrecognized fields) -- diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 99f99de935..f38efeeb8a 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -20,6 +20,8 @@ from zarr_metadata.model._validation import ( MetadataValidationError, ValidationProblem, + is_group_metadata_v2, + is_group_metadata_v3, parse_group_metadata_v2, parse_group_metadata_v3, validate_group_metadata_v2, @@ -151,6 +153,17 @@ def test_group_parser_materializes_abstract_mapping( assert parsed == document +def test_group_guards_reject_noncanonical_nested_json() -> None: + """Document guards cannot narrow values that only parsers can materialize.""" + v3 = {"zarr_format": 3, "node_type": "group", "extension": range(2)} + v2 = {"zarr_format": 2, "attributes": {"values": range(2)}} + + assert not is_group_metadata_v3(v3) + assert not is_group_metadata_v2(v2) + assert parse_group_metadata_v3(v3)["extension"] == (0, 1) + assert parse_group_metadata_v2(v2)["attributes"] == {"values": (0, 1)} + + def test_group_v3_extension_fields_are_validated() -> None: """Group extension payloads must be JSON values with a must-understand flag.""" doc = { diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 0b096b4b68..d15b3f118c 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -6,6 +6,7 @@ """ import json +import warnings import pytest from jsonschema import Draft202012Validator @@ -127,6 +128,24 @@ class Manifest(BaseModel): ] +def test_json_schema_generation_emits_no_warnings() -> None: + """Consumers can generate every public integration schema without warning filters.""" + field_types = ( + zmp.ZarrV3ArrayMetadata, + zmp.ZarrV2ArrayMetadata, + zmp.ZarrV3GroupMetadata, + zmp.ZarrV2GroupMetadata, + zmp.ZarrV3ConsolidatedMetadata, + zmp.ZarrV2ConsolidatedMetadata, + zmp.ZarrV3MetadataField, + ) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + for field_type in field_types: + TypeAdapter(field_type).json_schema() + + def test_v2_recursive_structured_dtype_is_in_pydantic_schema() -> None: """The schema accepts nested structured dtypes supported by the v2 specification.""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) @@ -163,6 +182,14 @@ def test_array_schemas_reject_negative_dimensions() -> None: _assert_runtime_and_schema_reject(field_type, doc) +def test_v2_array_schema_rejects_empty_filters() -> None: + """The v2 schema mirrors the runtime one-or-more filter rule.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["filters"] = [] + + _assert_runtime_and_schema_reject(zmp.ZarrV2ArrayMetadata, doc) + + @pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) def test_v3_array_schema_rejects_false_at_mandatory_extension_points(field: str) -> None: """Mandatory v3 extension points cannot opt out of understanding.""" From 693c9be800133efbc0a20c5686528a1e6bf1a262 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 21:28:05 +0200 Subject: [PATCH 48/48] fix(metadata): repair package CI Assisted-by: Codex:gpt-5 --- packages/zarr-metadata/pyproject.toml | 2 +- .../src/zarr_metadata/_common.py | 9 ++++++++- .../src/zarr_metadata/model/_validation.py | 19 ++++++++++++------- .../src/zarr_metadata/v2/array.py | 8 ++++++-- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 8dcf0a905a..10b97f1c1e 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -47,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/README.md" [dependency-groups] -test = ["pytest", "pydantic>=2.13"] +test = ["pytest", "pydantic>=2.13", "jsonschema"] [tool.hatch.version] source = "vcs" diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 23bdfd00ca..13c2457489 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -13,7 +13,14 @@ JSONValue = TypeAliasType( "JSONValue", - "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", + int + | float + | bool + | None + | str + | list["JSONValue"] + | tuple["JSONValue", ...] + | Mapping[str, "JSONValue"], ) """A recursive type alias for JSON-encodable values. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 35e6904edf..a12e1911b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -107,9 +107,13 @@ def _is_canonical_json(value: object) -> TypeIs[JSONValue]: if isinstance(value, (str, int, bool)) or value is None: return True if isinstance(value, (list, tuple)): - return all(_is_canonical_json(item) for item in value) + sequence = cast("list[object] | tuple[object, ...]", value) + return all(_is_canonical_json(item) for item in sequence) if isinstance(value, dict): - return all(isinstance(key, str) and _is_canonical_json(item) for key, item in value.items()) + mapping = cast("dict[object, object]", value) + return all( + isinstance(key, str) and _is_canonical_json(item) for key, item in mapping.items() + ) return False @@ -294,11 +298,12 @@ def validate_metadata_field_v3( def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: """Whether `value` is a v3 metadata field: a bare name or a named config.""" - return isinstance(value, str) or ( - isinstance(value, dict) - and _is_canonical_json(value) - and not validate_metadata_field_v3(value) - ) + if isinstance(value, str): + return True + if not isinstance(value, dict): + return False + field = cast("dict[object, object]", value) + return _is_canonical_json(field) and not validate_metadata_field_v3(field) def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 28085e4fbb..96c8b75036 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -10,8 +10,12 @@ ZarrV2DataTypeMetadata = TypeAliasType( "ZarrV2DataTypeMetadata", - "str | tuple[tuple[str, ZarrV2DataTypeMetadata] | " - "tuple[str, ZarrV2DataTypeMetadata, tuple[int, ...]], ...]", + str + | tuple[ + tuple[str, "ZarrV2DataTypeMetadata"] + | tuple[str, "ZarrV2DataTypeMetadata", tuple[int, ...]], + ..., + ], ) """The v2 dtype representation.