From d6f8483c765667bd2aa62f02923042b65923078f Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:26:12 +0200 Subject: [PATCH 1/9] chore: extract just recipes from hatch replacement proposal Extracts the Justfile from 98dcde719834bcd6fc010ffc4b07be5ae02fc2e4 (zarr-developers/zarr-python#4096). Environment and CI migration changes are adapted separately. Assisted-by: Codex:GPT-6 --- Justfile | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 Justfile diff --git a/Justfile b/Justfile new file mode 100644 index 0000000000..c8843bbed1 --- /dev/null +++ b/Justfile @@ -0,0 +1,140 @@ +# zarr-python developer task runner (https://github.com/casey/just). +# +# `just` is a thin verb-runner over uv; uv owns the environments. Each recipe is +# the single source of truth for a dev/CI task — CI calls the same recipes (via +# `uvx --from rust-just just `), so local and CI behavior cannot drift. +# +# Install just: uv tool install rust-just (or `brew install just`, `cargo install just`) +# List recipes: just (or `just --list`) +# +# The matrix lives in GitHub Actions; pass the Python version via UV_PYTHON +# (setup-uv sets it from `python-version`). Locally, override per call, e.g. +# UV_PYTHON=3.13 just test-optional + +# Extras + groups that make up the "optional" (full integration) test environment. +optional_deps := "--extra remote --extra optional --extra cli --extra cast-value-rs --group remote-tests" + +[private] +default: + @just --list + +[doc("Run the unit tests with the minimal dependency set")] +test-minimal *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Run the unit tests with the full (optional) integration dependency set")] +test-optional *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test {{optional_deps}} + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Generate an HTML coverage report (optional deps); open htmlcov/index.html")] +coverage-html *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test {{optional_deps}} + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks {{args}} + uv run --no-sync coverage html + +[doc("Run the slow Hypothesis property tests")] +hypothesis *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test {{optional_deps}} + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest -nauto --run-slow-hypothesis \ + tests/test_properties.py tests/test_store/test_stateful* {{args}} + uv run --no-sync coverage xml + +[doc("Validate executable code blocks in the docs (tests/test_docs.py)")] +doctest *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --extra remote --group remote-tests + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync --with pytest-examples pytest tests/test_docs.py -v {{args}} + +[doc("Run the benchmark suite (minimal deps)")] +benchmark *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test + uv run --no-sync pytest --benchmark-enable tests/benchmarks {{args}} + +[doc("Run the tests against the lowest supported direct dependency versions")] +min_deps *args: + #!/usr/bin/env bash + set -euo pipefail + # uv derives the floors from the `>=` constraints in pyproject.toml. + uv sync --resolution lowest-direct --no-default-groups \ + --group test --group remote-tests --extra remote --extra optional + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Run the tests against bleeding-edge (nightly + git main) dependencies")] +upstream *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --no-default-groups --group test --group remote-tests --extra remote + uv pip install --prerelease=allow \ + --index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ + --extra-index-url https://pypi.org/simple/ \ + numpy \ + "packaging @ git+https://github.com/pypa/packaging" \ + "numcodecs @ git+https://github.com/zarr-developers/numcodecs" \ + "s3fs @ git+https://github.com/fsspec/s3fs" \ + "universal_pathlib @ git+https://github.com/fsspec/universal_pathlib" \ + "typing_extensions @ git+https://github.com/python/typing_extensions" \ + "donfig @ git+https://github.com/pytroll/donfig" \ + "obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore" + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Run the GPU tests (requires CUDA + a GPU); `pytest -m gpu`")] +gpu *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test --extra gpu --extra optional + uv pip install pytest-examples + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest -m gpu --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Build the documentation (strict: warnings are errors)")] +docs-build *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --extra remote --group docs + DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true \ + uv run --no-sync mkdocs build --strict {{args}} + +[doc("Serve the documentation locally with live reload at http://0.0.0.0:8000/")] +docs-serve *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --extra remote --group docs + DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true \ + uv run --no-sync mkdocs serve --watch src {{args}} + +[doc("Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...)")] +lint *args: + prek run --all-files {{args}} + +[doc("Check that uv.lock is in sync with pyproject.toml")] +lock-check: + uv lock --check From de8e08ed49d13b37be9b59c13a6947d8862f5417 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:26:50 +0200 Subject: [PATCH 2/9] chore: use just for python helpers Extracts the Justfile changes from 693f694e50b9ae4cd7c6b0d93238b9ee9f8f6865 (zarr-developers/zarr-python#4096). Assisted-by: Codex:GPT-6 --- Justfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Justfile b/Justfile index c8843bbed1..9c38908d29 100644 --- a/Justfile +++ b/Justfile @@ -138,3 +138,11 @@ lint *args: [doc("Check that uv.lock is in sync with pyproject.toml")] lock-check: uv lock --check + +[doc("Check changelog entry filenames (pass a directory to check, default: changes/)")] +check-changelogs *dir: + uv run --no-sync python ci/check_changelog_entries.py {{dir}} + +[doc("Report unlinked types in the built docs (run `just docs-build` first)")] +check-doc-links *args: + uv run --no-sync python ci/check_unlinked_types.py {{args}} From b99abc42c29932a5ba8273570797044cb620dc0a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 14 Sep 2026 20:32:22 +0200 Subject: [PATCH 3/9] chore: centralize development and CI verbs in just Keep Hatch environment definitions while moving task commands into Justfile. Route root CI, Read the Docs, and contributor workflows through the same recipes, preserve quoted arguments, and ship recipes and helpers in the sdist. Assisted-by: Codex:GPT-6 --- .github/workflows/check_changelogs.yml | 13 +- .github/workflows/codspeed.yml | 4 +- .github/workflows/docs.yml | 11 +- .github/workflows/gpu_test.yml | 7 +- .github/workflows/hypothesis.yaml | 7 +- .github/workflows/lint.yml | 7 +- .github/workflows/nightly_wheels.yml | 4 +- .github/workflows/releases.yml | 4 +- .github/workflows/test.yml | 24 ++- .pre-commit-config.yaml | 2 +- .readthedocs.yaml | 6 +- Justfile | 244 ++++++++++++------------- changes/4096.misc.md | 1 + docs/contributing.md | 76 +++++--- pyproject.toml | 38 +--- 15 files changed, 218 insertions(+), 230 deletions(-) create mode 100644 changes/4096.misc.md diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index f642eb17ca..a8b84da7ba 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -24,14 +24,19 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - name: Install task runners + run: | + uv tool install hatch==1.16.5 + uv tool install rust-just==1.58.0 + - name: Check zarr-python changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py + run: just check-changelogs - name: Check zarr-metadata changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-metadata/changes + run: just check-changelogs packages/zarr-metadata/changes - name: Check zarr-indexing changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-indexing/changes + run: just check-changelogs packages/zarr-indexing/changes - name: Check zarr-http-server changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-http-server/changes + run: just check-changelogs packages/zarr-http-server/changes diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 1441950c3f..4694c8dbc7 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -31,10 +31,12 @@ jobs: uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' + - name: Install just + run: python -m pip install rust-just==1.58.0 - name: Run the benchmarks uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 env: ZARR_BENCHMARK_CLEAR_CACHE: '1' with: mode: walltime - run: hatch run test.py3.12-minimal:pytest tests/benchmarks --codspeed + run: HATCH_ENV=test.py3.12-minimal just benchmark-codspeed diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0f9c711a60..eafd188f45 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,18 +23,19 @@ jobs: with: persist-credentials: false - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - - run: uv sync --group docs + - run: uv tool install hatch==1.16.5 + - run: uv tool install rust-just==1.58.0 # Fast source-level guards that need no built site, so they run before the (slower) # build for a quick failure: every public export is in the API reference, and no # docstring/Markdown carries reStructuredText markup that MkDocs won't render. - - run: uv run python ci/check_documented_exports.py docs/api - - run: uv run python ci/lint_docs.py + - run: just check-doc-exports + - run: just lint-docs # --strict turns warnings into errors, so a docs code block that fails to execute # at build time (e.g. a non-exec python fence disrupting a later exec="true" block) # fails CI instead of merging as a silent warning. - - run: uv run mkdocs build --strict + - run: just docs-build env: DISABLE_MKDOCS_2_WARNING: "true" NO_MKDOCS_2_WARNING: "true" - - run: uv run python ci/check_unlinked_types.py + - run: just check-doc-links continue-on-error: true diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index c902bfdaa8..c4417b7a46 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -67,17 +67,18 @@ jobs: uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' + - name: Install just + run: python -m pip install rust-just==1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: gputest.py${{ matrix.python-version }} run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + just setup - name: Run Tests env: HATCH_ENV: gputest.py${{ matrix.python-version }} run: | - hatch env run --env "$HATCH_ENV" run-coverage + just gpu - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 352888f749..1937b95e79 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -65,12 +65,13 @@ jobs: uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' + - name: Install just + run: python -m pip install rust-just==1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + just setup # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - name: Restore cached hypothesis directory id: restore-hypothesis-cache @@ -89,7 +90,7 @@ jobs: PYTEST_ADDOPTS: "--report-log=output-${{ matrix.python-version }}-log.jsonl" run: | echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE" - hatch env run --env "$HATCH_ENV" run-hypothesis + just hypothesis # explicitly save the cache so it gets updated, also do this even if it fails. - name: Save cached hypothesis directory diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 24521aadc5..04b0167313 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,9 @@ jobs: uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true - - uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 + - name: Install just + run: uv tool install rust-just==1.58.0 + - name: Check justfile formatting + run: just just-check + - name: Lint + run: just lint diff --git a/.github/workflows/nightly_wheels.yml b/.github/workflows/nightly_wheels.yml index 2cb913511e..de625a8847 100644 --- a/.github/workflows/nightly_wheels.yml +++ b/.github/workflows/nightly_wheels.yml @@ -39,7 +39,9 @@ jobs: version: '1.16.5' - name: Build wheel and sdist - run: hatch build + run: | + pip install rust-just==1.58.0 + just build - name: Upload nightly wheels uses: scientific-python/upload-nightly-action@e76cfec8a4611fd02808a801b0ff5a7d7c1b2d99 diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 0865314915..b87c1fa7f7 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -42,7 +42,9 @@ jobs: with: version: '1.16.5' - name: Build wheel and sdist - run: hatch build + run: | + pip install rust-just==1.58.0 + just build - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: releases diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 650932309c..4cc249b806 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,19 +68,18 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install Hatch - run: python -m pip install hatch==1.16.5 + run: python -m pip install hatch==1.16.5 rust-just==1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + just setup - name: Run Tests env: HYPOTHESIS_PROFILE: ci HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | - hatch env run --env "$HATCH_ENV" run-coverage + just coverage - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 @@ -117,18 +116,17 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install Hatch - run: python -m pip install hatch==1.16.5 + run: python -m pip install hatch==1.16.5 rust-just==1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: ${{ matrix.dependency-set }} run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + just setup - name: Run Tests env: HATCH_ENV: ${{ matrix.dependency-set }} run: | - hatch env run --env "$HATCH_ENV" run-coverage + just coverage - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: @@ -152,13 +150,13 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install Hatch - run: python -m pip install hatch==1.16.5 + run: python -m pip install hatch==1.16.5 rust-just==1.58.0 - name: Set Up Hatch Env run: | - hatch run doctest:pip list + HATCH_ENV=doctest just setup - name: Run Tests run: | - hatch run doctest:test + just doctest benchmarks: name: Benchmark smoke test @@ -176,12 +174,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install Hatch - run: python -m pip install hatch==1.16.5 + run: python -m pip install hatch==1.16.5 rust-just==1.58.0 - name: Run Benchmarks env: ZARR_BENCHMARK_CLEAR_CACHE: '1' run: | - hatch env run --env "test.py3.13-minimal" run-benchmark + HATCH_ENV=test.py3.13-minimal just benchmark test-complete: name: Test complete diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54345c819e..b5d210511d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: - id: mypy name: mypy language: system - entry: uv run --frozen mypy + entry: just typecheck pass_filenames: false always_run: true types_or: [python, pyi] diff --git a/.readthedocs.yaml b/.readthedocs.yaml index dddf8449a4..3190c7180b 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,15 +19,15 @@ build: fi install: - pip install --upgrade pip - - pip install .[remote] --group docs + - pip install hatch==1.16.5 rust-just==1.58.0 pre_build: - | if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ]; then - towncrier build --version Unreleased --yes; + just changelog-build --version Unreleased --yes; fi build: html: - - mkdocs build --strict --site-dir $READTHEDOCS_OUTPUT/html + - just docs-build --site-dir $READTHEDOCS_OUTPUT/html mkdocs: configuration: mkdocs.yml diff --git a/Justfile b/Justfile index 9c38908d29..322235517b 100644 --- a/Justfile +++ b/Justfile @@ -1,148 +1,136 @@ -# zarr-python developer task runner (https://github.com/casey/just). -# -# `just` is a thin verb-runner over uv; uv owns the environments. Each recipe is -# the single source of truth for a dev/CI task — CI calls the same recipes (via -# `uvx --from rust-just just `), so local and CI behavior cannot drift. -# -# Install just: uv tool install rust-just (or `brew install just`, `cargo install just`) -# List recipes: just (or `just --list`) -# -# The matrix lives in GitHub Actions; pass the Python version via UV_PYTHON -# (setup-uv sets it from `python-version`). Locally, override per call, e.g. -# UV_PYTHON=3.13 just test-optional - -# Extras + groups that make up the "optional" (full integration) test environment. -optional_deps := "--extra remote --extra optional --extra cli --extra cast-value-rs --group remote-tests" - -[private] +# Development and CI verbs live here; Hatch owns Python environments in pyproject.toml. +# Install: pip install hatch==1.16.5 rust-just==1.58.0 uv +# Select test dependencies/interpreter: HATCH_ENV=test.py3.13-minimal just test +# On Windows, use Git Bash (the same shell used by the test workflow). +set shell := ["bash", "-eu", "-o", "pipefail", "-c"] +set windows-shell := ["bash", "-eu", "-o", "pipefail", "-c"] +set positional-arguments + +hatch_env := env("HATCH_ENV", "test.py3.12-optional") + +# List available recipes default: @just --list -[doc("Run the unit tests with the minimal dependency set")] -test-minimal *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --locked --no-default-groups --group test - if [ -n "${CI:-}" ]; then uv pip list; fi - uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ - --junitxml=junit.xml -o junit_family=legacy {{args}} - uv run --no-sync coverage xml - -[doc("Run the unit tests with the full (optional) integration dependency set")] -test-optional *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --locked --no-default-groups --group test {{optional_deps}} - if [ -n "${CI:-}" ]; then uv pip list; fi - uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ - --junitxml=junit.xml -o junit_family=legacy {{args}} - uv run --no-sync coverage xml - -[doc("Generate an HTML coverage report (optional deps); open htmlcov/index.html")] +# List available Python environments +envs: + hatch env show + +# Create the selected Python environment and list its installed packages +setup: + hatch env create {{ quote(hatch_env) }} + just list-env + +# List packages in the selected Python environment +list-env: + hatch run {{ quote(hatch_env) }}:pip list + +# Run unit tests; pass pytest arguments, e.g. just test -k 'array and resize' +test *args: + hatch run {{ quote(hatch_env) }}:pytest --ignore tests/benchmarks "$@" + +# Run unit tests and write coverage.xml and junit.xml +coverage *args: + hatch run {{ quote(hatch_env) }}:coverage run --source=src -m pytest --ignore tests/benchmarks --junitxml=junit.xml -o junit_family=legacy "$@" + hatch run {{ quote(hatch_env) }}:coverage xml + +# Run unit tests and generate an HTML coverage report coverage-html *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --locked --no-default-groups --group test {{optional_deps}} - uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks {{args}} - uv run --no-sync coverage html + hatch run {{ quote(hatch_env) }}:coverage run --source=src -m pytest --ignore tests/benchmarks "$@" + hatch run {{ quote(hatch_env) }}:coverage html -[doc("Run the slow Hypothesis property tests")] +# Serve the HTML coverage report (default port 8000) +coverage-serve *args: + hatch run {{ quote(hatch_env) }}:python -m http.server -d htmlcov "$@" + +# Run slow Hypothesis tests and write coverage.xml hypothesis *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --locked --no-default-groups --group test {{optional_deps}} - if [ -n "${CI:-}" ]; then uv pip list; fi - uv run --no-sync coverage run --source=src -m pytest -nauto --run-slow-hypothesis \ - tests/test_properties.py tests/test_store/test_stateful* {{args}} - uv run --no-sync coverage xml - -[doc("Validate executable code blocks in the docs (tests/test_docs.py)")] + hatch run {{ quote(hatch_env) }}:coverage run --source=src -m pytest -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* "$@" + hatch run {{ quote(hatch_env) }}:coverage xml + +# Validate executable documentation code blocks doctest *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --locked --no-default-groups --extra remote --group remote-tests - if [ -n "${CI:-}" ]; then uv pip list; fi - uv run --no-sync --with pytest-examples pytest tests/test_docs.py -v {{args}} + hatch run doctest:pytest tests/test_docs.py -v "$@" -[doc("Run the benchmark suite (minimal deps)")] +# Run the benchmark suite benchmark *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --locked --no-default-groups --group test - uv run --no-sync pytest --benchmark-enable tests/benchmarks {{args}} - -[doc("Run the tests against the lowest supported direct dependency versions")] -min_deps *args: - #!/usr/bin/env bash - set -euo pipefail - # uv derives the floors from the `>=` constraints in pyproject.toml. - uv sync --resolution lowest-direct --no-default-groups \ - --group test --group remote-tests --extra remote --extra optional - if [ -n "${CI:-}" ]; then uv pip list; fi - uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ - --junitxml=junit.xml -o junit_family=legacy {{args}} - uv run --no-sync coverage xml - -[doc("Run the tests against bleeding-edge (nightly + git main) dependencies")] -upstream *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --no-default-groups --group test --group remote-tests --extra remote - uv pip install --prerelease=allow \ - --index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ - --extra-index-url https://pypi.org/simple/ \ - numpy \ - "packaging @ git+https://github.com/pypa/packaging" \ - "numcodecs @ git+https://github.com/zarr-developers/numcodecs" \ - "s3fs @ git+https://github.com/fsspec/s3fs" \ - "universal_pathlib @ git+https://github.com/fsspec/universal_pathlib" \ - "typing_extensions @ git+https://github.com/python/typing_extensions" \ - "donfig @ git+https://github.com/pytroll/donfig" \ - "obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore" - if [ -n "${CI:-}" ]; then uv pip list; fi - uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ - --junitxml=junit.xml -o junit_family=legacy {{args}} - uv run --no-sync coverage xml - -[doc("Run the GPU tests (requires CUDA + a GPU); `pytest -m gpu`")] + hatch run {{ quote(hatch_env) }}:pytest --benchmark-enable tests/benchmarks "$@" + +# Run benchmarks under CodSpeed +benchmark-codspeed *args: + hatch run {{ quote(hatch_env) }}:pytest tests/benchmarks --codspeed "$@" + +# Run GPU tests with coverage (default environment: gputest.py3.12) gpu *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --locked --no-default-groups --group test --extra gpu --extra optional - uv pip install pytest-examples - if [ -n "${CI:-}" ]; then uv pip list; fi - uv run --no-sync coverage run --source=src -m pytest -m gpu --ignore tests/benchmarks \ - --junitxml=junit.xml -o junit_family=legacy {{args}} - uv run --no-sync coverage xml - -[doc("Build the documentation (strict: warnings are errors)")] + HATCH_ENV={{ quote(env("HATCH_ENV", "gputest.py3.12")) }} just coverage -m gpu "$@" + +# Build documentation (warnings are errors) docs-build *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --locked --no-default-groups --extra remote --group docs - DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true \ - uv run --no-sync mkdocs build --strict {{args}} + hatch run docs:mkdocs build --strict "$@" -[doc("Serve the documentation locally with live reload at http://0.0.0.0:8000/")] +# Serve documentation with live reload docs-serve *args: - #!/usr/bin/env bash - set -euo pipefail - uv sync --locked --no-default-groups --extra remote --group docs - DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true \ - uv run --no-sync mkdocs serve --watch src {{args}} + hatch run docs:mkdocs serve --watch src "$@" + +# Check that every public export has API documentation +check-doc-exports *args: + hatch run docs:python ci/check_documented_exports.py docs/api "$@" -[doc("Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...)")] +# Check documentation source conventions +lint-docs *args: + hatch run docs:python ci/lint_docs.py "$@" + +# Report unlinked types in built documentation +check-doc-links *args: + hatch run docs:python ci/check_unlinked_types.py "$@" + +# Run source documentation checks followed by a strict build +docs-check: check-doc-exports lint-docs docs-build + +# Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...) lint *args: - prek run --all-files {{args}} + uvx prek run --all-files "$@" + +# Run hooks with a custom selection, e.g. just hooks run --last-commit +hooks +args: + uvx prek "$@" + +# Install local pre-commit hooks +hooks-install: + uvx prek install + +# Type-check the library using the locked tooling environment +typecheck *args: + uv run --frozen mypy "$@" -[doc("Check that uv.lock is in sync with pyproject.toml")] +# Check that uv.lock is in sync with pyproject.toml lock-check: uv lock --check -[doc("Check changelog entry filenames (pass a directory to check, default: changes/)")] -check-changelogs *dir: - uv run --no-sync python ci/check_changelog_entries.py {{dir}} +# Update the dependency lockfile +lock *args: + uv lock "$@" -[doc("Report unlinked types in the built docs (run `just docs-build` first)")] -check-doc-links *args: - uv run --no-sync python ci/check_unlinked_types.py {{args}} +# Build the source distribution and wheel +build *args: + hatch build "$@" + +# Create a changelog fragment (interactive without arguments) +changelog *args: + hatch run docs:towncrier create "$@" + +# Preview the next release's changelog +changelog-draft *args: + hatch run docs:towncrier build --draft --version Unreleased "$@" + +# Build release notes; pass --version and --yes when preparing a release +changelog-build *args: + hatch run docs:towncrier build "$@" + +# Check changelog filenames (default: changes/; accepts a package changes directory) +check-changelogs *args: + hatch run dev:python ci/check_changelog_entries.py "$@" + +# Check recipe formatting +just-check: + just --fmt --check diff --git a/changes/4096.misc.md b/changes/4096.misc.md new file mode 100644 index 0000000000..ff4cf41a17 --- /dev/null +++ b/changes/4096.misc.md @@ -0,0 +1 @@ +Define development and CI commands in a root Justfile, with Hatch managing Python environments. diff --git a/docs/contributing.md b/docs/contributing.md index 369e60110a..5e374b49fd 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -80,19 +80,34 @@ git remote add upstream git@github.com:zarr-developers/zarr-python.git ### Creating a development environment -To work with the Zarr source code, it is recommended to use [hatch](https://hatch.pypa.io/latest/index.html) to create and manage development environments. Hatch will automatically install all Zarr dependencies using the same versions as are used by the core developers and continuous integration services. Assuming you have a Python 3 interpreter already installed, and you have cloned the Zarr source code and your current working directory is the root of the repository, you can do something like the following: +The root `Justfile` defines development and CI commands. [just](https://just.systems/) +runs these commands, while [Hatch](https://hatch.pypa.io/latest/index.html) manages +the Python environments declared in `pyproject.toml`. Install the task tools and uv (used by the lint and lock commands): ```bash -pip install hatch -hatch env show # list all available environments +pip install hatch==1.16.5 rust-just==1.58.0 uv +just # list available commands +just envs # list Python environments +just setup # create the default test environment +just test ``` -To verify that your development environment is working, you can run the unit tests for one of the test environments, e.g.: +Test recipes default to `test.py3.12-optional`. Set `HATCH_ENV` to select a different +interpreter or dependency set, just as CI does. On Windows, run these commands in +Git Bash. ```bash -hatch env run --env test.py3.12-optional run +HATCH_ENV=test.py3.13-minimal just test +HATCH_ENV=min_deps just coverage +HATCH_ENV=upstream just coverage +HATCH_ENV=gputest.py3.12 just gpu +just test tests/test_array.py -k 'resize and not async' ``` +Arguments after the recipe name are forwarded to the underlying tool. Use +`just --show test` to inspect a command. Package-specific commands live in the +`justfile` inside each package directory; run `just` there to list them. + ### Creating a branch Before you do any new work or submit a pull request, please open an issue on GitHub to report the bug or propose the feature you'd like to add. @@ -128,7 +143,7 @@ Again, any conflicts need to be resolved before submitting a pull request. Zarr includes a suite of unit tests. The simplest way to run the unit tests is to activate your development environment (see [creating a development environment](#creating-a-development-environment) above) and invoke: ```bash -hatch env run --env test.py3.12-optional run +just test ``` All tests are automatically run via GitHub Actions for every pull request and must pass before code can be accepted. Test coverage is also collected automatically via the Codecov service. @@ -137,46 +152,36 @@ All tests are automatically run via GitHub Actions for every pull request and mu All code must conform to the PEP8 standard. Regarding line length, lines up to 100 characters are allowed, although please try to keep under 90 wherever possible. -`Zarr` uses a set of git hooks managed by [`prek`](https://github.com/j178/prek), a fast, Rust-based pre-commit hook manager that is fully compatible with `.pre-commit-config.yaml` files. `prek` can be installed locally by running: - -```bash -uv tool install prek -``` - -or: - -```bash -pip install prek -``` +`Zarr` uses a set of git hooks managed by [`prek`](https://github.com/j178/prek), a fast, Rust-based pre-commit hook manager compatible with `.pre-commit-config.yaml`. The just recipes use `uvx` to run prek, installing it on demand. The hooks can be installed locally by running: ```bash -prek install +just hooks-install ``` This will run the checks every time a commit is created locally. The checks will by default only run on the files modified by a commit, but the checks can be triggered for all the files by running: ```bash -prek run --all-files +just lint ``` You can also run hooks only for files in a specific directory: ```bash -prek run --directory src/zarr +just hooks run --directory src/zarr ``` Or run hooks for files changed in the last commit: ```bash -prek run --last-commit +just hooks run --last-commit ``` To list all available hooks: ```bash -prek list +just hooks list ``` If you would like to skip the failing checks and push the code for further discussion, use the `--no-verify` option with `git commit`. @@ -188,7 +193,7 @@ If you would like to skip the failing checks and push the code for further discu Zarr strives to maintain 100% test coverage under the latest Python stable release. Both unit tests and docstring doctests are included when computing coverage. Running: ```bash -hatch env run --env test.py3.12-optional run-coverage +just coverage ``` will automatically run the test suite with coverage and produce an XML coverage report. This should be 100% before code can be accepted into the main code base. @@ -196,7 +201,7 @@ will automatically run the test suite with coverage and produce an XML coverage You can also generate an HTML coverage report by running: ```bash -hatch env run --env test.py3.12-optional run-coverage-html +just coverage-html ``` When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull request. Codecov coverage must also be 100% before code can be accepted. @@ -210,15 +215,17 @@ Zarr uses mkdocs for documentation, hosted on readthedocs.org. Documentation is The documentation can be built locally by running: ```bash -hatch --env docs run build +just docs-build ``` +`just docs-check` also runs the documentation source checks used in CI. + The resulting built documentation will be available in the `site` folder. -Hatch can also be used to serve continuously updating version of the documentation during development at [http://127.0.0.1:8000/](http://127.0.0.1:8000/). This can be done by running: +`just docs-serve` serves a continuously updating version of the documentation during development at [http://127.0.0.1:8000/](http://127.0.0.1:8000/). This can be done by running: ```bash -hatch --env docs run serve +just docs-serve ``` #### Adding executable code blocks in the documentation @@ -320,10 +327,10 @@ Sometimes, you may want the documentation to build quicker. You can disable code ### Changelog -zarr-python uses [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) to manage release notes. Most pull requests should include at least one news fragment describing the changes. To add a release note, you'll need the GitHub issue or pull request number and the type of your change (`feature`, `bugfix`, `doc`, `removal`, `misc`). With that, run `towncrier create` with your development environment, which will prompt you for the issue number, change type, and the news text: +zarr-python uses [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) to manage release notes. Most pull requests should include at least one news fragment describing the changes. To add a release note, you'll need the GitHub issue or pull request number and the type of your change (`feature`, `bugfix`, `doc`, `removal`, `misc`). With that, run `just changelog` with your development environment, which will prompt you for the issue number, change type, and the news text: ```bash -towncrier create +just changelog ``` Alternatively, you can manually create the files in the `changes` directory using the naming convention `{issue-number}.{change-type}.md`. @@ -437,6 +444,15 @@ Features in `zarr.experimental` carry no stability guarantees. They may be chang Zarr uses [pytest-benchmark](https://pytest-benchmark.readthedocs.io/en/latest/) for running performance benchmarks as part of our test suite. The benchmarks are found in `tests/benchmarks`. By default pytest is configured to run these benchmarks as plain tests (i.e., no benchmarking). To run -a benchmark with timing measurements, use the `--benchmark-enable` when invoking `pytest`. +a benchmark with timing measurements, run `just benchmark`. Pass pytest arguments +to select benchmarks, for example `just benchmark -k test_morton_order`. The benchmarks are run as part of the continuous integration suite through [codspeed](https://app.codspeed.io/zarr-developers/zarr-python). + +## Building distributions and maintaining dependencies + +Run `just build` to produce a source distribution and wheel in `dist/`. +Use `just lock-check` to check the dependency lockfile, or `just lock` to update it. +`just typecheck` runs the type checker independently of the other lint hooks. +Preview release notes with `just changelog-draft`; use `just check-changelogs` +to validate fragment names, optionally passing a package's `changes/` directory. diff --git a/pyproject.toml b/pyproject.toml index 57443aed80..5c5eb92bfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,8 @@ include = [ "/examples", "/mkdocs.yml", "/mkdocs_hooks.py", + "/Justfile", + "/ci", ] [project] @@ -205,25 +207,6 @@ matrix.deps.dependency-groups = [ {value = "remote-tests", if = ["optional"]}, ] -[tool.hatch.envs.test.scripts] -run-coverage = [ - "coverage run --source=src -m pytest --ignore tests/benchmarks --junitxml=junit.xml -o junit_family=legacy {args:}", - "coverage xml", -] -run-coverage-html = [ - "coverage run --source=src -m pytest --ignore tests/benchmarks {args:}", - "coverage html", -] -run = "pytest --ignore tests/benchmarks" -run-verbose = "run-coverage --verbose" -run-hypothesis = [ - "coverage run --source=src -m pytest -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* {args:}", - "coverage xml", -] -run-benchmark = "pytest --benchmark-enable tests/benchmarks" -serve-coverage-html = "python -m http.server -d htmlcov 8000" -list-env = "pip list" - [tool.hatch.envs.gputest] template = "test" extra-dependencies = [ @@ -238,13 +221,6 @@ features = ["gpu"] [[tool.hatch.envs.gputest.matrix]] python = ["3.12", "3.13"] -[tool.hatch.envs.gputest.scripts] -run-coverage = [ - "coverage run --source=src -m pytest -m gpu --junitxml=junit.xml -o junit_family=legacy --ignore tests/benchmarks {args:}", - "coverage xml", -] -run = "pytest -m gpu --ignore tests/benchmarks" - [tool.hatch.envs.upstream] template = 'test' python = "3.14" @@ -297,12 +273,6 @@ dependency-groups = ['docs'] DISABLE_MKDOCS_2_WARNING = "true" NO_MKDOCS_2_WARNING = "true" -[tool.hatch.envs.docs.scripts] -serve = "mkdocs serve --watch src" -build = "mkdocs build" -check = "mkdocs build --strict" -readthedocs = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT/html" - [tool.hatch.envs.doctest] description = "Test environment for validating executable code blocks in documentation" features = ['remote'] @@ -311,10 +281,6 @@ extra-dependencies = [ "pytest-examples", ] -[tool.hatch.envs.doctest.scripts] -test = "pytest tests/test_docs.py -v" -list-env = "pip list" - [tool.ruff] line-length = 100 force-exclude = true From 762a17dbd9bf7b9389b4a92f7777db3b8f29fc9b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 14 Sep 2026 20:42:37 +0200 Subject: [PATCH 4/9] chore: delegate package commands to their justfiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose package recipes from the root while retaining each package’s command definitions and working directory. Assisted-by: Codex:GPT-6 --- Justfile | 12 ++++++++++++ docs/contributing.md | 13 ++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 322235517b..dab90ee5e3 100644 --- a/Justfile +++ b/Justfile @@ -134,3 +134,15 @@ check-changelogs *args: # Check recipe formatting just-check: just --fmt --check + +# Run a zarr-metadata recipe, or list its recipes with no arguments +zarr-metadata *args: + just --justfile packages/zarr-metadata/justfile "$@" + +# Run a zarr-indexing recipe, or list its recipes with no arguments +zarr-indexing *args: + just --justfile packages/zarr-indexing/justfile "$@" + +# Run a zarr-http-server recipe, or list its recipes with no arguments +zarr-http-server *args: + just --justfile packages/zarr-http-server/justfile "$@" diff --git a/docs/contributing.md b/docs/contributing.md index 5e374b49fd..19ee571d6f 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -106,7 +106,18 @@ just test tests/test_array.py -k 'resize and not async' Arguments after the recipe name are forwarded to the underlying tool. Use `just --show test` to inspect a command. Package-specific commands live in the -`justfile` inside each package directory; run `just` there to list them. +`justfile` inside each package directory. The root recipes delegate to these files, +so you can also run package commands from the repository root: + +```bash +just zarr-metadata # list this package's recipes +just zarr-metadata test +just zarr-indexing test-tensorstore +just zarr-http-server docs-check +``` + +The package justfile sets the working directory and defines the command and its +environment. Root recipes forward arguments without duplicating those definitions. ### Creating a branch From df1ff8794508d37d8fa7de2003720375ff82789e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 10:18:24 +0200 Subject: [PATCH 5/9] fix(ci): harden the just migration's environment, pinning and release path Review follow-ups to the Justfile centralization: - `just gpu` read HATCH_ENV, so an exported CPU test environment silently redirected `pytest -m gpu` into an env built without the gpu feature. It now reads GPU_HATCH_ENV, which nothing else sets. - Pin `uvx prek` to 0.5.3 and restore hook-environment caching in the lint workflow, so lint no longer floats on whatever prek PyPI serves that day. - Restore the mypy hook's `uv run --frozen mypy` entry: routing it through `just typecheck` made just a prerequisite for committing in every existing clone, and expanded to the identical command. - Run the changelog filename check with `uv run --no-project python` instead of building the `dev` hatch env (test + remote-tests + docs + mypy) for a script that imports only sys and pathlib. - Extend `just just-check` over packages/*/justfile, which the root delegation recipes depend on and the formatting gate did not cover. - Set up Python in the codspeed job before pip-installing into it, matching the other workflows; that runner image was never given one. - Revert the release and nightly-wheel jobs to `hatch build`, so the publishing path does not fetch rust-just to run a pure alias. - Renumber the changelog fragment to this pull request and credit #4096 in the body, so the rendered note does not link to an unrelated upstream PR. Assisted-by: ClaudeCode:claude-opus-5 --- .github/workflows/check_changelogs.yml | 6 ++---- .github/workflows/codspeed.yml | 4 ++++ .github/workflows/gpu_test.yml | 5 ++++- .github/workflows/lint.yml | 10 ++++++++++ .github/workflows/nightly_wheels.yml | 4 +--- .github/workflows/releases.yml | 4 +--- .pre-commit-config.yaml | 2 +- Justfile | 21 ++++++++++++++------- changes/339.misc.md | 1 + changes/4096.misc.md | 1 - docs/contributing.md | 6 ++++-- 11 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 changes/339.misc.md delete mode 100644 changes/4096.misc.md diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index a8b84da7ba..7c81fbaba3 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -24,10 +24,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - - name: Install task runners - run: | - uv tool install hatch==1.16.5 - uv tool install rust-just==1.58.0 + - name: Install just + run: uv tool install rust-just==1.58.0 - name: Check zarr-python changelog entries run: just check-changelogs diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 4694c8dbc7..8e3ecdd82f 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -27,6 +27,10 @@ jobs: with: fetch-depth: 0 persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index c4417b7a46..bcf933fe9c 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -69,6 +69,9 @@ jobs: version: '1.16.5' - name: Install just run: python -m pip install rust-just==1.58.0 + # Two names for the same environment on purpose: `just setup` builds whatever + # HATCH_ENV points at, while `just gpu` reads GPU_HATCH_ENV so that an + # ambient HATCH_ENV can never redirect `pytest -m gpu` into a CPU environment. - name: Set Up Hatch Env env: HATCH_ENV: gputest.py${{ matrix.python-version }} @@ -76,7 +79,7 @@ jobs: just setup - name: Run Tests env: - HATCH_ENV: gputest.py${{ matrix.python-version }} + GPU_HATCH_ENV: gputest.py${{ matrix.python-version }} run: | just gpu diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 04b0167313..a41f83124c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,6 +32,16 @@ jobs: enable-cache: true - name: Install just run: uv tool install rust-just==1.58.0 + # `uvx prek` builds each hook's environment from scratch, so cache them the + # way the prek action used to. Keyed on the hook config: a new pinned rev or + # a new hook is exactly when the cached environments stop being valid. + - name: Cache prek hook environments + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/prek + key: prek-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + prek-${{ runner.os }}- - name: Check justfile formatting run: just just-check - name: Lint diff --git a/.github/workflows/nightly_wheels.yml b/.github/workflows/nightly_wheels.yml index de625a8847..2cb913511e 100644 --- a/.github/workflows/nightly_wheels.yml +++ b/.github/workflows/nightly_wheels.yml @@ -39,9 +39,7 @@ jobs: version: '1.16.5' - name: Build wheel and sdist - run: | - pip install rust-just==1.58.0 - just build + run: hatch build - name: Upload nightly wheels uses: scientific-python/upload-nightly-action@e76cfec8a4611fd02808a801b0ff5a7d7c1b2d99 diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index b87c1fa7f7..0865314915 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -42,9 +42,7 @@ jobs: with: version: '1.16.5' - name: Build wheel and sdist - run: | - pip install rust-just==1.58.0 - just build + run: hatch build - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: releases diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b5d210511d..54345c819e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: - id: mypy name: mypy language: system - entry: just typecheck + entry: uv run --frozen mypy pass_filenames: false always_run: true types_or: [python, pyi] diff --git a/Justfile b/Justfile index dab90ee5e3..0ebeb39ff0 100644 --- a/Justfile +++ b/Justfile @@ -7,6 +7,12 @@ set windows-shell := ["bash", "-eu", "-o", "pipefail", "-c"] set positional-arguments hatch_env := env("HATCH_ENV", "test.py3.12-optional") +# Deliberately a different variable from HATCH_ENV: `just gpu` must not inherit a +# CPU test environment that happens to be exported in the caller's shell, which +# would run `pytest -m gpu` against an environment built without the gpu feature. +gpu_env := env("GPU_HATCH_ENV", "gputest.py3.12") +# Pinned so a prek release cannot change what CI lints without a commit here. +prek_version := "0.5.3" # List available recipes default: @@ -60,9 +66,9 @@ benchmark *args: benchmark-codspeed *args: hatch run {{ quote(hatch_env) }}:pytest tests/benchmarks --codspeed "$@" -# Run GPU tests with coverage (default environment: gputest.py3.12) +# Run GPU tests with coverage; select the environment with GPU_HATCH_ENV gpu *args: - HATCH_ENV={{ quote(env("HATCH_ENV", "gputest.py3.12")) }} just coverage -m gpu "$@" + HATCH_ENV={{ quote(gpu_env) }} just coverage -m gpu "$@" # Build documentation (warnings are errors) docs-build *args: @@ -89,15 +95,15 @@ docs-check: check-doc-exports lint-docs docs-build # Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...) lint *args: - uvx prek run --all-files "$@" + uvx prek@{{ prek_version }} run --all-files "$@" # Run hooks with a custom selection, e.g. just hooks run --last-commit hooks +args: - uvx prek "$@" + uvx prek@{{ prek_version }} "$@" # Install local pre-commit hooks hooks-install: - uvx prek install + uvx prek@{{ prek_version }} install # Type-check the library using the locked tooling environment typecheck *args: @@ -129,11 +135,12 @@ changelog-build *args: # Check changelog filenames (default: changes/; accepts a package changes directory) check-changelogs *args: - hatch run dev:python ci/check_changelog_entries.py "$@" + uv run --no-project python ci/check_changelog_entries.py "$@" -# Check recipe formatting +# Check recipe formatting of the root Justfile and every package justfile just-check: just --fmt --check + for f in packages/*/justfile; do just --justfile "$f" --fmt --check; done # Run a zarr-metadata recipe, or list its recipes with no arguments zarr-metadata *args: diff --git a/changes/339.misc.md b/changes/339.misc.md new file mode 100644 index 0000000000..7b55499cf1 --- /dev/null +++ b/changes/339.misc.md @@ -0,0 +1 @@ +Define development and CI commands in a root Justfile, with Hatch managing Python environments. Extracted from [#4096](https://github.com/zarr-developers/zarr-python/pull/4096). diff --git a/changes/4096.misc.md b/changes/4096.misc.md deleted file mode 100644 index ff4cf41a17..0000000000 --- a/changes/4096.misc.md +++ /dev/null @@ -1 +0,0 @@ -Define development and CI commands in a root Justfile, with Hatch managing Python environments. diff --git a/docs/contributing.md b/docs/contributing.md index 19ee571d6f..42e7d71dfd 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -93,14 +93,16 @@ just test ``` Test recipes default to `test.py3.12-optional`. Set `HATCH_ENV` to select a different -interpreter or dependency set, just as CI does. On Windows, run these commands in +interpreter or dependency set, just as CI does. `just gpu` reads `GPU_HATCH_ENV` +instead, so an exported `HATCH_ENV` cannot silently send GPU tests to an +environment built without the `gpu` feature. On Windows, run these commands in Git Bash. ```bash HATCH_ENV=test.py3.13-minimal just test HATCH_ENV=min_deps just coverage HATCH_ENV=upstream just coverage -HATCH_ENV=gputest.py3.12 just gpu +GPU_HATCH_ENV=gputest.py3.12 just gpu just test tests/test_array.py -k 'resize and not async' ``` From 050e42dac1412646030af2e3b0e77b8e263a220f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 10:24:52 +0200 Subject: [PATCH 6/9] fix(ci): install prek persistently for git hooks; run setup steps in one just process `just hooks-install` ran `uvx prek install`, and prek writes a hook shim that hard-codes the binary it was installed from, falling back to `prek` on PATH. Under uvx that path is an entry in uv's archive cache, so the first `uv cache prune` (or a bump of the pinned version) broke every commit with `prek: not found`, and nothing put `prek` on PATH for the fallback. Install it as a pinned `uv tool` instead: a changed pin upgrades in place, a repeated run is a no-op, and the shim points at a path that stays. `setup` re-invoked `just list-env`, which dropped a `just hatch_env=... setup` override on the floor because the child process re-read HATCH_ENV. Run it as a subsequent dependency so both steps see the same value. Also retire the last two comments naming the removed hatch scripts. Assisted-by: ClaudeCode:claude-fable-5-1 --- Justfile | 11 +++++++---- docs/contributing.md | 4 ++-- lychee.toml | 2 +- tests/test_docs.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Justfile b/Justfile index 0ebeb39ff0..641671e6ac 100644 --- a/Justfile +++ b/Justfile @@ -23,9 +23,8 @@ envs: hatch env show # Create the selected Python environment and list its installed packages -setup: +setup: && list-env hatch env create {{ quote(hatch_env) }} - just list-env # List packages in the selected Python environment list-env: @@ -101,9 +100,13 @@ lint *args: hooks +args: uvx prek@{{ prek_version }} "$@" -# Install local pre-commit hooks +# Install local pre-commit hooks. prek is installed as a persistent uv tool, not run +# through uvx: the hook shim prek writes into .git/hooks hard-codes the binary path +# it was installed from and falls back to `prek` on PATH, and a uvx archive path +# stops existing at the next `uv cache prune`. hooks-install: - uvx prek@{{ prek_version }} install + uv tool install prek=={{ prek_version }} + prek install # Type-check the library using the locked tooling environment typecheck *args: diff --git a/docs/contributing.md b/docs/contributing.md index 42e7d71dfd..5c3cfbe8b0 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -82,7 +82,7 @@ git remote add upstream git@github.com:zarr-developers/zarr-python.git The root `Justfile` defines development and CI commands. [just](https://just.systems/) runs these commands, while [Hatch](https://hatch.pypa.io/latest/index.html) manages -the Python environments declared in `pyproject.toml`. Install the task tools and uv (used by the lint and lock commands): +the Python environments declared in `pyproject.toml`. Install the task tools and uv (several recipes run tooling through it): ```bash pip install hatch==1.16.5 rust-just==1.58.0 uv @@ -165,7 +165,7 @@ All tests are automatically run via GitHub Actions for every pull request and mu All code must conform to the PEP8 standard. Regarding line length, lines up to 100 characters are allowed, although please try to keep under 90 wherever possible. -`Zarr` uses a set of git hooks managed by [`prek`](https://github.com/j178/prek), a fast, Rust-based pre-commit hook manager compatible with `.pre-commit-config.yaml`. The just recipes use `uvx` to run prek, installing it on demand. +`Zarr` uses a set of git hooks managed by [`prek`](https://github.com/j178/prek), a fast, Rust-based pre-commit hook manager compatible with `.pre-commit-config.yaml`. The recipes pin the prek version: `just lint` and `just hooks` run it through `uvx`, and `just hooks-install` installs it as a persistent `uv tool` so the git hook can find it on later commits. The hooks can be installed locally by running: diff --git a/lychee.toml b/lychee.toml index dccb3001dc..9bb9be5caf 100644 --- a/lychee.toml +++ b/lychee.toml @@ -16,7 +16,7 @@ exclude_path = [ # URL patterns to ignore (regex, matched against the full URL). exclude = [ - # Local docs preview server shown in the contributing guide ("hatch run serve"), + # Local docs preview server shown in the contributing guide ("just docs-serve"), # documentation of a command rather than a reachable link. '^https?://0\.0\.0\.0', '^https?://(localhost|127\.0\.0\.1)(:\d+)?', diff --git a/tests/test_docs.py b/tests/test_docs.py index 874d90d4e5..6361591a5e 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -210,7 +210,7 @@ def test_test_only_blocks_come_last() -> None: Because we cannot statically tell which later blocks are state-dependent, this guard enforces the simple, safe convention only for the blocks we author this way (test="true" marker-bound examples like s3/gpu). It is NOT a complete build-hazard - check -- the authoritative check is `mkdocs build --strict` (the docs:check CI job), + check -- the authoritative check is `mkdocs build --strict` (`just docs-build` in CI), which catches the exec="false" case too. This guard just turns the common test-only case into a fast, local failure.""" # Collect, per published-docs file, the start lines of test-only and exec blocks. From 4aa6b9565e5d3f33ebc7793f95f6d1066801ae3c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 10:43:33 +0200 Subject: [PATCH 7/9] chore: renumber the changelog fragment to the upstream pull request towncrier renders the fragment name as a link into zarr-developers/zarr-python, so the fork's PR number would have pointed the released note at an unrelated upstream pull request. #4372 is the PR that merges this work. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- changes/{339.misc.md => 4372.misc.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{339.misc.md => 4372.misc.md} (100%) diff --git a/changes/339.misc.md b/changes/4372.misc.md similarity index 100% rename from changes/339.misc.md rename to changes/4372.misc.md From 7014615850650a588a30e55a3f28a499b412eeed Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 11:37:56 +0200 Subject: [PATCH 8/9] ci: install just as a binary in the benchmark job Adding `actions/setup-python` to this job so that `pip install rust-just` had an interpreter also changed which interpreter hatch built `test.py3.12-minimal` on: the runner's own Python was replaced by /opt/hostedtoolcache/Python/3.12.5/arm64. CodSpeed measured the result as a 12.97% degradation across 26 benchmarks, spread -12.76% to -13.99% -- a range of 1.2 points over slice indexing, sharded Morton indexing, three array shapes and two store types. Nothing in this branch touches src/, tests/benchmarks/ or packages/; that uniformity is an interpreter swap, not a regression, and CodSpeed flagged it as "different runtime environments detected". Install just with extractions/setup-just instead, matching the three package workflows. No interpreter is set up, so hatch resolves the same Python the baseline used, and `just-version` is pinned to satisfy zizmor's unpinned-tools audit. The benchmarks need a re-run to produce a comparison against a matching environment; the numbers on the previous run should not be acknowledged as a regression. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .github/workflows/codspeed.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 8e3ecdd82f..03a7c0d9b6 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -27,16 +27,19 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' + # Installed as a binary rather than with `pip install rust-just`, which would + # need an interpreter set up in this job. Benchmarks are only comparable across + # runs if the interpreter underneath them does not move: pinning a Python here + # measured a uniform ~13% slowdown on every benchmark, because hatch then built + # the environment on that interpreter instead of the runner's own. - name: Install just - run: python -m pip install rust-just==1.58.0 + uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + with: + just-version: 1.58.0 - name: Run the benchmarks uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 env: From ea821011fcc40d123fd70a4faec657afdd9dcef2 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 17 Sep 2026 16:14:14 +0200 Subject: [PATCH 9/9] ci: install just from official release binaries; lock the docs toolchain Follow-ups from an adversarial audit of this branch. Supply chain. `rust-just` is a third-party repackaging of just on PyPI (Repository: github.com/gnpaone/rust-just) with no attestation on any of its 17 wheels, and this branch had introduced it in ten places including the self-hosted GPU runner and Read the Docs. All 23 just installs across the repo now come from casey/just's own release binaries via extractions/setup-crate@7577c1bd (v2.0.1), a node action whose SHA pins every line of executed code -- rather than extractions/setup-just, a composite wrapper that astral-sh/python-build-standalone#771 moved off for that reason. This also retires the 12 pre-existing setup-just call sites, so the repo has one mechanism instead of four. setup-crate does not verify checksums (extractions/setup-just#20 is open); Read the Docs, where no action is available, fetches the release tarball and verifies it against the SHA256SUMS casey/just publishes. Lockfile. docs.yml had stopped resolving from uv.lock: `uv sync --group docs` became `hatch run docs:`, and hatch never reads the lockfile, so the entire docs toolchain floated -- including numcodecs[msgpack], which carries no version constraint. Dependabot's uv ecosystem exists to keep that lock fresh and no job exercised it any more. The eight docs and changelog recipes now run `uv run --frozen --group docs`, so single-version tooling comes from the lock (as mypy already did) while hatch keeps the test environments, which exist per interpreter and per dependency set and cannot live in one lockfile. The hatch docs environment is removed as unused, and neither docs.yml nor Read the Docs installs hatch at all now. Argument forwarding. The three package justfiles splatted `{{ args }}` unquoted, so `just zarr-metadata test -k 'a and b'` reached pytest as `-k a` plus two stray paths, and command substitution in an argument executed. They now use `set positional-arguments` and `"$@"` like the root, which is what the contributing guide already claimed. Also, regressions from my own earlier commits on this branch: `just lint` had dropped `--show-diff-on-failure --color=always`, which j178/prek-action passed by default, so a hook that rewrote a file failed with no diff; a four-line comment above `hooks-install` displaced its `just --list` description, since just reads only the last comment line; the prek cache key did not cover prek_version and had a restore-keys fallback that could reuse a store built by a different prek; `just just-check` ran before `just lint`, so a cosmetic formatting nit hid every real lint result; and the `packages/*/justfile` glob lacked nullglob, which breaks in the sdist, where /Justfile ships but /packages deliberately does not. Docs: the install instructions used `pip install`, which fails on an externally-managed interpreter; `just just-check` was an undocumented required check; two sentences still told contributors to activate an environment that no longer exists; and the changelog fragment did not mention that the Hatch script tables were removed. Adds a .gitattributes rule so a Windows checkout does not get a CRLF Justfile that `just --fmt --check` rejects on every line. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .gitattributes | 6 +++++ .github/workflows/check_changelogs.yml | 5 +++- .github/workflows/codspeed.yml | 14 +++++----- .github/workflows/docs.yml | 7 +++-- .github/workflows/gpu_test.yml | 5 +++- .github/workflows/hypothesis.yaml | 5 +++- .github/workflows/lint.yml | 19 ++++++++----- .github/workflows/test.yml | 28 ++++++++++++++++--- .github/workflows/zarr-http-server.yml | 28 +++++++++---------- .github/workflows/zarr-indexing.yml | 28 +++++++++---------- .github/workflows/zarr-metadata.yml | 28 +++++++++---------- .readthedocs.yaml | 11 +++++++- Justfile | 37 +++++++++++++++----------- changes/4372.misc.md | 2 +- docs/contributing.md | 18 +++++++++---- packages/zarr-http-server/justfile | 5 +++- packages/zarr-indexing/justfile | 7 +++-- packages/zarr-metadata/justfile | 5 +++- pyproject.toml | 8 ------ 19 files changed, 162 insertions(+), 104 deletions(-) diff --git a/.gitattributes b/.gitattributes index 57eb8a8807..f507ce96ca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,9 @@ *.py linguist-language=python *.ipynb linguist-documentation .git_archival.txt export-subst + +# just refuses to format a CRLF file, so `just just-check` would fail on a Windows +# checkout with core.autocrlf=true before the contributor has changed anything. +Justfile text eol=lf +*/justfile text eol=lf +packages/*/justfile text eol=lf diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index 740c0ae346..960451cc4c 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -25,7 +25,10 @@ jobs: uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - run: uv tool install rust-just==1.58.0 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Check zarr-python changelog entries run: just check-changelogs diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 03a7c0d9b6..a58ce40a9d 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -31,15 +31,15 @@ jobs: uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' - # Installed as a binary rather than with `pip install rust-just`, which would - # need an interpreter set up in this job. Benchmarks are only comparable across - # runs if the interpreter underneath them does not move: pinning a Python here - # measured a uniform ~13% slowdown on every benchmark, because hatch then built - # the environment on that interpreter instead of the runner's own. + # No interpreter is set up in this job on purpose. Benchmarks are only comparable + # across runs if the interpreter underneath them does not move: adding a pinned + # Python here measured a uniform ~13% slowdown on every benchmark, because hatch + # then built the environment on that interpreter instead of the runner's own. - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run the benchmarks uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 env: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dc30c7f210..990866a109 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,8 +23,11 @@ jobs: with: persist-credentials: false - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - - run: uv tool install hatch==1.16.5 - - run: uv tool install rust-just==1.58.0 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 # Fast source-level guards that need no built site, so they run before the (slower) # build for a quick failure: every public export is in the API reference, and no # docstring/Markdown carries reStructuredText markup that MkDocs won't render. diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 4432d512fd..a8e989f580 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -68,7 +68,10 @@ jobs: with: version: '1.16.5' - name: Install just - run: python -m pip install rust-just==1.58.0 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 # Two names for the same environment on purpose: `just setup` builds whatever # HATCH_ENV points at, while `just gpu` reads GPU_HATCH_ENV so that an # ambient HATCH_ENV can never redirect `pytest -m gpu` into a CPU environment. diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 5293557b11..3be6f3ba60 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -66,7 +66,10 @@ jobs: with: version: '1.16.5' - name: Install just - run: python -m pip install rust-just==1.58.0 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ffd0ae4e87..f47fbcac11 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -31,7 +31,10 @@ jobs: with: enable-cache: true - name: Install just - run: uv tool install rust-just==1.58.0 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 # `uvx prek` builds each hook's environment from scratch, so cache them the # way the prek action used to. Keyed on the hook config: a new pinned rev or # a new hook is exactly when the cached environments stop being valid. @@ -39,10 +42,14 @@ jobs: uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/prek - key: prek-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} - restore-keys: | - prek-${{ runner.os }}- - - name: Check justfile formatting - run: just just-check + # Justfile is hashed too: prek_version lives there, and a bumped prek + # must not restore a store built by the previous one. No restore-keys + # fallback for the same reason. + key: prek-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml', 'Justfile') }} - name: Lint run: just lint + - name: Check justfile formatting + # After the linters, and never masking them: this is cosmetic, and a + # mis-formatted recipe should not cost someone their ruff/mypy results. + if: always() + run: just just-check diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 566cc79927..0d7d396fff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch - run: python -m pip install hatch==1.16.5 rust-just==1.58.0 + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} @@ -116,7 +121,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch - run: python -m pip install hatch==1.16.5 rust-just==1.58.0 + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: ${{ matrix.dependency-set }} @@ -150,7 +160,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch - run: python -m pip install hatch==1.16.5 rust-just==1.58.0 + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env run: | HATCH_ENV=doctest just setup @@ -174,7 +189,12 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch - run: python -m pip install hatch==1.16.5 rust-just==1.58.0 + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Run Benchmarks env: ZARR_BENCHMARK_CLEAR_CACHE: '1' diff --git a/.github/workflows/zarr-http-server.yml b/.github/workflows/zarr-http-server.yml index b90528d40f..4a8fcca101 100644 --- a/.github/workflows/zarr-http-server.yml +++ b/.github/workflows/zarr-http-server.yml @@ -53,11 +53,10 @@ jobs: - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Sync test dependency groups # The examples group carries the deps the README examples need, so the # test that reads a served array back with a zarr client runs here @@ -80,11 +79,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run ruff run: just lint @@ -106,11 +104,10 @@ jobs: - name: Set up Python run: uv python install 3.12 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Sync test dependency group run: uv sync --group test --python 3.12 - name: Run mypy @@ -132,11 +129,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Build docs run: just docs-check diff --git a/.github/workflows/zarr-indexing.yml b/.github/workflows/zarr-indexing.yml index 5d74c6a5e9..9af8ed1e3d 100644 --- a/.github/workflows/zarr-indexing.yml +++ b/.github/workflows/zarr-indexing.yml @@ -40,11 +40,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} # The suite imports nothing from `zarr`; it runs against the repo-root @@ -73,11 +72,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run ruff # The ruff version pin lives in packages/zarr-indexing/justfile. run: just lint @@ -102,11 +100,10 @@ jobs: - name: Sync test dependency group run: uv sync --group test --python 3.12 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run pyright # The pyright invocation lives in packages/zarr-indexing/justfile. run: just typecheck @@ -127,11 +124,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Build docs # The strict mkdocs build lives in packages/zarr-indexing/justfile. run: just docs-check diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 521a4f270c..3c42f4810f 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -43,11 +43,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Sync test dependency group @@ -69,11 +68,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run ruff run: just lint @@ -93,11 +91,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run pyright # The pyright version and interpreter pins live in the justfile. run: just typecheck @@ -118,11 +115,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Build docs run: just docs-check diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3190c7180b..872eb2be80 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,7 +19,16 @@ build: fi install: - pip install --upgrade pip - - pip install hatch==1.16.5 rust-just==1.58.0 + # The docs recipes resolve their toolchain from uv.lock, so uv is all that is + # needed here besides just itself. + - pip install uv==0.12.9 + # No GitHub Action is available on Read the Docs, so fetch the official just + # release and verify it against the checksum casey/just publishes. Installed + # into the build virtualenv's bin, which is already on PATH. + - | + curl -fsSL -o /tmp/just.tar.gz https://github.com/casey/just/releases/download/1.58.0/just-1.58.0-x86_64-unknown-linux-musl.tar.gz + echo "4a5cc2f53e6f0f8c59092a6cc38291eb729d46a7dd95d3ae582008881b84931d /tmp/just.tar.gz" | sha256sum -c - + tar -xzf /tmp/just.tar.gz -C "$READTHEDOCS_VIRTUALENV_PATH/bin" just pre_build: - | if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ]; diff --git a/Justfile b/Justfile index 641671e6ac..6e56644207 100644 --- a/Justfile +++ b/Justfile @@ -1,5 +1,5 @@ # Development and CI verbs live here; Hatch owns Python environments in pyproject.toml. -# Install: pip install hatch==1.16.5 rust-just==1.58.0 uv +# Install: uv tool install hatch==1.16.5 && uv tool install rust-just==1.58.0 # Select test dependencies/interpreter: HATCH_ENV=test.py3.13-minimal just test # On Windows, use Git Bash (the same shell used by the test workflow). set shell := ["bash", "-eu", "-o", "pipefail", "-c"] @@ -13,6 +13,13 @@ hatch_env := env("HATCH_ENV", "test.py3.12-optional") gpu_env := env("GPU_HATCH_ENV", "gputest.py3.12") # Pinned so a prek release cannot change what CI lints without a commit here. prek_version := "0.5.3" +# Documentation and changelog tooling resolves from uv.lock, not a hatch environment. +# There is only ever one docs toolchain, so it can be locked and hash-verified, and +# dependabot's uv ecosystem keeps it current. Hatch still owns the test environments, +# which exist per interpreter and per dependency set and cannot live in one lockfile. +docs_run := "uv run --frozen --group docs" +# The hatch docs environment used to set these; they belong with the mkdocs calls now. +mkdocs_env := "DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true" # List available recipes default: @@ -71,39 +78,39 @@ gpu *args: # Build documentation (warnings are errors) docs-build *args: - hatch run docs:mkdocs build --strict "$@" + {{ mkdocs_env }} {{ docs_run }} mkdocs build --strict "$@" # Serve documentation with live reload docs-serve *args: - hatch run docs:mkdocs serve --watch src "$@" + {{ mkdocs_env }} {{ docs_run }} mkdocs serve --watch src "$@" # Check that every public export has API documentation check-doc-exports *args: - hatch run docs:python ci/check_documented_exports.py docs/api "$@" + {{ docs_run }} python ci/check_documented_exports.py docs/api "$@" # Check documentation source conventions lint-docs *args: - hatch run docs:python ci/lint_docs.py "$@" + {{ docs_run }} python ci/lint_docs.py "$@" # Report unlinked types in built documentation check-doc-links *args: - hatch run docs:python ci/check_unlinked_types.py "$@" + {{ docs_run }} python ci/check_unlinked_types.py "$@" # Run source documentation checks followed by a strict build docs-check: check-doc-exports lint-docs docs-build # Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...) lint *args: - uvx prek@{{ prek_version }} run --all-files "$@" + uvx prek@{{ prek_version }} run --show-diff-on-failure --color=always --all-files "$@" # Run hooks with a custom selection, e.g. just hooks run --last-commit hooks +args: uvx prek@{{ prek_version }} "$@" -# Install local pre-commit hooks. prek is installed as a persistent uv tool, not run -# through uvx: the hook shim prek writes into .git/hooks hard-codes the binary path -# it was installed from and falls back to `prek` on PATH, and a uvx archive path -# stops existing at the next `uv cache prune`. +# prek is installed as a persistent uv tool rather than run through uvx: the hook shim +# prek writes into .git/hooks hard-codes the binary path it was installed from and falls +# back to `prek` on PATH, and a uvx archive path stops existing at the next cache prune. +# Install local pre-commit hooks hooks-install: uv tool install prek=={{ prek_version }} prek install @@ -126,15 +133,15 @@ build *args: # Create a changelog fragment (interactive without arguments) changelog *args: - hatch run docs:towncrier create "$@" + {{ docs_run }} towncrier create "$@" # Preview the next release's changelog changelog-draft *args: - hatch run docs:towncrier build --draft --version Unreleased "$@" + {{ docs_run }} towncrier build --draft --version Unreleased "$@" # Build release notes; pass --version and --yes when preparing a release changelog-build *args: - hatch run docs:towncrier build "$@" + {{ docs_run }} towncrier build "$@" # Check changelog filenames (default: changes/; accepts a package changes directory) check-changelogs *args: @@ -143,7 +150,7 @@ check-changelogs *args: # Check recipe formatting of the root Justfile and every package justfile just-check: just --fmt --check - for f in packages/*/justfile; do just --justfile "$f" --fmt --check; done + shopt -s nullglob; for f in packages/*/justfile; do just --justfile "$f" --fmt --check; done # Run a zarr-metadata recipe, or list its recipes with no arguments zarr-metadata *args: diff --git a/changes/4372.misc.md b/changes/4372.misc.md index 7b55499cf1..55a800a3f3 100644 --- a/changes/4372.misc.md +++ b/changes/4372.misc.md @@ -1 +1 @@ -Define development and CI commands in a root Justfile, with Hatch managing Python environments. Extracted from [#4096](https://github.com/zarr-developers/zarr-python/pull/4096). +Define development and CI commands in a root Justfile, with Hatch managing Python environments and documentation tooling resolving from `uv.lock`. The Hatch script tables are removed, so invocations like `hatch env run --env test.py3.12-optional run-coverage` become `just coverage`; see the contributing guide for the full set. Extracted from [#4096](https://github.com/zarr-developers/zarr-python/pull/4096). diff --git a/docs/contributing.md b/docs/contributing.md index 5c3cfbe8b0..45e401e064 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -82,10 +82,14 @@ git remote add upstream git@github.com:zarr-developers/zarr-python.git The root `Justfile` defines development and CI commands. [just](https://just.systems/) runs these commands, while [Hatch](https://hatch.pypa.io/latest/index.html) manages -the Python environments declared in `pyproject.toml`. Install the task tools and uv (several recipes run tooling through it): +the Python environments declared in `pyproject.toml`. Install +[uv](https://docs.astral.sh/uv/getting-started/installation/) first, then the two task +tools. `uv tool install` puts them in their own environments, which a plain `pip install` +cannot do on a Python that marks itself externally managed (Debian, Ubuntu, Homebrew): ```bash -pip install hatch==1.16.5 rust-just==1.58.0 uv +uv tool install hatch==1.16.5 +uv tool install rust-just==1.58.0 # or any option from https://just.systems/man/en/packages.html just # list available commands just envs # list Python environments just setup # create the default test environment @@ -95,7 +99,8 @@ just test Test recipes default to `test.py3.12-optional`. Set `HATCH_ENV` to select a different interpreter or dependency set, just as CI does. `just gpu` reads `GPU_HATCH_ENV` instead, so an exported `HATCH_ENV` cannot silently send GPU tests to an -environment built without the `gpu` feature. On Windows, run these commands in +environment built without the `gpu` feature, and `just doctest` always runs in the +`doctest` environment. On Windows, run these commands in Git Bash. ```bash @@ -153,7 +158,7 @@ Again, any conflicts need to be resolved before submitting a pull request. ### Running the test suite -Zarr includes a suite of unit tests. The simplest way to run the unit tests is to activate your development environment (see [creating a development environment](#creating-a-development-environment) above) and invoke: +Zarr includes a suite of unit tests. The simplest way to run the unit tests is to invoke: ```bash just test @@ -340,7 +345,7 @@ Sometimes, you may want the documentation to build quicker. You can disable code ### Changelog -zarr-python uses [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) to manage release notes. Most pull requests should include at least one news fragment describing the changes. To add a release note, you'll need the GitHub issue or pull request number and the type of your change (`feature`, `bugfix`, `doc`, `removal`, `misc`). With that, run `just changelog` with your development environment, which will prompt you for the issue number, change type, and the news text: +zarr-python uses [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) to manage release notes. Most pull requests should include at least one news fragment describing the changes. To add a release note, you'll need the GitHub issue or pull request number and the type of your change (`feature`, `bugfix`, `doc`, `removal`, `misc`). With that, run `just changelog`, which will prompt you for the issue number, change type, and the news text: ```bash just changelog @@ -464,6 +469,9 @@ The benchmarks are run as part of the continuous integration suite through [cods ## Building distributions and maintaining dependencies +`just just-check` verifies that the root `Justfile` and each package `justfile` are +formatted the way CI expects; `just --fmt` rewrites them in place if it complains. + Run `just build` to produce a source distribution and wheel in `dist/`. Use `just lock-check` to check the dependency lockfile, or `just lock` to update it. `just typecheck` runs the type checker independently of the other lint hooks. diff --git a/packages/zarr-http-server/justfile b/packages/zarr-http-server/justfile index d65b1b53b7..5daac00ebc 100644 --- a/packages/zarr-http-server/justfile +++ b/packages/zarr-http-server/justfile @@ -10,6 +10,9 @@ # newest release: when ruff 0.16 began selecting BLE001 under the root # config's `B` prefix, this job failed on rules the pinned ruff never enforced, # with no code change to blame. Bump alongside the pre-commit rev. +# Quoted arguments must survive delegation from the root Justfile. +set positional-arguments + ruff_version := "0.16.0" # List available recipes @@ -21,7 +24,7 @@ default: # silently skipping. # Run the test suite; extra args are passed to pytest test *args: - uv run --group test --group examples pytest tests {{ args }} + uv run --group test --group examples pytest tests "$@" # Lint the package sources and tests lint: diff --git a/packages/zarr-indexing/justfile b/packages/zarr-indexing/justfile index e32f177ff0..29f6ce885f 100644 --- a/packages/zarr-indexing/justfile +++ b/packages/zarr-indexing/justfile @@ -2,6 +2,9 @@ # directory as the working directory regardless of where `just` is invoked. # List available recipes +# Quoted arguments must survive delegation from the root Justfile. +set positional-arguments + default: @just --list @@ -10,7 +13,7 @@ default: # overlay, using the same test invocation as CI. # Run the test suite; extra args are passed to pytest test *args: - uv run --project ../.. --group test --with-editable . python -m pytest tests src/zarr_indexing {{ args }} + uv run --project ../.. --group test --with-editable . python -m pytest tests src/zarr_indexing "$@" # TensorStore is the oracle for the parity suites, which skip without it. It # ships binary wheels only, so it rides in as a run-time overlay rather than @@ -18,7 +21,7 @@ test *args: # gate the CI job that calls this on the matrix version. # Run the tensorstore parity suites; extra args are passed to pytest test-tensorstore *args: - uv run --project ../.. --group test --with-editable . --with 'tensorstore>=0.1.84' python -m pytest tests/test_ndsel_tensorstore.py tests/test_tensorstore_parity.py {{ args }} + uv run --project ../.. --group test --with-editable . --with 'tensorstore>=0.1.84' python -m pytest tests/test_ndsel_tensorstore.py tests/test_tensorstore_parity.py "$@" # Lint with the same invocation CI uses. Ruff is pinned to the repo-wide # version in the root .pre-commit-config.yaml; bump together. diff --git a/packages/zarr-metadata/justfile b/packages/zarr-metadata/justfile index 0f1861ed7d..843286b89f 100644 --- a/packages/zarr-metadata/justfile +++ b/packages/zarr-metadata/justfile @@ -2,12 +2,15 @@ # directory as the working directory regardless of where `just` is invoked. # List available recipes +# Quoted arguments must survive delegation from the root Justfile. +set positional-arguments + default: @just --list # Run the test suite; extra args are passed to pytest test *args: - uv run --group test pytest tests {{ args }} + uv run --group test pytest tests "$@" # Lint with the same invocation CI uses lint: diff --git a/pyproject.toml b/pyproject.toml index 75802b0a06..a4dafe4d6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -265,14 +265,6 @@ extra-dependencies = [ [tool.hatch.envs.default] installer = "uv" -[tool.hatch.envs.docs] -features = ['remote'] -dependency-groups = ['docs'] - -[tool.hatch.envs.docs.env-vars] -DISABLE_MKDOCS_2_WARNING = "true" -NO_MKDOCS_2_WARNING = "true" - [tool.hatch.envs.doctest] description = "Test environment for validating executable code blocks in documentation" features = ['remote']