Support zarr v3

.github/scripts/generate_zarr_v2_fixtures.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""
+Generate zarr v2 fixtures for backward compatibility tests.
+
+Run this script with an old spikeinterface version and zarr<3, e.g.:
+    pip install "spikeinterface==0.104.0" "zarr<3"
+    python generate_zarr_v2_fixtures.py --output /tmp/zarr_v2_fixtures
+
+The script saves:
+  - recording.zarr  : a small ZarrRecordingExtractor
+  - sorting.zarr    : a small ZarrSortingExtractor
+  - expected_values.json : key values used to verify correct loading
+"""
+import argparse
+import shutil
+import json
+from pathlib import Path
+
+import numpy as np
+import zarr
+
+import spikeinterface as si
+
+
+def main(output_dir: Path) -> None:
+    print(f"spikeinterface version : {si.__version__}")
+    print(f"zarr version           : {zarr.__version__}")
+
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    recording, sorting = si.generate_ground_truth_recording(durations=[10, 5],num_channels=32, num_units=10, seed=0)
+    # save to binary to make them JSON serializable for later expected values extraction
+    recording = recording.save(folder=output_dir / "recording_binary", overwrite=True)
+    sorting = sorting.save(folder=output_dir / "sorting_binary", overwrite=True)
+    # --- save recording ---
+    recording_path = output_dir / "recording.zarr"
+    recording_zarr = recording.save(format="zarr", folder=recording_path, overwrite=True)
+    print(f"Saved recording  -> {recording_path}")
+
+    # --- save sorting ---
+    sorting_path = output_dir / "sorting.zarr"
+    sorting_zarr = sorting.save(format="zarr", folder=sorting_path, overwrite=True)
+    print(f"Saved sorting    -> {sorting_path}")
+
+    # --- save SortingAnalyzer ---
+    # Reload the recording from zarr so it is a serializable ZarrRecordingExtractor,
+    # which the analyzer can store as provenance.
+    analyzer_path = output_dir / "analyzer.zarr"
+    if analyzer_path.is_dir():
+        shutil.rmtree(analyzer_path)
+    analyzer = si.create_sorting_analyzer(
+        sorting_zarr, recording_zarr, format="zarr", folder=analyzer_path, overwrite=True
+    )
+    analyzer.compute(["random_spikes", "templates"])
+    print(f"Saved analyzer   -> {analyzer_path}")
+
+    # Reload to verify templates are accessible before writing expected values
+    templates_array = analyzer.get_extension("templates").get_data()
+
+    # --- capture expected values for later assertion ---
+    expected = {
+        "spikeinterface_version": si.__version__,
+        "zarr_version": zarr.__version__,
+        "recording": {
+            "num_channels": int(recording.get_num_channels()),
+            "num_segments": int(recording.get_num_segments()),
+            "sampling_frequency": float(recording.get_sampling_frequency()),
+            "num_samples_per_segment": [int(recording.get_num_samples(seg)) for seg in range(recording.get_num_segments())],
+            "channel_ids": recording.get_channel_ids().tolist(),
+            "dtype": str(recording.get_dtype()),
+            # first 10 frames of segment 0 for all channels
+            "traces_seg0_first10": recording.get_traces(start_frame=0, end_frame=10, segment_index=0).tolist(),
+        },
+        "sorting": {
+            "num_segments": int(sorting.get_num_segments()),
+            "sampling_frequency": float(sorting.get_sampling_frequency()),
+            "unit_ids": sorting.get_unit_ids().tolist(),
+            "spike_trains_seg0": {
+                str(uid): sorting.get_unit_spike_train(unit_id=uid, segment_index=0).tolist()
+                for uid in sorting.unit_ids
+            },
+        },
+        "analyzer": {
+            "num_units": int(analyzer.get_num_units()),
+            "num_channels": int(analyzer.get_num_channels()),
+            "templates_shape": list(templates_array.shape),
+        },
+    }
+
+    expected_path = output_dir / "expected_values.json"
+    with open(expected_path, "w") as f:
+        json.dump(expected, f, indent=2)
+    print(f"Saved expected   -> {expected_path}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Generate zarr v2 fixtures for backward compatibility tests")
+    parser.add_argument("--output", type=Path, required=True, help="Directory to write fixtures into")
+    args = parser.parse_args()
+    main(args.output)

.github/workflows/all-tests.yml

@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.13"]  # Lower and higher versions we support
+        python-version: ["3.11", "3.13"]  # Lower and higher versions we support
         os: [macos-latest, windows-latest, ubuntu-latest]
     steps:
       - uses: actions/checkout@v6

.github/workflows/deepinterpolation.yml

@@ -1,10 +1,8 @@
 name: Testing deepinterpolation
 
+# Manual only — deepinterpolation requires Python 3.10, incompatible with 3.11+ required by Zarr 3.0.0+
 on:
-  pull_request:
-    types: [synchronize, opened, reopened]
-    branches:
-      - main
+  workflow_dispatch:
 
 concurrency:  # Cancel previous workflows on the same pull request
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/test_containers_docker.yml

@@ -15,10 +15,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           enable-cache: true
       - name: Python version
         run:

.github/workflows/test_containers_singularity.yml

@@ -16,10 +16,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           enable-cache: true
       - uses: eWaterCycle/setup-singularity@v7
         with:

.github/workflows/test_zarr_compat.yml

@@ -0,0 +1,51 @@
+name: Test zarr backwards compatibility
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [synchronize, opened, reopened]
+    branches:
+      - main
+    paths:
+      - "src/spikeinterface/core/zarrextractors.py"
+      - "src/spikeinterface/core/zarrrecordingextractor.py"
+      - "src/spikeinterface/core/tests/test_zarr_backwards_compat.py"
+      - ".github/workflows/test_zarr_compat.yml"
+      - ".github/scripts/generate_zarr_v2_fixtures.py"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-zarr-compat:
+    name: zarr v2 -> v3 backwards compatibility
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+
+      - name: Install SI 0.104.0 with zarr v2
+        run: uv pip install --system "spikeinterface==0.104.0" "zarr<3"
+
+      - name: Generate zarr v2 fixtures
+        run: python .github/scripts/generate_zarr_v2_fixtures.py --output /tmp/zarr_v2_fixtures
+
+      - name: Install current SI with zarr v3
+        run: uv pip install --system -e . --group test-core
+
+      - name: Check zarr version is v3
+        run: python -c "import zarr; v = zarr.__version__; print(f'zarr {v}'); assert int(v.split('.')[0]) >= 3"
+
+      - name: Run backward compatibility tests
+        env:
+          ZARR_V2_FIXTURES_PATH: /tmp/zarr_v2_fixtures
+        run: pytest src/spikeinterface/core/tests/test_zarr_backwards_compat.py -v

pyproject.toml

@@ -7,7 +7,7 @@ authors = [
 ]
 description = "Python toolkit for analysis, visualization, and comparison of spike sorting output"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 classifiers = [
     "Programming Language :: Python :: 3 :: Only",
     "License :: OSI Approved :: MIT License",
@@ -24,15 +24,16 @@ dependencies = [
     "numpy>=2.0.0;python_version>='3.13'",
     "threadpoolctl>=3.0.0",
     "tqdm",
-    "zarr>=2.18,<3",
+    "zarr>=3,<4",
     # for release we need pypi, so this needs to be commented
     "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
-    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    # for testing we need probeinterface Zarr v3 branch, so this needs to be commented
+    #"probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    "probeinterface @ git+https://github.com/alejoe91/probeinterface.git@zarrv3",
     # "neo>=0.14.4",
     # "probeinterface>=0.3.2",
     "packaging",
     "pydantic",
-    "numcodecs<0.16.0", # For supporting zarr < 3
 ]
 
 [build-system]
@@ -67,7 +68,9 @@ changelog = "https://spikeinterface.readthedocs.io/en/latest/whatisnew.html"
 extractors = [
     "MEArec>=1.8",
     "pynwb>=2.6.0",
-    "hdmf-zarr>=0.11.0",
+    # FOR TESTING
+    "hdmf-zarr @ git+https://github.com/hdmf-dev/hdmf-zarr.git@zarr-v3-migration",
+    # "hdmf-zarr>=0.11.0",
     "pyedflib>=0.1.30",
     "sonpy;python_version<'3.10'",
     "lxml", # lxml for neuroscope
@@ -84,9 +87,11 @@ streaming_extractors = [
     "fsspec",
     "aiohttp",
     "requests",
-    "hdmf-zarr>=0.11.0",
+    # FOR TESTING
+    "hdmf-zarr @ git+https://github.com/hdmf-dev/hdmf-zarr.git@zarr-v3-migration",
+    # "hdmf-zarr>=0.11.0",
     "remfile",
-    "s3fs"
+    "s3fs>=2025.7.0"
 ]
 
 
@@ -221,7 +226,7 @@ docs = [
     "MEArec", # Use as an example
     "pandas<3", # in the modules gallery comparison tutorial
     "hdbscan>=0.8.33",   # For sorters spykingcircus2 + tridesclous
-    "numba", # For many postprocessing functions
+    "numba>=0.59", # For many postprocessing functions
     "networkx",
     "seaborn",
     "skops", # For automated curation

readthedocs.yml

@@ -4,7 +4,7 @@ version: 2
 build:
   os: "ubuntu-24.04"
   tools:
-    python: "3.10"
+    python: "3.13"
   commands:
     - asdf plugin add uv
     - asdf install uv latest

src/spikeinterface/core/base.py

@@ -880,6 +880,8 @@ def save(self, **kwargs) -> "BaseExtractor":
             * dump_ext: "json" or "pkl", default "json" (if format is "folder")
             * verbose: if True output is verbose
             * **save_kwargs: additional kwargs format-dependent and job kwargs for recording
+              (check `save_to_memory()`, `save_to_folder()`, `save_to_zarr()` for more details on format-dependent
+              kwargs)
             {}
 
         Returns
@@ -899,13 +901,27 @@ def save(self, **kwargs) -> "BaseExtractor":
     save.__doc__ = save.__doc__.format(_shared_job_kwargs_doc)
 
     def save_to_memory(self, sharedmem=True, **save_kwargs) -> "BaseExtractor":
+        """
+        Save the object to memory.
+
+        Parameters
+        ----------
+        sharedmem : bool, default: True
+            If True, the object is saved to shared memory, allowing it to be accessed by multiple processes without
+            copying. If False, the object is saved to regular memory, which may involve copying when accessed by
+            multiple processes.
+
+        Returns
+        -------
+        BaseExtractor
+            A saved copy of the extractor in memory.
+        """
         save_kwargs.pop("format", None)
 
         cached = self._save(format="memory", sharedmem=sharedmem, **save_kwargs)
         self.copy_metadata(cached)
         return cached
 
-    # TODO rename to saveto_binary_folder
     def save_to_folder(
         self,
         name: str | None = None,
@@ -951,8 +967,7 @@ def save_to_folder(
             If True, an existing folder at the specified path will be deleted before saving.
         verbose : bool, default: True
             If True, print information about the cache folder being used.
-        **save_kwargs
-            Additional keyword arguments to be passed to the underlying save method.
+        {}
 
         Returns
         -------
@@ -1017,7 +1032,6 @@ def save_to_zarr(
         folder=None,
         overwrite=False,
         storage_options=None,
-        channel_chunk_size=None,
         verbose=True,
         **save_kwargs,
     ):
@@ -1037,26 +1051,9 @@ def save_to_zarr(
         storage_options: dict or None, default: None
             Storage options for zarr `store`. E.g., if "s3://" or "gcs://" they can provide authentication methods, etc.
             For cloud storage locations, this should not be None (in case of default values, use an empty dict)
-        channel_chunk_size: int or None, default: None
-            Channels per chunk (only for BaseRecording)
-        compressor: numcodecs.Codec or None, default: None
-            Global compressor. If None, Blosc-zstd, level 5, with bit shuffle is used
-        filters: list[numcodecs.Codec] or None, default: None
-            Global filters for zarr (global)
-        compressor_by_dataset: dict or None, default: None
-            Optional compressor per dataset:
-                - traces
-                - times
-            If None, the global compressor is used
-        filters_by_dataset: dict or None, default: None
-            Optional filters per dataset:
-                - traces
-                - times
-            If None, the global filters are used
         verbose: bool, default: True
             If True, the output is verbose
-        auto_cast_uint: bool, default: True
-            If True, unsigned integers are cast to signed integers to avoid issues with zarr (only for BaseRecording)
+        {}
 
         Returns
         -------
@@ -1092,7 +1089,6 @@ def save_to_zarr(
             assert not zarr_path.exists(), f"Path {zarr_path} already exists, choose another name"
         save_kwargs["zarr_path"] = zarr_path
         save_kwargs["storage_options"] = storage_options
-        save_kwargs["channel_chunk_size"] = channel_chunk_size
         cached = self._save(format="zarr", verbose=verbose, **save_kwargs)
         cached = read_zarr(zarr_path)