From 9bfe855a6cfd6985073fd5e4e5b5be027628eca8 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 14 Jul 2026 13:25:16 +0200 Subject: [PATCH 1/5] fix: byte-order handling for structured dtypes in the bytes codec (#220) * fix: byte-order handling for structured dtypes in the bytes codec The bytes codec neither byte-swapped structured-dtype fields to its configured endian on encode (numpy reports byteorder '|' for void dtypes, so the top-level byteorder comparison never detected a mismatch) nor honored its endian when decoding, silently corrupting any structured data whose field byte order differed from the stored one (e.g. virtual references to external big-endian data). Encode now detects byte-order mismatches by comparing full dtypes via newbyteorder, and decode reinterprets raw bytes in the stored byte order before converting to the data type's declared byte order, so the stored layout (codec state) and the in-memory layout (array data type) are independent. Closes #4141 Assisted-by: ClaudeCode:claude-fable-5 * test: fold structured byte-order cases into existing bytes codec tests Extend test_endian's parametrization with structured dtypes and test_bytes_codec_sync_roundtrip with endian/dtype parametrization plus stored-layout and decoded-dtype assertions, instead of adding parallel test functions for the same properties. Assisted-by: ClaudeCode:claude-fable-5 * refactor: rename stored_dtype to view_dtype in BytesCodec decode The variable is the dtype used to view the raw chunk bytes (byte order from the codec's endian configuration), not a property of the stored data or of the returned buffer, which always carries the array's declared dtype. Assisted-by: ClaudeCode:claude-fable-5 * docs: note that the decode-side byte-order conversion copies the chunk Assisted-by: ClaudeCode:claude-fable-5 --- changes/4141.bugfix.md | 1 + src/zarr/codecs/bytes.py | 33 +++++++++++----- tests/test_codecs/test_bytes.py | 69 ++++++++++++++++++++++++++------- 3 files changed, 80 insertions(+), 23 deletions(-) create mode 100644 changes/4141.bugfix.md diff --git a/changes/4141.bugfix.md b/changes/4141.bugfix.md new file mode 100644 index 0000000000..6a132da3f5 --- /dev/null +++ b/changes/4141.bugfix.md @@ -0,0 +1 @@ +Fix silent byte-order corruption for structured dtypes with the `bytes` codec: multi-byte fields are now byte-swapped to the codec's configured `endian` on write and decoded honoring it on read, so non-native-endian structured data (e.g. big-endian fields, as produced by virtual references to external data) round-trips correctly. diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 240c077627..fae762fd08 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -100,14 +100,28 @@ def _decode_sync( chunk_spec: ArraySpec, ) -> NDBuffer: endian_str = self.endian + dtype = chunk_spec.dtype.to_native_dtype() + # The byte order of the stored data is set by this codec's `endian` + # configuration; the byte order of the decoded array is set by the array's + # data type. The two are independent: the raw bytes are viewed with a dtype + # in the stored byte order, then converted to the declared dtype if needed. if isinstance(chunk_spec.dtype, HasEndianness): - dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] + view_dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] + elif isinstance(chunk_spec.dtype, Struct) and endian_str is not None: + # Per the struct data type spec, all multi-byte fields are stored in the + # byte order configured on this codec. + view_dtype = dtype.newbyteorder(endian_str) else: - dtype = chunk_spec.dtype.to_native_dtype() + view_dtype = dtype as_array_like = chunk_bytes.as_array_like() chunk_array = chunk_spec.prototype.nd_buffer.from_ndarray_like( - as_array_like.view(dtype=dtype) # type: ignore[attr-defined] + as_array_like.view(dtype=view_dtype) # type: ignore[attr-defined] ) + if view_dtype != dtype: + # This byte-swapping conversion copies the chunk. The dtype inequality + # guard keeps the common case, where the stored and declared byte orders + # already match, on the zero-copy view path above. + chunk_array = chunk_array.astype(dtype) # ensure correct chunk shape if chunk_array.shape != chunk_spec.shape: @@ -129,13 +143,14 @@ def _encode_sync( chunk_spec: ArraySpec, ) -> Buffer | None: assert isinstance(chunk_array, NDBuffer) - if ( - chunk_array.dtype.itemsize > 1 - and self.endian is not None - and self.endian != chunk_array.byteorder - ): + if chunk_array.dtype.itemsize > 1 and self.endian is not None: + # Compare full dtypes rather than the top-level byteorder: numpy reports + # byteorder '|' for structured dtypes even when their fields are + # byte-order-sensitive, so newbyteorder is the only reliable way to + # detect (and normalize) a byte-order mismatch. new_dtype = chunk_array.dtype.newbyteorder(self.endian) - chunk_array = chunk_array.astype(new_dtype) + if new_dtype != chunk_array.dtype: + chunk_array = chunk_array.astype(new_dtype) nd_array = chunk_array.as_ndarray_like() # Flatten the nd-array (only copy if needed) and reinterpret as bytes diff --git a/tests/test_codecs/test_bytes.py b/tests/test_codecs/test_bytes.py index 03dd0b40c6..ead778f526 100644 --- a/tests/test_codecs/test_bytes.py +++ b/tests/test_codecs/test_bytes.py @@ -33,29 +33,50 @@ @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) -@pytest.mark.parametrize("input_dtype", [">u2", "u2", + "f4"), ("mask", ">i4")], + [("flux", "u2", " None: """ The `bytes` codec stores multi-byte data in the byte order configured on the codec, regardless of the input array's byte order, and reads it back to the - original values. The input-dtype/store-endian cross-product exercises the - encode-side byteswap (input byte order != store byte order) and the no-op - case alike. Compression is disabled so the stored chunk is the codec's raw - output and its byte layout can be asserted directly. + original values. For structured dtypes this applies to every multi-byte + field, per the `struct` data type spec; the struct cases guard against the + endianness bugs from + https://github.com/zarr-developers/zarr-python/issues/4141, where the + encode path never byte-swapped struct fields (numpy reports byteorder '|' + for void dtypes) and the decode path ignored the codec's endian entirely. + The input-dtype/store-endian cross-product exercises the encode-side + byteswap (input byte order != store byte order) and the no-op case alike. + Compression is disabled so the stored chunk is the codec's raw output and + its byte layout can be asserted directly. """ - data = np.arange(0, 256, dtype=input_dtype).reshape((16, 16)) + dtype = np.dtype(input_dtype) + if dtype.fields is None: + data = np.arange(0, 256, dtype=dtype).reshape((16, 16)) + else: + data = np.zeros((16, 16), dtype=dtype) + data["flux"] = np.arange(0, 256).reshape((16, 16)) + data["mask"] = np.arange(256, 512).reshape((16, 16)) path = "endian" spath = StorePath(store, path) a = await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(16, 16), - dtype="uint16", + dtype=dtype, fill_value=0, compressors=None, serializer=BytesCodec(endian=store_endian), @@ -66,8 +87,7 @@ async def test_endian( # The stored chunk is laid out in the byte order configured on the codec. stored = await store.get(f"{path}/c/0/0", prototype=default_buffer_prototype()) assert stored is not None - expected_dtype = ">u2" if store_endian == "big" else " None: assert isinstance(BytesCodec(), SupportsSyncCodec) -def test_bytes_codec_sync_roundtrip() -> None: - codec = BytesCodec() - arr = np.arange(100, dtype="float64") +@pytest.mark.parametrize("endian", ENDIAN) +@pytest.mark.parametrize( + "native_dtype", + [np.dtype("float64"), np.dtype(">u2"), np.dtype([("a", ">f4"), ("b", " None: + """ + The synchronous encode/decode path round-trips data, and the two byte + orders involved are independent: the codec's `endian` configuration governs + only the stored byte layout (every multi-byte value, including struct + fields, is laid out in the codec's byte order regardless of the input + array's byte order), while the decoded buffer's byte order is governed by + the array's data type regardless of the codec's. The mixed-endian struct + case pins that per-field byte order of the in-memory dtype survives a + roundtrip through a single stored byte order. + """ + if native_dtype.fields is None: + arr = np.arange(100, dtype=native_dtype) + else: + arr = np.array([(1.5, 2), (3.5, 4), (5.5, 6), (7.5, 8)], dtype=native_dtype) zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, @@ -91,11 +129,14 @@ def test_bytes_codec_sync_roundtrip() -> None: ) nd_buf: NDBuffer = default_buffer_prototype().nd_buffer.from_numpy_array(arr) - codec = codec.evolve_from_array_spec(spec) + codec = BytesCodec(endian=endian).evolve_from_array_spec(spec) encoded = codec._encode_sync(nd_buf, spec) assert encoded is not None + assert encoded.to_bytes() == arr.astype(native_dtype.newbyteorder(endian)).tobytes() + decoded = codec._decode_sync(encoded, spec) + assert decoded.dtype == zdtype.to_native_dtype() np.testing.assert_array_equal(arr, decoded.as_numpy_array()) From 65031539ab125faaa13006bec2e516790d4ea7b4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 17 Jul 2026 18:24:05 +0200 Subject: [PATCH 2/5] fix(store): FsspecStore.close() no longer closes the filesystem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FsspecStore.close() closed the underlying filesystem's session, on the premise that a store built by from_url "owns" the filesystem it created. That premise does not hold: fsspec caches and shares filesystem instances across callers (its instance cache keys on storage options, not path), and users can hand one filesystem to many stores directly. Closing one store therefore killed the session that sibling stores were still using, and left the dead filesystem in fsspec's cache for later callers. Determining whether a filesystem is actually shared requires reaching into fsspec's private instance cache (_cache, _fs_token, cachable) and walking wrapper chains for caching/proxy filesystems — an implementation detail that leaks upward and that we would have to keep in sync with fsspec forever, getting it subtly wrong in between. The wrapper case alone (simplecache::/dir://) already slipped through a cache-membership check. The filesystem's lifecycle is simply not the store's to manage. This removes the ownership model added in the unreleased gh-4003: no _owns_fs, no _close_fs, no ownership transfer in with_read_only, and close() just marks the store not-open. The only thing given up is suppressing an "Unclosed client session" ResourceWarning, which was true anyway — the session belongs to a cached filesystem that outlives the store. Since gh-4003 never shipped (latest release is v3.2.1), its changelog fragment is removed rather than superseded. Assisted-by: ClaudeCode:claude-opus-4.8 --- changes/226.bugfix.md | 10 +++ changes/4003.bugfix.md | 19 ----- src/zarr/storage/_fsspec.py | 70 +++------------- tests/test_store/test_fsspec.py | 138 ++++++++------------------------ 4 files changed, 57 insertions(+), 180 deletions(-) create mode 100644 changes/226.bugfix.md delete mode 100644 changes/4003.bugfix.md diff --git a/changes/226.bugfix.md b/changes/226.bugfix.md new file mode 100644 index 0000000000..5913ab43c5 --- /dev/null +++ b/changes/226.bugfix.md @@ -0,0 +1,10 @@ +`FsspecStore.close()` no longer closes the underlying fsspec filesystem or its +network session. fsspec caches and shares filesystem instances across callers, +so a store cannot know whether it is the only user, and closing a shared +session broke sibling stores built from the same host — and left the closed +filesystem in fsspec's cache for later callers. The filesystem's lifecycle now +stays with whoever created it; use fsspec's own tools (for example +`AbstractFileSystem.clear_instance_cache()`) to release it. + +This removes the store-owns-the-filesystem behavior added in the unreleased +gh-4003, which is why there is no net change for released versions. diff --git a/changes/4003.bugfix.md b/changes/4003.bugfix.md deleted file mode 100644 index 36327b55df..0000000000 --- a/changes/4003.bugfix.md +++ /dev/null @@ -1,19 +0,0 @@ -`FsspecStore.from_url()` and `from_mapper()` now close the async filesystem -they create when `store.close()` is called. Previously the underlying aiohttp -`ClientSession` was left open until garbage collection, producing -`"Unclosed client session"` `ResourceWarning`s from aiohttp. - -The fix introduces `FsspecStore._owns_fs`, a boolean that is ``True`` only when -`FsspecStore` itself created the filesystem (via `from_url` or `from_mapper` -when a sync→async conversion was performed). When `_owns_fs` is ``True``, -`store.close()` calls the new `_close_fs()` helper, which invokes -`fs.set_session()` and closes the returned client. Callers who supply their own -filesystem instance to `FsspecStore()` directly remain responsible for its -lifecycle; `_owns_fs` is ``False`` for those stores. - -**Scope note**: This fix closes the S3 client session that is active at the time -`store.close()` is called. Some S3-backed filesystem implementations (e.g. -s3fs with ``cache_regions=True``) may internally refresh and replace their -client during I/O operations, abandoning prior sessions before ``store.close()`` -is invoked. Those intermediate sessions are outside the scope of this fix and -are an issue in the upstream filesystem library. diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 617980ac19..37d134dd95 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -3,7 +3,6 @@ import json import warnings from contextlib import suppress -from logging import getLogger from typing import TYPE_CHECKING, Any from packaging.version import parse as parse_version @@ -19,8 +18,6 @@ from zarr.errors import ZarrUserWarning from zarr.storage._utils import _dereference_path -logger = getLogger(__name__) - if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable @@ -38,26 +35,6 @@ ) -async def _close_fs(fs: AsyncFileSystem) -> None: - """ - Best-effort async close of an fsspec async filesystem owned by FsspecStore. - - For filesystems that expose `set_session()` (e.g. s3fs) the underlying - aiohttp `ClientSession` is closed explicitly, which prevents - "Unclosed client session" `ResourceWarning`s from aiohttp. For all - other filesystem types the call is a no-op (not every implementation - manages an HTTP session directly). - - Note that `set_session()` lazily creates a session if none exists yet, so - closing a store that never performed any I/O may instantiate a session - purely to close it. This is accepted best-effort behavior; fsspec does not - expose a stable, cross-implementation way to test for an existing session. - """ - if hasattr(fs, "set_session"): - session = await fs.set_session() - await session.close() - - def _make_async(fs: AbstractFileSystem) -> AsyncFileSystem: """Convert a sync FSSpec filesystem to an async FFSpec filesystem @@ -126,6 +103,15 @@ class FsspecStore(Store): ZarrUserWarning If the file system (fs) was not created with `asynchronous=True`. + Notes + ----- + Closing the store does not close the underlying filesystem or its network + session. fsspec caches and shares filesystem instances across callers, so + the store cannot know whether it is the only user, and closing a shared + session would break other stores. The filesystem's lifecycle belongs to + whoever created it; use fsspec's own tools (e.g. `clear_instance_cache`) + to release it. + See Also -------- FsspecStore.from_upath @@ -152,9 +138,6 @@ def __init__( self.fs = fs self.path = path self.allowed_exceptions = allowed_exceptions - # True only when this store created fs itself (from_url / from_mapper with new instance). - # Callers who supply their own fs remain responsible for its lifecycle. - self._owns_fs: bool = False if not self.fs.async_impl: raise TypeError("Filesystem needs to support async operations.") @@ -220,17 +203,13 @@ def from_mapper( ------- FsspecStore """ - original_fs = fs_map.fs - fs = _make_async(original_fs) - store = cls( + fs = _make_async(fs_map.fs) + return cls( fs=fs, path=fs_map.root, read_only=read_only, allowed_exceptions=allowed_exceptions, ) - # _make_async returns a new instance when converting sync→async; own it. - store._owns_fs = fs is not original_fs - return store @classmethod def from_url( @@ -272,39 +251,16 @@ def from_url( if not fs.async_impl: fs = _make_async(fs) - store = cls(fs=fs, path=path, read_only=read_only, allowed_exceptions=allowed_exceptions) - store._owns_fs = True - return store + return cls(fs=fs, path=path, read_only=read_only, allowed_exceptions=allowed_exceptions) def with_read_only(self, read_only: bool = False) -> FsspecStore: # docstring inherited - new_store = type(self)( + return type(self)( fs=self.fs, path=self.path, allowed_exceptions=self.allowed_exceptions, read_only=read_only, ) - # The derived store shares the same fs. Transfer ownership so the - # surviving store closes it, and clear ours to avoid a double-close. - # Otherwise the common `from_url(...).with_read_only()` chain would - # drop the only owner (the unreferenced source) and leak the session. - new_store._owns_fs = self._owns_fs - self._owns_fs = False - return new_store - - def close(self) -> None: - # docstring inherited - if self._owns_fs: - from zarr.core.sync import sync as zarr_sync - - # Best-effort: a failure to release the session must not block close(), - # but log it so a genuine regression in the close path stays observable - # rather than silently reverting to the leaking behavior. - try: - zarr_sync(_close_fs(self.fs)) - except Exception: - logger.debug("Failed to close owned filesystem %r", self.fs, exc_info=True) - super().close() async def clear(self) -> None: # docstring inherited diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index 898d49ec08..99c811e76b 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -276,75 +276,20 @@ async def test_delete_dir_unsupported_deletes(self, store: FsspecStore) -> None: ): await store.delete_dir("test_prefix") - # ── Filesystem lifecycle (ownership) ────────────────────────────────────── + # ── Filesystem lifecycle ────────────────────────────────────────────────── - def test_from_url_owns_filesystem(self, endpoint_url: str) -> None: - """FsspecStore.from_url() creates the async fs; it must own it.""" + async def test_close_marks_store_closed(self, endpoint_url: str) -> None: + """close() must succeed and mark the store not-open.""" store = FsspecStore.from_url( f"s3://{test_bucket_name}/lifecycle/", storage_options={"endpoint_url": endpoint_url, "anon": False}, ) - assert store._owns_fs - store.close() - - async def test_from_url_close_releases_store(self, endpoint_url: str) -> None: - """ - close() on a from_url() store must succeed without error and mark the - store as closed. For the owned filesystem, _close_fs() is invoked to - release the underlying S3 client / aiohttp connection pool. - """ - store = FsspecStore.from_url( - f"s3://{test_bucket_name}/lifecycle/", - storage_options={"endpoint_url": endpoint_url, "anon": False}, - ) - # Materialise the S3 client and connection pool. await store.set("probe", cpu.Buffer.from_bytes(b"x")) store.close() assert not store._is_open - def test_direct_construction_does_not_own_filesystem(self, endpoint_url: str) -> None: - """Direct FsspecStore() must not claim ownership — the caller owns the fs.""" - try: - from fsspec import url_to_fs - except ImportError: - from fsspec.core import url_to_fs - fs, path = url_to_fs( - f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=True - ) - store = FsspecStore(fs=fs, path=path) - assert not store._owns_fs - - @pytest.mark.skipif( - parse_version(fsspec.__version__) < parse_version("2024.03.01"), - reason="Prior bug in from_upath", - ) - def test_from_upath_does_not_own_filesystem(self, endpoint_url: str) -> None: - """from_upath() uses the UPath's existing fs; the store must not own it.""" - upath = pytest.importorskip("upath") - path = upath.UPath( - f"s3://{test_bucket_name}/foo/bar/", - endpoint_url=endpoint_url, - anon=False, - asynchronous=True, - ) - store = FsspecStore.from_upath(path) - assert not store._owns_fs - - def test_from_mapper_does_not_own_already_async_filesystem(self, endpoint_url: str) -> None: - """from_mapper() with an already-async fs must not claim ownership.""" - s3_filesystem = s3fs.S3FileSystem( - asynchronous=True, - endpoint_url=endpoint_url, - anon=False, - skip_instance_cache=True, - ) - mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/") - store = FsspecStore.from_mapper(mapper) - # _make_async returns the same instance for an already-async fs. - assert not store._owns_fs - def array_roundtrip(store: FsspecStore) -> None: """ @@ -574,47 +519,47 @@ def test_open_s3map_raises(endpoint_url: str) -> None: zarr.open(store=mapper, storage_options={"anon": True}, mode="w", shape=(3, 3)) -async def test_close_fs_closes_s3_client() -> None: - """ - _close_fs() must call set_session() and then close() on the returned - S3 client. This is verified with mocks to avoid a real S3 connection. - """ - from unittest.mock import AsyncMock +async def test_close_does_not_close_filesystem_session() -> None: + """close() must not touch the filesystem's session. - from zarr.storage._fsspec import _close_fs + fsspec caches and shares filesystem instances across callers, so the + session is not the store's to close. HTTP is used because its aiohttp + session is observably closed for good; s3fs transparently reconnects, which + would hide a regression. No request is issued — set_session() only + constructs the session. + """ + pytest.importorskip("aiohttp") + store = FsspecStore.from_url("http://example.com/a") + session = await store.fs.set_session() - mock_client = AsyncMock() - mock_fs = AsyncMock() - mock_fs.set_session = AsyncMock(return_value=mock_client) + store.close() - await _close_fs(mock_fs) + assert not session.closed - mock_fs.set_session.assert_called_once() - mock_client.close.assert_called_once() +async def test_close_does_not_break_a_sibling_store() -> None: + """Closing one store must not close a session another store is using. -async def test_close_fs_no_op_for_fs_without_set_session() -> None: - """_close_fs() must be a no-op for filesystems that don't expose set_session().""" - from unittest.mock import AsyncMock + Two stores from different URLs on one host are handed the same cached + filesystem; a store that closed it on close() would take the sibling's + session down too. This is the regression guard for that bug. + """ + pytest.importorskip("aiohttp") + s1 = FsspecStore.from_url("http://example.com/a") + s2 = FsspecStore.from_url("http://example.com/b") + session = await s2.fs.set_session() - from zarr.storage._fsspec import _close_fs + s1.close() - mock_fs = AsyncMock(spec=[]) # empty spec — no set_session attribute - await _close_fs(mock_fs) # must not raise + assert not session.closed @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) -def test_from_mapper_owns_wrapped_sync_filesystem(tmp_path: pathlib.Path) -> None: - """ - from_mapper() with a sync fs must wrap it in AsyncFileSystemWrapper and - claim ownership so that close() cleans it up. - - The local filesystem is synchronous; _make_async() produces a new - AsyncFileSystemWrapper instance — a different object from the original fs. - """ +def test_from_mapper_wraps_sync_filesystem(tmp_path: pathlib.Path) -> None: + """from_mapper() with a sync fs wraps it in an AsyncFileSystemWrapper.""" import fsspec as _fsspec from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper @@ -622,32 +567,17 @@ def test_from_mapper_owns_wrapped_sync_filesystem(tmp_path: pathlib.Path) -> Non mapper = fs.get_mapper(str(tmp_path)) store = FsspecStore.from_mapper(mapper) assert isinstance(store.fs, AsyncFileSystemWrapper) - assert store._owns_fs -@pytest.mark.skipif( - parse_version(fsspec.__version__) < parse_version("2024.12.0"), - reason="No AsyncFileSystemWrapper", -) -def test_with_read_only_transfers_filesystem_ownership(tmp_path: pathlib.Path) -> None: - """ - with_read_only() must transfer fs ownership to the derived store and clear - it on the source, so the surviving store closes the shared fs exactly once. - - In the common ``from_url(...).with_read_only()`` chain the source store is - immediately unreferenced; if ownership were not transferred, the only owner - would be garbage-collected without close() and the session would leak. - """ +def test_with_read_only_shares_filesystem(tmp_path: pathlib.Path) -> None: + """with_read_only() returns a store sharing the source's filesystem.""" source = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": False}) - assert source._owns_fs derived = source.with_read_only(read_only=True) - # Ownership moved to the survivor; the source no longer owns it (no double-close). - assert derived._owns_fs - assert not source._owns_fs - # The derived store shares the same underlying fs. assert derived.fs is source.fs + assert derived.read_only + assert not source.read_only @pytest.mark.parametrize("asynchronous", [True, False]) From 9ce5392694ef6111c987505d118adde4a52b956e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 20 Jul 2026 08:58:56 +0200 Subject: [PATCH 3/5] test: skip with_read_only fs test when AsyncFileSystemWrapper is absent test_with_read_only_shares_filesystem replaced an ownership test that carried a guard for fsspec < 2024.12.0, and the guard was dropped in the rewrite. The test still opens a file:// URL, which needs AsyncFileSystemWrapper, so it failed the min_deps job. Assisted-by: ClaudeCode:claude-opus-4.8 --- tests/test_store/test_fsspec.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index 99c811e76b..515e1526b6 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -569,6 +569,10 @@ def test_from_mapper_wraps_sync_filesystem(tmp_path: pathlib.Path) -> None: assert isinstance(store.fs, AsyncFileSystemWrapper) +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) def test_with_read_only_shares_filesystem(tmp_path: pathlib.Path) -> None: """with_read_only() returns a store sharing the source's filesystem.""" source = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": False}) From 27eac5b61f16b721e52e6c0b4b2efe65d2bad01e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 20 Jul 2026 09:00:39 +0200 Subject: [PATCH 4/5] docs: correct changelog claim about gh-4003 release status The fragment said gh-4003 was unreleased with no net change for released versions. Its text is already in the staged 3.3.0 release notes, so the revert is a real behavior change for anyone relying on close() releasing the session. Assisted-by: ClaudeCode:claude-opus-4.8 --- changes/226.bugfix.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/changes/226.bugfix.md b/changes/226.bugfix.md index 5913ab43c5..63f92d7b35 100644 --- a/changes/226.bugfix.md +++ b/changes/226.bugfix.md @@ -6,5 +6,6 @@ filesystem in fsspec's cache for later callers. The filesystem's lifecycle now stays with whoever created it; use fsspec's own tools (for example `AbstractFileSystem.clear_instance_cache()`) to release it. -This removes the store-owns-the-filesystem behavior added in the unreleased -gh-4003, which is why there is no net change for released versions. +This removes the store-owns-the-filesystem behavior added in gh-4003. Code that +relied on `store.close()` releasing the filesystem's session must now close the +filesystem itself. From 6160c9eb1976a83cd1f678adb3a8e82cf7886d1d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 20 Jul 2026 15:54:50 +0200 Subject: [PATCH 5/5] docs: remove changelog entry for unreleased versions --- changes/226.bugfix.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 changes/226.bugfix.md diff --git a/changes/226.bugfix.md b/changes/226.bugfix.md deleted file mode 100644 index 63f92d7b35..0000000000 --- a/changes/226.bugfix.md +++ /dev/null @@ -1,11 +0,0 @@ -`FsspecStore.close()` no longer closes the underlying fsspec filesystem or its -network session. fsspec caches and shares filesystem instances across callers, -so a store cannot know whether it is the only user, and closing a shared -session broke sibling stores built from the same host — and left the closed -filesystem in fsspec's cache for later callers. The filesystem's lifecycle now -stays with whoever created it; use fsspec's own tools (for example -`AbstractFileSystem.clear_instance_cache()`) to release it. - -This removes the store-owns-the-filesystem behavior added in gh-4003. Code that -relied on `store.close()` releasing the filesystem's session must now close the -filesystem itself.