From 7e524575c618e8ca3e51d7182e956fbeda54b7be Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Fri, 5 Jun 2026 16:17:45 -0500 Subject: [PATCH 1/3] XRootD dedicated support --- dev/fsspec_inspector/generate_flavours.py | 2 +- pyproject.toml | 1 + upath/_chain.py | 7 +++- upath/_flavour_sources.py | 4 +- upath/implementations/xrootd.py | 46 ++++++++++++++++++++++ upath/registry.py | 4 ++ upath/tests/conftest.py | 45 +++++++++++++++++++++ upath/tests/implementations/test_xrootd.py | 20 ++++++++++ upath/types/storage_options.py | 20 ++++++++++ 9 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 upath/implementations/xrootd.py create mode 100644 upath/tests/implementations/test_xrootd.py diff --git a/dev/fsspec_inspector/generate_flavours.py b/dev/fsspec_inspector/generate_flavours.py index 2d58e1c3..3eb20fb2 100644 --- a/dev/fsspec_inspector/generate_flavours.py +++ b/dev/fsspec_inspector/generate_flavours.py @@ -179,7 +179,7 @@ def _fix_oss_file_system(x: str) -> str: def _fix_xrootd_file_system(x: str) -> str: x = re.sub( r"return client[.]URL\(path\)[.]path_with_params", - "x = urlsplit(path); return (x.path + f'?{x.query}' if x.query else '')", + "x = urlsplit(path); return (x.path + (f'?{x.query}' if x.query else '')).removeprefix('/')", # noqa: E501 x, ) x = re.sub(r"client[.]URL\(u\)", "urlsplit(u)", x) diff --git a/pyproject.toml b/pyproject.toml index aa9e96f0..e145d8fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dev = [ "adlfs >=2024", "huggingface_hub", "webdav4[fsspec]", + "fsspec-xrootd", # testing "moto[s3,server]", "wsgidav", diff --git a/upath/_chain.py b/upath/_chain.py index 23a25741..f141dd21 100644 --- a/upath/_chain.py +++ b/upath/_chain.py @@ -297,7 +297,12 @@ def chain(self, segments: Sequence[ChainSegment]) -> tuple[str, dict[str, Any]]: if segment.path.startswith(f"{segment.protocol}:/"): urlpath = segment.path else: - urlpath = f"{segment.protocol}://{segment.path}" + # If "hostid" is specified in the storage options then insert it + # relevant for XRootDPath + if (hostid := segment.storage_options.get("hostid")) is not None: + urlpath = f"{segment.protocol}://{hostid}/{segment.path}" + else: + urlpath = f"{segment.protocol}://{segment.path}" elif segment.protocol: urlpath = segment.protocol elif segment.path is not None: diff --git a/upath/_flavour_sources.py b/upath/_flavour_sources.py index 63fb8a9f..de49c368 100644 --- a/upath/_flavour_sources.py +++ b/upath/_flavour_sources.py @@ -988,7 +988,7 @@ def _strip_protocol(cls, path: str) -> str: class XRootDFileSystemFlavour(AbstractFileSystemFlavour): __orig_class__ = 'fsspec_xrootd.xrootd.XRootDFileSystem' - __orig_version__ = '0.5.1' + __orig_version__ = '0.5.5' protocol = ('root',) root_marker = '/' sep = '/' @@ -997,7 +997,7 @@ class XRootDFileSystemFlavour(AbstractFileSystemFlavour): def _strip_protocol(cls, path: str | list[str]) -> Any: if isinstance(path, str): if path.startswith(cls.protocol): - x = urlsplit(path); return (x.path + f'?{x.query}' if x.query else '').rstrip("/") or cls.root_marker + x = urlsplit(path); return (x.path + (f'?{x.query}' if x.query else '')).removeprefix('/').rstrip("/") or cls.root_marker # assume already stripped return path.rstrip("/") or cls.root_marker elif isinstance(path, list): diff --git a/upath/implementations/xrootd.py b/upath/implementations/xrootd.py new file mode 100644 index 00000000..6afb0681 --- /dev/null +++ b/upath/implementations/xrootd.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING + +from upath.core import UPath +from upath.types import JoinablePathLike + +if TYPE_CHECKING: + from typing import Literal + + if sys.version_info >= (3, 11): + from typing import Unpack + else: + from typing_extensions import Unpack + + from upath._chain import FSSpecChainParser + from upath.types.storage_options import XRootDStorageOptions + +__all__ = ["XRootDPath"] + + +class XRootDPath(UPath): + __slots__ = () + + if TYPE_CHECKING: + + def __init__( + self, + *args: JoinablePathLike, + protocol: Literal["root"] | None = ..., + chain_parser: FSSpecChainParser = ..., + **storage_options: Unpack[XRootDStorageOptions], + ) -> None: ... + + def mkdir( + self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False + ) -> None: + if not parents and not exist_ok and self.exists(): + raise FileExistsError(self.path) + super().mkdir(mode=mode, parents=parents, exist_ok=exist_ok) + + def write_bytes(self, data: bytes) -> int: + with self.fs.open(self.path, "wb") as f: + rv = f.write(data) + return rv diff --git a/upath/registry.py b/upath/registry.py index fb88ed4b..daac5165 100644 --- a/upath/registry.py +++ b/upath/registry.py @@ -70,6 +70,7 @@ from upath.implementations.smb import SMBPath as _SMBPath from upath.implementations.tar import TarPath as _TarPath from upath.implementations.webdav import WebdavPath as _WebdavPath + from upath.implementations.xrootd import XRootDPath as _XRootDPath from upath.implementations.zip import ZipPath as _ZipPath @@ -102,6 +103,7 @@ class _Registry(MutableMapping[str, "type[upath.UPath]"]): "http": "upath.implementations.http.HTTPPath", "https": "upath.implementations.http.HTTPPath", "memory": "upath.implementations.memory.MemoryPath", + "root": "upath.implementations.xrootd.XRootDPath", "s3": "upath.implementations.cloud.S3Path", "s3a": "upath.implementations.cloud.S3Path", "simplecache": "upath.implementations.cached.SimpleCachePath", @@ -269,6 +271,8 @@ def get_upath_class(protocol: Literal["file", "local"]) -> type[_FilePath]: ... @overload def get_upath_class(protocol: Literal["memory"]) -> type[_MemoryPath]: ... @overload + def get_upath_class(protocol: Literal["root"]) -> type[_XRootDPath]: ... + @overload def get_upath_class(protocol: Literal["sftp", "ssh"]) -> type[_SFTPPath]: ... @overload def get_upath_class(protocol: Literal["smb"]) -> type[_SMBPath]: ... diff --git a/upath/tests/conftest.py b/upath/tests/conftest.py index 4fdeae62..50d1d92b 100644 --- a/upath/tests/conftest.py +++ b/upath/tests/conftest.py @@ -742,3 +742,48 @@ def ftp_server(ftp_server_process): del_func(file_path) except Exception: pass + + +@pytest.fixture(scope="module") +def xrootd_server(): + if shutil.which("docker") is None: + pytest.skip("docker not installed") + + name = "fsspec_test_xrootd" + stop_docker(name) + cmd = ( + "docker run" + " -d" + f" --name {name}" + f" -u {os.getuid()}" + " -p 1094:1094" + " --tmpfs /shared" + " ponyisi/xrootd_docker xrootd /shared" + ) + try: + subprocess.run(shlex.split(cmd)) + yield "root://localhost//shared" + finally: + stop_docker(name) + + +@pytest.fixture +def xrootd_fixture(local_testdir, xrootd_server): + xrootd_url = xrootd_server + fs = fsspec.filesystem("root", hostid="localhost") + fs.clear_instance_cache() + pth_testdir = Path(local_testdir) + dirname = pth_testdir.parent + # XRootD filesystem does not support put() so copy files the hard way + for pth_tup in Path(local_testdir).walk(): + tdir = xrootd_url + "/" + str(pth_tup[0].relative_to(dirname)) + fs.mkdir("/shared/" + str(pth_tup[0].relative_to(dirname))) + for fpath in pth_tup[2]: + with (pth_tup[0] / fpath).open("rb") as infile: + with fsspec.open(tdir + "/" + fpath, mode="wb") as outfile: + outfile.write(infile.read()) + + try: + yield xrootd_url + "/" + Path(local_testdir).name + finally: + fs.rm("/shared/" + Path(local_testdir).name, recursive=True) diff --git a/upath/tests/implementations/test_xrootd.py b/upath/tests/implementations/test_xrootd.py new file mode 100644 index 00000000..80d72bff --- /dev/null +++ b/upath/tests/implementations/test_xrootd.py @@ -0,0 +1,20 @@ +from pathlib import Path + +import pytest + +from upath import UPath +from upath.implementations.xrootd import XRootDPath + +from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import overrides_base + + +class TestUPathXRootD(BaseTests, metaclass=OverrideMeta): + @pytest.fixture(autouse=True, scope="function") + def path(self, xrootd_fixture): + self.path = UPath(xrootd_fixture) + + @overrides_base + def test_is_correct_class(self): + assert isinstance(self.path, XRootDPath) diff --git a/upath/types/storage_options.py b/upath/types/storage_options.py index 0212a25e..1468b9d9 100644 --- a/upath/types/storage_options.py +++ b/upath/types/storage_options.py @@ -392,6 +392,26 @@ class SMBStorageOptions(_AbstractStorageOptions, total=False): auto_mkdir: bool # Whether to create parent directories when opening files +class XRootDStorageOptions(_AbstractStorageOptions, total=False): + """Storage options for XRootD filesystem""" + + # Connection settings + hostid: str # The remote server to connect to (default from ) + timeout: int # Connection timeout; 0 means no timeout (default: 0) + + # File handle cache settings + filehandle_cache_size: int # Maximum number of items (default: 256) + filehandle_cache_ttl: int # TTL in seconds (default: 30) + + # Source lookup settings + locate_all_sources: ( + bool # For reading, find all locations hosting file, ignoring redirector + ) + valid_sources: ( + list[str] | None + ) # If given and locate_all_sources is True, restrict sources to this server list + + class WebdavStorageOptions(_AbstractStorageOptions, total=False): """Storage options for WebDAV filesystem""" From 1fbdd755562fc76e27ce5deb4767e53060fd7855 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Fri, 5 Jun 2026 16:28:40 -0500 Subject: [PATCH 2/3] Fix tests in older Python versions --- upath/tests/conftest.py | 8 ++++---- upath/tests/implementations/test_xrootd.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/upath/tests/conftest.py b/upath/tests/conftest.py index 50d1d92b..19a8c132 100644 --- a/upath/tests/conftest.py +++ b/upath/tests/conftest.py @@ -775,11 +775,11 @@ def xrootd_fixture(local_testdir, xrootd_server): pth_testdir = Path(local_testdir) dirname = pth_testdir.parent # XRootD filesystem does not support put() so copy files the hard way - for pth_tup in Path(local_testdir).walk(): - tdir = xrootd_url + "/" + str(pth_tup[0].relative_to(dirname)) - fs.mkdir("/shared/" + str(pth_tup[0].relative_to(dirname))) + for pth_tup in os.walk(local_testdir): + tdir = xrootd_url + "/" + str(Path(pth_tup[0]).relative_to(dirname)) + fs.mkdir("/shared/" + str(Path(pth_tup[0]).relative_to(dirname))) for fpath in pth_tup[2]: - with (pth_tup[0] / fpath).open("rb") as infile: + with (Path(pth_tup[0]) / fpath).open("rb") as infile: with fsspec.open(tdir + "/" + fpath, mode="wb") as outfile: outfile.write(infile.read()) diff --git a/upath/tests/implementations/test_xrootd.py b/upath/tests/implementations/test_xrootd.py index 80d72bff..54e2de5c 100644 --- a/upath/tests/implementations/test_xrootd.py +++ b/upath/tests/implementations/test_xrootd.py @@ -1,5 +1,3 @@ -from pathlib import Path - import pytest from upath import UPath From d562908af4dea388b359fc1f4bb9a642c365bdc1 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Fri, 5 Jun 2026 16:46:09 -0500 Subject: [PATCH 3/3] Fix registry test --- upath/tests/test_registry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/upath/tests/test_registry.py b/upath/tests/test_registry.py index dfbcc555..abca707b 100644 --- a/upath/tests/test_registry.py +++ b/upath/tests/test_registry.py @@ -29,6 +29,7 @@ "https", "local", "memory", + "root", "s3", "s3a", "simplecache",