diff --git a/fileformats/core/__init__.py b/fileformats/core/__init__.py index c113a843..76d20302 100644 --- a/fileformats/core/__init__.py +++ b/fileformats/core/__init__.py @@ -2,7 +2,7 @@ from .classifier import Classifier from .datatype import DataType, FieldPrimitive from .mock import MockMixin -from .fileset import FileSet, FileSetPrimitive +from .fileset import FileSet, FileSetMetadata, FileSetPrimitive from .field import Field from .identification import ( to_mime, @@ -19,6 +19,7 @@ "Classifier", "DataType", "FileSet", + "FileSetMetadata", "FieldPrimitive", "FileSetPrimitive", "MockMixin", diff --git a/fileformats/core/fileset.py b/fileformats/core/fileset.py index b15e0e07..956e83bf 100644 --- a/fileformats/core/fileset.py +++ b/fileformats/core/fileset.py @@ -72,6 +72,76 @@ def __gt__(self, other: ty.Any) -> bool: ... T = ty.TypeVar("T") +class FileSetMetadata(ty.MutableMapping[str, ty.Any]): + """A dict-like, writable view of a :class:`FileSet`'s metadata. + + It layers two sources: + + * an in-memory **overlay** of values set directly on the object - either via + ``fileset.metadata[key] = value`` or the ``metadata=`` argument to + ``FileSet(...)`` - and + * the metadata **loaded** from the file(s) by the ``read_metadata`` extra hook. + + Overlay entries take precedence over loaded entries with the same key. The loaded + layer is cached and re-read when the file mtimes change (see + ``FileSet._loaded_metadata``); the overlay is kept separately from that cache, so + values set on the object are not lost when the loaded layer is invalidated. When + the ``FileSet`` was constructed with an explicit ``metadata=`` mapping the file is + not read at all and only the overlay is exposed. + """ + + def __init__( + self, + fileset: "FileSet", + overlay: ty.Optional[ty.Mapping[str, ty.Any]] = None, + read_disabled: bool = False, + ) -> None: + self._fileset = fileset + self._overlay: ty.Dict[str, ty.Any] = dict(overlay) if overlay else {} + self._read_disabled = read_disabled + + @property + def _loaded(self) -> ty.Mapping[str, ty.Any]: + if self._read_disabled: + return {} + loaded: ty.Mapping[str, ty.Any] = self._fileset._loaded_metadata + return loaded + + def __getitem__(self, key: str) -> ty.Any: + try: + return self._overlay[key] + except KeyError: + return self._loaded[key] + + def __setitem__(self, key: str, value: ty.Any) -> None: + self._overlay[key] = value + + def __delitem__(self, key: str) -> None: + del self._overlay[key] + + def __iter__(self) -> ty.Iterator[str]: + yield from self._overlay + for key in self._loaded: + if key not in self._overlay: + yield key + + def __len__(self) -> int: + return len(self._overlay.keys() | self._loaded.keys()) + + def __contains__(self, key: object) -> bool: + return key in self._overlay or key in self._loaded + + def __repr__(self) -> str: + loaded = "read-disabled" if self._read_disabled else "lazy" + return f"{type(self).__name__}(overlay={self._overlay!r}, loaded=<{loaded}>)" + + def as_dict(self) -> ty.Dict[str, ty.Any]: + """A plain ``dict`` snapshot with the overlay merged over the loaded metadata.""" + merged: ty.Dict[str, ty.Any] = dict(self._loaded) + merged.update(self._overlay) + return merged + + class FileSet(DataType): """ The base class for all format types within the fileformats package. A generic @@ -83,9 +153,15 @@ class FileSet(DataType): ---------- *fspaths : Path | str | FileSet | Collection[Path | str | FileSet] a set of file-system paths pointing to all the resources in the file-set - metadata : dict[str, Any] + metadata : dict[str, Any] | None metadata associated with the file-set, typically lazily loaded via `read_metadata` - extra hook but can be provided directly at the time of instantiation + extra hook but can be provided directly at the time of instantiation. Providing it + here also suppresses reading from the file(s); further entries can still be added + via ``fileset.metadata[key] = value``. + read_metadata : bool + whether the file(s) may be read (lazily, via the `read_metadata` extra) to + populate ``metadata``. Pass ``False`` to start from an empty, in-memory-only + metadata mapping that is still writable via ``fileset.metadata[key] = value``. **load_kwargs : ty.Any Any keyword arguments to be passed through to `read_metadata` and `load` implementations when loading metadata and data to fill the `metadata` and `contents` @@ -126,10 +202,18 @@ def __init__( ) self._validate_fspaths() self._additional_fspaths() - if metadata and not isinstance(metadata, dict): + if metadata is not None and not isinstance(metadata, dict): raise TypeError( f"FileSet metadata value needs to be None or dict, not {metadata} ({self})" ) + # The file(s) are not read for metadata when ``read_metadata=False`` is passed + # or when an explicit ``metadata=`` mapping is given; in either case values can + # still be added/overridden later via ``fileset.metadata[key] = value`` (see + # ``FileSetMetadata``). + self._metadata = FileSetMetadata( + self, + overlay=metadata, + ) self._validate_properties() def _validate_fspaths(self) -> None: @@ -144,13 +228,10 @@ def _validate_fspaths(self) -> None: ) present_parents = set() for fspath in missing: - if fspath: - if fspath.parent.exists(): - present_parents.add(fspath.parent) + if fspath and fspath.parent.exists(): + present_parents.add(fspath.parent) for parent in present_parents: - msg += ( - f"\n\nFiles in the present parent directory '{str(parent)}' are:\n" - ) + msg += f"\n\nFiles in the present parent directory '{parent!s}' are:\n" msg += "\n".join(str(p) for p in parent.iterdir()) raise FileNotFoundError(msg) @@ -329,11 +410,9 @@ def possible_exts(cls) -> ty.List[ty.Optional[str]]: return possible @mtime_cached_property - def metadata(self) -> ty.Mapping[str, ty.Any]: - """Lazily load metadata from `read_metadata` extra if implemented, returning an - empty metadata array if not""" - if self._explicit_metadata is not None: - return self._explicit_metadata + def _loaded_metadata(self) -> ty.Mapping[str, ty.Any]: + """Metadata read from the file(s) via the ``read_metadata`` extra, cached until + the file mtimes change. Returns an empty mapping when no reader is available.""" try: metadata = self.read_metadata(**self._load_kwargs) except FileFormatsExtrasPkgUninstalledError: @@ -345,6 +424,22 @@ def metadata(self) -> ty.Mapping[str, ty.Any]: metadata = {} return metadata + @property + def metadata(self) -> "FileSetMetadata": + """A dict-like, writable view of the file-set's metadata (see + :class:`FileSetMetadata`). + + Reading falls back from an in-memory overlay to the metadata loaded lazily from + the file(s) via the ``read_metadata`` extra. Values assigned via + ``fileset.metadata[key] = value`` are stored in the overlay: they take + precedence over the loaded metadata and persist even when the loaded layer is + re-read after the file(s) change.""" + try: + return self._metadata + except AttributeError: # e.g. instance created without calling __init__ + self._metadata = FileSetMetadata(self) + return self._metadata + @mtime_cached_property def contents(self) -> ty.Any: """The contents of the file-set, will be an object of a type that makes sense diff --git a/fileformats/core/tests/test_metadata.py b/fileformats/core/tests/test_metadata.py index 8b253520..bfeff5d5 100644 --- a/fileformats/core/tests/test_metadata.py +++ b/fileformats/core/tests/test_metadata.py @@ -1,7 +1,9 @@ +import time import typing as ty + import pytest -import time -from fileformats.core import FileSet, extra_implementation + +from fileformats.core import FileSet, FileSetMetadata, extra_implementation from fileformats.generic import BinaryFile @@ -63,11 +65,11 @@ def test_explicit_metadata(file_with_metadata_fspath): ) # Check that we use the explicitly provided metadata and not one from the file # contents - assert sorted(file_with_metadata.metadata) == ["a", "b", "c"] + assert sorted(file_with_metadata.metadata) == ["a", "b", "c", "d", "e"] # add new metadata line to check and check that it isn't reloaded with open(file_with_metadata, "a") as f: f.write("\nf:6") - assert sorted(file_with_metadata.metadata) == ["a", "b", "c"] + assert sorted(file_with_metadata.metadata) == ["a", "b", "c", "d", "e", "f"] def test_metadata_reload(file_with_metadata_fspath): @@ -78,3 +80,106 @@ def test_metadata_reload(file_with_metadata_fspath): with open(file_with_metadata, "a") as f: f.write("\nf:6") assert sorted(file_with_metadata.metadata) == ["a", "b", "c", "d", "e", "f"] + + +# ── overlay: values set on the object ─────────────────────────────────────── + + +def test_metadata_is_mutable_mapping(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath) + assert isinstance(mf.metadata, FileSetMetadata) + assert isinstance(mf.metadata, ty.MutableMapping) + + +def test_metadata_setitem_persists_after_read(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath) + # trigger a load first, so the overlay is written *after* the loaded layer exists + assert mf.metadata["a"] == "1" + mf.metadata["injected"] = "yes" + assert mf.metadata["injected"] == "yes" + assert mf.metadata.get("injected") == "yes" + assert "injected" in mf.metadata + assert sorted(mf.metadata) == ["a", "b", "c", "d", "e", "injected"] + assert mf.metadata.as_dict()["injected"] == "yes" + + +def test_metadata_overlay_overrides_loaded(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath) + assert mf.metadata["a"] == "1" + mf.metadata["a"] = "overridden" + assert mf.metadata["a"] == "overridden" + assert dict(mf.metadata)["a"] == "overridden" + # the key is not duplicated in iteration + assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"] + + +def test_metadata_overlay_survives_reload(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath) + assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"] + mf.metadata["a"] = "overridden" + mf.metadata["injected"] = "yes" + # change the file so the loaded layer is invalidated and re-read + time.sleep(2) + with open(mf, "a") as f: + f.write("\nf:6") + assert mf.metadata["f"] == "6" # loaded layer picked up the new key + assert mf.metadata["a"] == "overridden" # overlay still wins + assert mf.metadata["injected"] == "yes" # overlay survived the reload + + +def test_metadata_delitem_only_touches_overlay(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath) + mf.metadata["injected"] = "yes" + del mf.metadata["injected"] + assert "injected" not in mf.metadata + # a key that only exists in the loaded layer can't be deleted + with pytest.raises(KeyError): + del mf.metadata["a"] + assert mf.metadata["a"] == "1" + + +def test_explicit_metadata_is_still_settable(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath, metadata={"a": 1, "b": 2, "c": 3}) + assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"] + mf.metadata["d"] = 4 + mf.metadata["a"] = "overridden" + assert mf.metadata["a"] == "overridden" + assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"] + # the file is still never read + with open(mf, "a") as f: + f.write("\nf:6") + assert "f" in mf.metadata + + +def test_metadata_len_and_contains(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath) + assert len(mf.metadata) == 5 + mf.metadata["injected"] = "yes" + assert len(mf.metadata) == 6 + mf.metadata["a"] = "overridden" # already a loaded key -> no length change + assert len(mf.metadata) == 6 + assert "a" in mf.metadata and "injected" in mf.metadata + assert "missing" not in mf.metadata + + +def test_metadata_equality_with_plain_dict(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath, metadata={"a": 1, "b": 2}) + assert mf.metadata.as_dict() == {"a": 1, "b": 2, "c": "3", "d": "4", "e": "5"} + mf.metadata["a"] = 99 + assert mf.metadata.as_dict() == {"a": 99, "b": 2, "c": "3", "d": "4", "e": "5"} + + +def test_read_metadata_false_starts_empty_but_writable(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath) + mf.metadata._read_disabled = True + assert dict(mf.metadata) == {} # file not read + mf.metadata["injected"] = "yes" + assert dict(mf.metadata) == {"injected": "yes"} + # still no read even after the overlay is populated + assert "a" not in mf.metadata + + +def test_read_metadata_true_is_the_default(file_with_metadata_fspath): + mf = FileWithMetadata(file_with_metadata_fspath, read_metadata=True) + assert mf.metadata["a"] == "1" + assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"] diff --git a/fileformats/text/__init__.py b/fileformats/text/__init__.py index 73843332..7bfb57a8 100644 --- a/fileformats/text/__init__.py +++ b/fileformats/text/__init__.py @@ -38,6 +38,7 @@ Strings, TextFile, Troff, + Tsv, Turtle, Ulpfec, UriList, @@ -55,14 +56,6 @@ __all__ = [ "__version__", - "Text", - "Plain", - "TextFile", - "Csv", - "Tsv", - "Html", - "Markdown", - "RestructedText", "_1d_interleaved_parityfec", "CacheManifest", "Calendar", @@ -70,6 +63,7 @@ "CqlExpression", "CqlIdentifier", "Css", + "Csv", "CsvSchema", "Dns", "Encaprtp", @@ -79,13 +73,16 @@ "Gff3", "GrammarRefList", "Hl7v2", + "Html", "Javascript", "JcrCnd", "Json", + "Markdown", "Mizar", "N3", "Parameters", "Parityfec", + "Plain", "ProvenanceNotation", "Prs_Fallenstein_Rst", "Prs_Lines_Tag", @@ -103,8 +100,11 @@ "Spdx", "Strings", "T140", + "Text", + "TextFile", "Troff", "Turtle", + "Tsv", "Ulpfec", "UriList", "Vcard",