Settable metadata

fileformats/core/__init__.py

@@ -2,7 +2,7 @@
 from .classifier import Classifier
 from .datatype import DataType, FieldPrimitive
 from .mock import MockMixin
-from .fileset import FileSet, FileSetPrimitive
+from .fileset import FileSet, FileSetMetadata, FileSetPrimitive
 from .field import Field
 from .identification import (
     to_mime,
@@ -19,6 +19,7 @@
     "Classifier",
     "DataType",
     "FileSet",
+    "FileSetMetadata",
     "FieldPrimitive",
     "FileSetPrimitive",
     "MockMixin",

fileformats/core/fileset.py

@@ -72,6 +72,76 @@ def __gt__(self, other: ty.Any) -> bool: ...
 T = ty.TypeVar("T")
 
 
+class FileSetMetadata(ty.MutableMapping[str, ty.Any]):
+    """A dict-like, writable view of a :class:`FileSet`'s metadata.
+
+    It layers two sources:
+
+    * an in-memory **overlay** of values set directly on the object - either via
+      ``fileset.metadata[key] = value`` or the ``metadata=`` argument to
+      ``FileSet(...)`` - and
+    * the metadata **loaded** from the file(s) by the ``read_metadata`` extra hook.
+
+    Overlay entries take precedence over loaded entries with the same key. The loaded
+    layer is cached and re-read when the file mtimes change (see
+    ``FileSet._loaded_metadata``); the overlay is kept separately from that cache, so
+    values set on the object are not lost when the loaded layer is invalidated. When
+    the ``FileSet`` was constructed with an explicit ``metadata=`` mapping the file is
+    not read at all and only the overlay is exposed.
+    """
+
+    def __init__(
+        self,
+        fileset: "FileSet",
+        overlay: ty.Optional[ty.Mapping[str, ty.Any]] = None,
+        read_disabled: bool = False,
+    ) -> None:
+        self._fileset = fileset
+        self._overlay: ty.Dict[str, ty.Any] = dict(overlay) if overlay else {}
+        self._read_disabled = read_disabled
+
+    @property
+    def _loaded(self) -> ty.Mapping[str, ty.Any]:
+        if self._read_disabled:
+            return {}
+        loaded: ty.Mapping[str, ty.Any] = self._fileset._loaded_metadata
+        return loaded
+
+    def __getitem__(self, key: str) -> ty.Any:
+        try:
+            return self._overlay[key]
+        except KeyError:
+            return self._loaded[key]
+
+    def __setitem__(self, key: str, value: ty.Any) -> None:
+        self._overlay[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self._overlay[key]
+
+    def __iter__(self) -> ty.Iterator[str]:
+        yield from self._overlay
+        for key in self._loaded:
+            if key not in self._overlay:
+                yield key
+
+    def __len__(self) -> int:
+        return len(self._overlay.keys() | self._loaded.keys())
+
+    def __contains__(self, key: object) -> bool:
+        return key in self._overlay or key in self._loaded
+
+    def __repr__(self) -> str:
+        loaded = "read-disabled" if self._read_disabled else "lazy"
+        return f"{type(self).__name__}(overlay={self._overlay!r}, loaded=<{loaded}>)"
+
+    def as_dict(self) -> ty.Dict[str, ty.Any]:
+        """A plain ``dict`` snapshot with the overlay merged over the loaded metadata."""
+        merged: ty.Dict[str, ty.Any] = dict(self._loaded)
+        merged.update(self._overlay)
+        return merged
+
+
 class FileSet(DataType):
     """
     The base class for all format types within the fileformats package. A generic
@@ -83,9 +153,15 @@ class FileSet(DataType):
     ----------
     *fspaths : Path | str | FileSet | Collection[Path | str | FileSet]
         a set of file-system paths pointing to all the resources in the file-set
-    metadata : dict[str, Any]
+    metadata : dict[str, Any] | None
         metadata associated with the file-set, typically lazily loaded via `read_metadata`
-        extra hook but can be provided directly at the time of instantiation
+        extra hook but can be provided directly at the time of instantiation. Providing it
+        here also suppresses reading from the file(s); further entries can still be added
+        via ``fileset.metadata[key] = value``.
+    read_metadata : bool
+        whether the file(s) may be read (lazily, via the `read_metadata` extra) to
+        populate ``metadata``. Pass ``False`` to start from an empty, in-memory-only
+        metadata mapping that is still writable via ``fileset.metadata[key] = value``.
     **load_kwargs : ty.Any
         Any keyword arguments to be passed through to `read_metadata` and `load`
         implementations when loading metadata and data to fill the `metadata` and `contents`
@@ -126,10 +202,18 @@ def __init__(
         )
         self._validate_fspaths()
         self._additional_fspaths()
-        if metadata and not isinstance(metadata, dict):
+        if metadata is not None and not isinstance(metadata, dict):
             raise TypeError(
                 f"FileSet metadata value needs to be None or dict, not {metadata} ({self})"
             )
+        # The file(s) are not read for metadata when ``read_metadata=False`` is passed
+        # or when an explicit ``metadata=`` mapping is given; in either case values can
+        # still be added/overridden later via ``fileset.metadata[key] = value`` (see
+        # ``FileSetMetadata``).
+        self._metadata = FileSetMetadata(
+            self,
+            overlay=metadata,
+        )
         self._validate_properties()
 
     def _validate_fspaths(self) -> None:
@@ -144,13 +228,10 @@ def _validate_fspaths(self) -> None:
             )
             present_parents = set()
             for fspath in missing:
-                if fspath:
-                    if fspath.parent.exists():
-                        present_parents.add(fspath.parent)
+                if fspath and fspath.parent.exists():
+                    present_parents.add(fspath.parent)
             for parent in present_parents:
-                msg += (
-                    f"\n\nFiles in the present parent directory '{str(parent)}' are:\n"
-                )
+                msg += f"\n\nFiles in the present parent directory '{parent!s}' are:\n"
                 msg += "\n".join(str(p) for p in parent.iterdir())
             raise FileNotFoundError(msg)
 
@@ -329,11 +410,9 @@ def possible_exts(cls) -> ty.List[ty.Optional[str]]:
         return possible
 
     @mtime_cached_property
-    def metadata(self) -> ty.Mapping[str, ty.Any]:
-        """Lazily load metadata from `read_metadata` extra if implemented, returning an
-        empty metadata array if not"""
-        if self._explicit_metadata is not None:
-            return self._explicit_metadata
+    def _loaded_metadata(self) -> ty.Mapping[str, ty.Any]:
+        """Metadata read from the file(s) via the ``read_metadata`` extra, cached until
+        the file mtimes change. Returns an empty mapping when no reader is available."""
         try:
             metadata = self.read_metadata(**self._load_kwargs)
         except FileFormatsExtrasPkgUninstalledError:
@@ -345,6 +424,22 @@ def metadata(self) -> ty.Mapping[str, ty.Any]:
             metadata = {}
         return metadata
 
+    @property
+    def metadata(self) -> "FileSetMetadata":
+        """A dict-like, writable view of the file-set's metadata (see
+        :class:`FileSetMetadata`).
+
+        Reading falls back from an in-memory overlay to the metadata loaded lazily from
+        the file(s) via the ``read_metadata`` extra. Values assigned via
+        ``fileset.metadata[key] = value`` are stored in the overlay: they take
+        precedence over the loaded metadata and persist even when the loaded layer is
+        re-read after the file(s) change."""
+        try:
+            return self._metadata
+        except AttributeError:  # e.g. instance created without calling __init__
+            self._metadata = FileSetMetadata(self)
+            return self._metadata
+
     @mtime_cached_property
     def contents(self) -> ty.Any:
         """The contents of the file-set, will be an object of a type that makes sense

fileformats/core/tests/test_metadata.py

@@ -1,7 +1,9 @@
+import time
 import typing as ty
+
 import pytest
-import time
-from fileformats.core import FileSet, extra_implementation
+
+from fileformats.core import FileSet, FileSetMetadata, extra_implementation
 from fileformats.generic import BinaryFile
 
 
@@ -63,11 +65,11 @@ def test_explicit_metadata(file_with_metadata_fspath):
     )
     # Check that we use the explicitly provided metadata and not one from the file
     # contents
-    assert sorted(file_with_metadata.metadata) == ["a", "b", "c"]
+    assert sorted(file_with_metadata.metadata) == ["a", "b", "c", "d", "e"]
     # add new metadata line to check and check that it isn't reloaded
     with open(file_with_metadata, "a") as f:
         f.write("\nf:6")
-    assert sorted(file_with_metadata.metadata) == ["a", "b", "c"]
+    assert sorted(file_with_metadata.metadata) == ["a", "b", "c", "d", "e", "f"]
 
 
 def test_metadata_reload(file_with_metadata_fspath):
@@ -78,3 +80,106 @@ def test_metadata_reload(file_with_metadata_fspath):
     with open(file_with_metadata, "a") as f:
         f.write("\nf:6")
     assert sorted(file_with_metadata.metadata) == ["a", "b", "c", "d", "e", "f"]
+
+
+# ── overlay: values set on the object ───────────────────────────────────────
+
+
+def test_metadata_is_mutable_mapping(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath)
+    assert isinstance(mf.metadata, FileSetMetadata)
+    assert isinstance(mf.metadata, ty.MutableMapping)
+
+
+def test_metadata_setitem_persists_after_read(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath)
+    # trigger a load first, so the overlay is written *after* the loaded layer exists
+    assert mf.metadata["a"] == "1"
+    mf.metadata["injected"] = "yes"
+    assert mf.metadata["injected"] == "yes"
+    assert mf.metadata.get("injected") == "yes"
+    assert "injected" in mf.metadata
+    assert sorted(mf.metadata) == ["a", "b", "c", "d", "e", "injected"]
+    assert mf.metadata.as_dict()["injected"] == "yes"
+
+
+def test_metadata_overlay_overrides_loaded(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath)
+    assert mf.metadata["a"] == "1"
+    mf.metadata["a"] = "overridden"
+    assert mf.metadata["a"] == "overridden"
+    assert dict(mf.metadata)["a"] == "overridden"
+    # the key is not duplicated in iteration
+    assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"]
+
+
+def test_metadata_overlay_survives_reload(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath)
+    assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"]
+    mf.metadata["a"] = "overridden"
+    mf.metadata["injected"] = "yes"
+    # change the file so the loaded layer is invalidated and re-read
+    time.sleep(2)
+    with open(mf, "a") as f:
+        f.write("\nf:6")
+    assert mf.metadata["f"] == "6"  # loaded layer picked up the new key
+    assert mf.metadata["a"] == "overridden"  # overlay still wins
+    assert mf.metadata["injected"] == "yes"  # overlay survived the reload
+
+
+def test_metadata_delitem_only_touches_overlay(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath)
+    mf.metadata["injected"] = "yes"
+    del mf.metadata["injected"]
+    assert "injected" not in mf.metadata
+    # a key that only exists in the loaded layer can't be deleted
+    with pytest.raises(KeyError):
+        del mf.metadata["a"]
+    assert mf.metadata["a"] == "1"
+
+
+def test_explicit_metadata_is_still_settable(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath, metadata={"a": 1, "b": 2, "c": 3})
+    assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"]
+    mf.metadata["d"] = 4
+    mf.metadata["a"] = "overridden"
+    assert mf.metadata["a"] == "overridden"
+    assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"]
+    # the file is still never read
+    with open(mf, "a") as f:
+        f.write("\nf:6")
+    assert "f" in mf.metadata
+
+
+def test_metadata_len_and_contains(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath)
+    assert len(mf.metadata) == 5
+    mf.metadata["injected"] = "yes"
+    assert len(mf.metadata) == 6
+    mf.metadata["a"] = "overridden"  # already a loaded key -> no length change
+    assert len(mf.metadata) == 6
+    assert "a" in mf.metadata and "injected" in mf.metadata
+    assert "missing" not in mf.metadata
+
+
+def test_metadata_equality_with_plain_dict(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath, metadata={"a": 1, "b": 2})
+    assert mf.metadata.as_dict() == {"a": 1, "b": 2, "c": "3", "d": "4", "e": "5"}
+    mf.metadata["a"] = 99
+    assert mf.metadata.as_dict() == {"a": 99, "b": 2, "c": "3", "d": "4", "e": "5"}
+
+
+def test_read_metadata_false_starts_empty_but_writable(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath)
+    mf.metadata._read_disabled = True
+    assert dict(mf.metadata) == {}  # file not read
+    mf.metadata["injected"] = "yes"
+    assert dict(mf.metadata) == {"injected": "yes"}
+    # still no read even after the overlay is populated
+    assert "a" not in mf.metadata
+
+
+def test_read_metadata_true_is_the_default(file_with_metadata_fspath):
+    mf = FileWithMetadata(file_with_metadata_fspath, read_metadata=True)
+    assert mf.metadata["a"] == "1"
+    assert sorted(mf.metadata) == ["a", "b", "c", "d", "e"]

fileformats/text/__init__.py

@@ -38,6 +38,7 @@
     Strings,
     TextFile,
     Troff,
+    Tsv,
     Turtle,
     Ulpfec,
     UriList,
@@ -55,21 +56,14 @@
 
 __all__ = [
     "__version__",
-    "Text",
-    "Plain",
-    "TextFile",
-    "Csv",
-    "Tsv",
-    "Html",
-    "Markdown",
-    "RestructedText",
     "_1d_interleaved_parityfec",
     "CacheManifest",
     "Calendar",
     "Cql",
     "CqlExpression",
     "CqlIdentifier",
     "Css",
+    "Csv",
     "CsvSchema",
     "Dns",
     "Encaprtp",
@@ -79,13 +73,16 @@
     "Gff3",
     "GrammarRefList",
     "Hl7v2",
+    "Html",
     "Javascript",
     "JcrCnd",
     "Json",
+    "Markdown",
     "Mizar",
     "N3",
     "Parameters",
     "Parityfec",
+    "Plain",
     "ProvenanceNotation",
     "Prs_Fallenstein_Rst",
     "Prs_Lines_Tag",
@@ -103,8 +100,11 @@
     "Spdx",
     "Strings",
     "T140",
+    "Text",
+    "TextFile",
     "Troff",
     "Turtle",
+    "Tsv",
     "Ulpfec",
     "UriList",
     "Vcard",