Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
5037a3d
fix(#1469): codec-driven GC discovery + file-level schema-addressed o…
dimitri-yatsenko Jul 2, 2026
6e89a7d
fix(gc): coverage-based orphan matching for directory-valued objects
dimitri-yatsenko Jul 6, 2026
4a898c4
refactor(gc): remove dead _extract_hash_refs/_extract_schema_refs
dimitri-yatsenko Jul 6, 2026
49bdbb0
fix(gc): hash-subtree skip by first path segment; share prefix constant
dimitri-yatsenko Jul 6, 2026
068b365
fix(gc): honor filepath_prefix as a protected namespace; re-document …
dimitri-yatsenko Jul 6, 2026
6e139a3
fix(filepath): keep the fixed _hash layout reserved when hash_prefix …
dimitri-yatsenko Jul 6, 2026
c74a830
fix(storage): honor hash_prefix/schema_prefix settings in writers and GC
dimitri-yatsenko Jul 6, 2026
f494a68
refactor(storage): drop builder prefix fallbacks — settings is the si…
dimitri-yatsenko Jul 6, 2026
9253600
fix(gc): never flag a live hash object as a schema orphan (prefix-cha…
dimitri-yatsenko Jul 6, 2026
93eb695
docs(gc): correct docstrings for settings-driven path construction an…
dimitri-yatsenko Jul 6, 2026
81d3516
refactor(gc): remove format_stats
dimitri-yatsenko Jul 6, 2026
b0dad04
feat(gc): per-schema scoping — scan/collect confine orphan detection …
dimitri-yatsenko Jul 6, 2026
e5bdb9b
refactor(gc): require schema_name; walk only the schema's own sections
dimitri-yatsenko Jul 6, 2026
06c9269
refactor(heading): storage-model classification lives on Heading/Attr…
dimitri-yatsenko Jul 6, 2026
b463702
refactor(gc): rename list_stored_hashes -> list_hash_paths
dimitri-yatsenko Jul 6, 2026
162ccb7
refactor(gc): store-bound GarbageCollector class as the sole API
dimitri-yatsenko Jul 6, 2026
e6cd1a9
docs(test): refer to GarbageCollector.scan in test docstring
dimitri-yatsenko Jul 6, 2026
6f20aef
refactor(gc): schemas in the constructor; merge scan into collect
dimitri-yatsenko Jul 6, 2026
f6bcccb
refactor(gc): consistent hash-side stat names (hash_paths_*, orphaned…
dimitri-yatsenko Jul 6, 2026
8218f64
refactor(gc): rename orphaned_paths -> orphaned_schema_paths for full…
dimitri-yatsenko Jul 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/datajoint/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ def FreeTable(conn_or_name, full_table_name: str | None = None) -> _FreeTable:
"diagram": (".diagram", None), # Return the module itself
# cli imports click
"cli": (".cli", "cli"),
# gc — exposed lazily so `dj.gc.scan(...)` works as documented in gc.py
# gc — exposed lazily so `dj.gc.GarbageCollector(...)` works as documented in gc.py
# and in the user docs (how-to/garbage-collection.md).
"gc": (".gc", None), # Return the module itself
}
Expand Down
2 changes: 1 addition & 1 deletion src/datajoint/builtin_codecs/hash.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class HashCodec(Codec):
The database column stores JSON metadata: ``{hash, store, size}``.
Duplicate content is automatically deduplicated across all tables.

Deletion: Requires garbage collection via ``dj.gc.collect()``.
Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.

External only - requires @ modifier.

Expand Down
2 changes: 1 addition & 1 deletion src/datajoint/builtin_codecs/npy.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ class Recording(dj.Manual):
- Path: ``{schema}/{table}/{pk}/{attribute}.npy``
- Database column: JSON with ``{path, store, dtype, shape}``

Deletion: Requires garbage collection via ``dj.gc.collect()``.
Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.

See Also
--------
Expand Down
2 changes: 1 addition & 1 deletion src/datajoint/builtin_codecs/object.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def make(self, key):

{store_root}/{schema}/{table}/{pk}/{field}/

Deletion: Requires garbage collection via ``dj.gc.collect()``.
Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.

Comparison with hash-addressed::

Expand Down
4 changes: 3 additions & 1 deletion src/datajoint/builtin_codecs/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ def _build_path(
Build schema-addressed storage path.

Constructs a path that mirrors the database schema structure:
``{schema}/{table}/{pk_values}/{field}{ext}``
``{schema_prefix}/{schema}/{table}/{pk_values}/{field}{ext}``

Supports partitioning if configured in the store.

Expand Down Expand Up @@ -150,6 +150,7 @@ def _build_path(
spec = config.get_store_spec(store_name)
partition_pattern = spec.get("partition_pattern")
token_length = spec.get("token_length", 8)
schema_prefix = spec["schema_prefix"] # always present: settings applies the default

return build_object_path(
schema=schema,
Expand All @@ -159,6 +160,7 @@ def _build_path(
ext=ext,
partition_pattern=partition_pattern,
token_length=token_length,
schema_prefix=schema_prefix,
)

def _get_backend(self, store_name: str | None = None, config=None):
Expand Down
40 changes: 40 additions & 0 deletions src/datajoint/codecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class MyTable(dj.Manual):

from __future__ import annotations

import json
import logging
from abc import ABC, abstractmethod
from typing import Any
Expand Down Expand Up @@ -219,6 +220,45 @@ def validate(self, value: Any) -> None:
"""
pass

def referenced_paths(self, stored: Any) -> list[tuple[str, str | None]]:
"""
Return the external store paths this stored value references.

Garbage collection and delete-time cleanup call this on the *stored*
(encoded) representation of a column value — the JSON/dict already in
the database — so discovery stays metadata-only (no download or decode).

The default recognizes DataJoint's standard external-storage metadata:
a dict (or JSON string) carrying a ``path`` and optional ``store``. This
covers hash-addressed (``<hash@>``/``<blob@>``/``<attach@>``) and
schema-addressed (``<object@>``/``<npy@>`` and any :class:`SchemaCodec`
subclass) storage with no extra work, and returns an empty list for
values that are not externally stored (e.g. in-table ``<blob>``).

Override only if your codec stores external references in a non-standard
shape. Returning the paths here is what lets garbage collection see a
custom codec's files as *referenced* rather than orphaned (see #1469).

Parameters
----------
stored : any
The stored (encoded) value as read from the database column.

Returns
-------
list[tuple[str, str | None]]
``(path, store_name)`` for each external artifact referenced.
"""
value = stored
if isinstance(value, str):
try:
value = json.loads(value)
except (json.JSONDecodeError, TypeError, ValueError):
return []
if isinstance(value, dict) and "path" in value:
return [(value["path"], value.get("store"))]
return []

def __repr__(self) -> str:
return f"<{self.__class__.__name__}(name={self.name!r})>"

Expand Down
Loading
Loading