Skip to content

Commit 6f20aef

Browse files
refactor(gc): schemas in the constructor; merge scan into collect
Bind the schema set at construction instead of passing it into every method; and since collect(dry_run=True) is exactly the read-only scan, fold scan in. - GarbageCollector(*schemas, store=None, config=None): schemas required and fixed; config DEFAULTS to schemas[0].connection._config. Store resolved eagerly. - references/collect operate on self.schemas (no schema args). list_hash_paths /list_schema_paths still take a schema_name. - Removed scan(); collect(dry_run=True) default returns the full per-section report plus deletion outcome; dry_run=False also deletes. Size maps from the scan are reused for the byte tally (no second listing). - References gathered across all schemas at once (paths embed the schema, so combined coverage equals per-schema coverage). GC 39/39; hash+object+unit 377 passed.
1 parent e6cd1a9 commit 6f20aef

2 files changed

Lines changed: 180 additions & 249 deletions

File tree

src/datajoint/gc.py

Lines changed: 101 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,20 @@
2121
Deletion: Requires garbage collection
2222
2323
Garbage collection is **store-specific** and handles one store at a time. The
24-
:class:`GarbageCollector` is bound to a store (and the config that defines it)
25-
at construction; its store and config are not threaded through each call::
24+
:class:`GarbageCollector` is bound to its schemas and store at construction;
25+
neither is threaded through each call (the config defaults to the schemas')::
2626
2727
import datajoint as dj
2828
29-
collector = dj.gc.GarbageCollector(store="mystore")
30-
stats = collector.scan(schema1, schema2) # read-only
31-
stats = collector.collect(schema1, schema2, dry_run=False)
29+
collector = dj.gc.GarbageCollector(schema1, schema2, store="mystore")
30+
stats = collector.collect() # read-only report (dry_run=True default)
31+
stats = collector.collect(dry_run=False) # actually delete the orphans
3232
3333
Both sections embed the schema name in every path, so garbage collection is
3434
**per-schema**: each schema is scanned against its own subtree only. Orphan
35-
detection is therefore confined to the schemas you pass — any subset of the
36-
schemas sharing a store may be scanned/collected safely, and GC never touches
37-
another schema's objects or user-managed content elsewhere in the store.
35+
detection is therefore confined to the collector's schemas — any subset of the
36+
schemas sharing a store may be collected safely, and GC never touches another
37+
schema's objects or user-managed content elsewhere in the store.
3838
3939
See Also
4040
--------
@@ -87,65 +87,71 @@ def _is_covered(path: str, referenced: set[str]) -> bool:
8787

8888
class GarbageCollector:
8989
"""
90-
Store-specific garbage collector — handles exactly one store.
90+
Store-specific garbage collector — one store, a fixed set of schemas.
9191
92-
Bound to a store name and the config that defines it at construction, so
93-
neither is threaded through individual operations. Storage-side work (the
94-
stored-file listings, deletion) uses this store/config; database metadata
95-
is read through each scanned schema's own connection.
92+
The schemas to collect and the store are bound at construction, so neither
93+
is threaded through individual operations. The store is resolved eagerly
94+
(an unknown/misconfigured store raises immediately). Storage-side work (the
95+
stored-file listings, deletion) uses this store; database metadata is read
96+
through each schema's own connection.
9697
9798
Parameters
9899
----------
100+
*schemas : Schema
101+
The schemas to collect. At least one is required. Every managed path
102+
embeds its schema, so collection is per-schema and confined to these
103+
schemas: objects of any other schema sharing the store are never listed
104+
and never at risk — pass any subset safely.
99105
store : str, optional
100-
Store name (None = the default store). Resolved at construction — an
101-
unknown/misconfigured store raises immediately.
106+
Store name (None = the default store), keyword-only.
102107
config : Config, optional
103-
Config that defines the store. If None, the global ``settings.config``
104-
is used.
108+
Config that defines the store. Defaults to the first schema's
109+
connection config (``schemas[0].connection._config``).
105110
"""
106111

107-
def __init__(self, store: str | None = None, *, config=None) -> None:
108-
if config is None:
109-
from .settings import config as _global_config
110-
111-
config = _global_config
112+
def __init__(self, *schemas: "Schema", store: str | None = None, config=None) -> None:
113+
if not schemas:
114+
raise DataJointError("At least one schema must be provided")
115+
self.schemas = schemas
112116
self.store = store
113-
self.config = config
117+
self.config = config if config is not None else schemas[0].connection._config
114118
# Resolve the store eagerly: validates it exists and pins the backend
115119
# and section prefixes for this collector's lifetime (one store only).
116-
self.backend = get_store_backend(store, config=config)
117-
spec = config.get_store_spec(store)
120+
self.backend = get_store_backend(store, config=self.config)
121+
spec = self.config.get_store_spec(store)
118122
self._hash_prefix = spec["hash_prefix"].strip("/") # settings applies the "_hash" default
119123
self._schema_prefix = spec["schema_prefix"].strip("/") # ... "_schema" default
120124

121125
# ------------------------------------------------------------------ #
122126
# References — extracted from database metadata (per-schema connection)
123127
# ------------------------------------------------------------------ #
124-
def hash_references(self, *schemas: "Schema", verbose: bool = False) -> set[str]:
128+
def hash_references(self, verbose: bool = False) -> set[str]:
125129
"""
126-
Full relative store paths of hash-addressed objects referenced by live rows.
130+
Full relative store paths of hash-addressed objects referenced by live
131+
rows across this collector's schemas.
127132
128133
Reads columns classified by ``Heading.hash_objects`` (``<hash@>``, and
129134
external ``<blob@>``/``<attach@>``). Paths are taken verbatim from each
130135
row's metadata (independent of the store's current prefixes) and
131136
filtered to this collector's store.
132137
"""
133-
return self._references("hash_objects", schemas, verbose)
138+
return self._references("hash_objects", verbose)
134139

135-
def schema_references(self, *schemas: "Schema", verbose: bool = False) -> set[str]:
140+
def schema_references(self, verbose: bool = False) -> set[str]:
136141
"""
137-
Full relative store paths of schema-addressed objects referenced by live rows.
142+
Full relative store paths of schema-addressed objects referenced by live
143+
rows across this collector's schemas.
138144
139145
Reads columns classified by ``Heading.schema_objects`` — any
140146
``SchemaCodec`` (``<object@>``, ``<npy@>``, custom subclasses,
141147
recognized by type per #1469). For a directory-valued object the
142148
recorded path is the directory prefix, not its individual files.
143149
"""
144-
return self._references("schema_objects", schemas, verbose)
150+
return self._references("schema_objects", verbose)
145151

146-
def _references(self, heading_attr: str, schemas, verbose: bool) -> set[str]:
152+
def _references(self, heading_attr: str, verbose: bool) -> set[str]:
147153
referenced: set[str] = set()
148-
for schema in schemas:
154+
for schema in self.schemas:
149155
if verbose:
150156
logger.info(f"Scanning schema {schema.database} for {heading_attr}")
151157
for table_name in schema.list_tables():
@@ -294,133 +300,98 @@ def delete_schema_path(self, path: str) -> bool:
294300
# ------------------------------------------------------------------ #
295301
# Orchestration
296302
# ------------------------------------------------------------------ #
297-
def scan(self, *schemas: "Schema", verbose: bool = False) -> dict[str, Any]:
303+
def collect(self, dry_run: bool = True, verbose: bool = False) -> dict[str, Any]:
298304
"""
299-
Scan for orphaned storage items without deleting.
305+
Report — and, unless ``dry_run``, remove — orphaned storage.
300306
301-
Scans both hash-addressed and schema-addressed storage. Each schema is
302-
scanned against ITS OWN subtree, so orphan detection is confined to the
303-
schemas passed — objects of other schemas sharing the store are never
304-
listed and never at risk; any subset may be passed safely. Objects
305-
under a *former* prefix stay readable via their metadata paths but are
306-
not reclamation candidates until the setting is restored.
307+
``dry_run=True`` (the default) is a **read-only scan**: it reports what
308+
would be removed and deletes nothing. ``dry_run=False`` additionally
309+
deletes the orphans. The report is the same either way, so the safe
310+
default doubles as the inspection call.
307311
308-
Returns a dict with per-section statistics:
312+
Every managed path embeds its schema, so a schema's stored files can
313+
only match that schema's references — orphan detection is confined to
314+
this collector's schemas; objects of any other schema on the store are
315+
never listed, deleted, or at risk. Objects under a *former* prefix stay
316+
readable via their metadata paths but are not reclamation candidates
317+
until the setting is restored.
318+
319+
Returns a dict with per-section stats and the deletion outcome:
309320
310321
- hash_referenced / hash_stored / hash_orphaned / hash_orphaned_bytes
311322
- orphaned_hashes: orphaned hash items as full relative store paths
312323
(NOT bare hashes — passed as-is to the deleter)
313324
- schema_paths_referenced / schema_paths_stored / schema_paths_orphaned
314325
/ schema_paths_orphaned_bytes
315326
- orphaned_paths: orphaned schema files as full relative store paths
327+
- hash_deleted / schema_paths_deleted / deleted / bytes_freed (0 when
328+
dry_run) / errors / dry_run
316329
"""
317-
if not schemas:
318-
raise DataJointError("At least one schema must be provided")
330+
# References can be gathered across all schemas at once: because paths
331+
# embed the schema, a file under schema X is only ever covered by an X
332+
# reference, so combined coverage equals per-schema coverage.
333+
hash_referenced = self.hash_references(verbose=verbose)
334+
schema_paths_referenced = self.schema_references(verbose=verbose)
319335

320-
hash_referenced: set[str] = set()
321336
hash_stored: dict[str, int] = {}
322-
orphaned_hashes: set[str] = set()
323-
schema_paths_referenced: set[str] = set()
324337
schema_paths_stored: dict[str, int] = {}
325-
orphaned_paths: set[str] = set()
326-
327-
for schema in schemas:
328-
# --- Hash-addressed storage (this schema's subtree) ---
329-
h_ref = self.hash_references(schema, verbose=verbose)
330-
h_stored = self.list_hash_paths(schema.database)
331-
orphaned_hashes |= set(h_stored.keys()) - h_ref
332-
333-
# --- Schema-addressed storage (this schema's section) ---
334-
s_ref = self.schema_references(schema, verbose=verbose)
335-
s_stored = self.list_schema_paths(schema.database)
336-
# Coverage, not exact set difference: a referenced path may be a
337-
# directory-valued object (many files + a manifest). The walk is
338-
# confined to this schema's section, so coverage against s_ref alone
339-
# is complete (no hash objects can appear here).
340-
orphaned_paths |= {p for p in s_stored if not _is_covered(p, s_ref)}
341-
342-
hash_referenced |= h_ref
343-
hash_stored.update(h_stored)
344-
schema_paths_referenced |= s_ref
345-
schema_paths_stored.update(s_stored)
338+
for schema in self.schemas:
339+
hash_stored.update(self.list_hash_paths(schema.database))
340+
schema_paths_stored.update(self.list_schema_paths(schema.database))
341+
342+
orphaned_hashes = sorted(set(hash_stored.keys()) - hash_referenced)
343+
# Coverage, not exact set difference: a referenced path may be a
344+
# directory-valued object (many files + a manifest sidecar).
345+
orphaned_paths = sorted(p for p in schema_paths_stored if not _is_covered(p, schema_paths_referenced))
346+
347+
hash_deleted = 0
348+
schema_paths_deleted = 0
349+
bytes_freed = 0
350+
errors = 0
351+
352+
if not dry_run:
353+
# The size maps from the scan above are reused for the byte tally —
354+
# no second listing pass.
355+
for path in orphaned_hashes:
356+
try:
357+
if delete_path(path, self.store, config=self.config):
358+
hash_deleted += 1
359+
bytes_freed += hash_stored.get(path, 0)
360+
if verbose:
361+
logger.info(f"Deleted: {path} ({hash_stored.get(path, 0)} bytes)")
362+
except Exception as e:
363+
errors += 1
364+
logger.warning(f"Failed to delete {path}: {e}")
365+
366+
for path in orphaned_paths:
367+
try:
368+
if self.delete_schema_path(path):
369+
schema_paths_deleted += 1
370+
bytes_freed += schema_paths_stored.get(path, 0)
371+
if verbose:
372+
logger.info(f"Deleted schema path: {path} ({schema_paths_stored.get(path, 0)} bytes)")
373+
except Exception as e:
374+
errors += 1
375+
logger.warning(f"Failed to delete schema path {path}: {e}")
346376

347377
return {
348378
# Hash-addressed storage stats
349379
"hash_referenced": len(hash_referenced),
350380
"hash_stored": len(hash_stored),
351381
"hash_orphaned": len(orphaned_hashes),
352382
"hash_orphaned_bytes": sum(hash_stored.get(h, 0) for h in orphaned_hashes),
353-
"orphaned_hashes": sorted(orphaned_hashes),
383+
"orphaned_hashes": orphaned_hashes,
354384
# Schema-addressed storage stats
355385
"schema_paths_referenced": len(schema_paths_referenced),
356386
"schema_paths_stored": len(schema_paths_stored),
357387
"schema_paths_orphaned": len(orphaned_paths),
358388
"schema_paths_orphaned_bytes": sum(schema_paths_stored.get(p, 0) for p in orphaned_paths),
359-
"orphaned_paths": sorted(orphaned_paths),
360-
}
361-
362-
def collect(self, *schemas: "Schema", dry_run: bool = True, verbose: bool = False) -> dict[str, Any]:
363-
"""
364-
Remove orphaned storage items.
365-
366-
Scans the given schemas (via :meth:`scan`), then removes the orphans it
367-
identifies. Orphan identity comes entirely from :meth:`scan`, so live
368-
objects outside the current section (after a prefix change), other
369-
schemas' objects, and the ``filepath_prefix`` namespace are never
370-
deleted.
371-
372-
Returns a dict with: hash_deleted, schema_paths_deleted, deleted,
373-
bytes_freed (0 if dry_run), errors, dry_run, and the per-section orphan
374-
counts (hash_orphaned, schema_paths_orphaned).
375-
"""
376-
stats = self.scan(*schemas, verbose=verbose)
377-
378-
hash_deleted = 0
379-
schema_paths_deleted = 0
380-
bytes_freed = 0
381-
errors = 0
382-
383-
if not dry_run:
384-
if stats["hash_orphaned"] > 0:
385-
# Size map for bytes_freed, merged across the passed schemas.
386-
hash_stored: dict[str, int] = {}
387-
for schema in schemas:
388-
hash_stored.update(self.list_hash_paths(schema.database))
389-
for path in stats["orphaned_hashes"]:
390-
try:
391-
size = hash_stored.get(path, 0)
392-
if delete_path(path, self.store, config=self.config):
393-
hash_deleted += 1
394-
bytes_freed += size
395-
if verbose:
396-
logger.info(f"Deleted: {path} ({size} bytes)")
397-
except Exception as e:
398-
errors += 1
399-
logger.warning(f"Failed to delete {path}: {e}")
400-
401-
if stats["schema_paths_orphaned"] > 0:
402-
schema_paths_stored: dict[str, int] = {}
403-
for schema in schemas:
404-
schema_paths_stored.update(self.list_schema_paths(schema.database))
405-
for path in stats["orphaned_paths"]:
406-
try:
407-
size = schema_paths_stored.get(path, 0)
408-
if self.delete_schema_path(path):
409-
schema_paths_deleted += 1
410-
bytes_freed += size
411-
if verbose:
412-
logger.info(f"Deleted schema path: {path} ({size} bytes)")
413-
except Exception as e:
414-
errors += 1
415-
logger.warning(f"Failed to delete schema path {path}: {e}")
416-
417-
return {
389+
"orphaned_paths": orphaned_paths,
390+
# Deletion outcome (all zero when dry_run)
418391
"hash_deleted": hash_deleted,
419392
"schema_paths_deleted": schema_paths_deleted,
420393
"deleted": hash_deleted + schema_paths_deleted,
421394
"bytes_freed": bytes_freed,
422395
"errors": errors,
423396
"dry_run": dry_run,
424-
"hash_orphaned": stats["hash_orphaned"],
425-
"schema_paths_orphaned": stats["schema_paths_orphaned"],
426397
}

0 commit comments

Comments
 (0)