|
21 | 21 | Deletion: Requires garbage collection |
22 | 22 |
|
23 | 23 | Garbage collection is **store-specific** and handles one store at a time. The |
24 | | -:class:`GarbageCollector` is bound to a store (and the config that defines it) |
25 | | -at construction; its store and config are not threaded through each call:: |
| 24 | +:class:`GarbageCollector` is bound to its schemas and store at construction; |
| 25 | +neither is threaded through each call (the config defaults to the schemas'):: |
26 | 26 |
|
27 | 27 | import datajoint as dj |
28 | 28 |
|
29 | | - collector = dj.gc.GarbageCollector(store="mystore") |
30 | | - stats = collector.scan(schema1, schema2) # read-only |
31 | | - stats = collector.collect(schema1, schema2, dry_run=False) |
| 29 | + collector = dj.gc.GarbageCollector(schema1, schema2, store="mystore") |
| 30 | + stats = collector.collect() # read-only report (dry_run=True default) |
| 31 | + stats = collector.collect(dry_run=False) # actually delete the orphans |
32 | 32 |
|
33 | 33 | Both sections embed the schema name in every path, so garbage collection is |
34 | 34 | **per-schema**: each schema is scanned against its own subtree only. Orphan |
35 | | -detection is therefore confined to the schemas you pass — any subset of the |
36 | | -schemas sharing a store may be scanned/collected safely, and GC never touches |
37 | | -another schema's objects or user-managed content elsewhere in the store. |
| 35 | +detection is therefore confined to the collector's schemas — any subset of the |
| 36 | +schemas sharing a store may be collected safely, and GC never touches another |
| 37 | +schema's objects or user-managed content elsewhere in the store. |
38 | 38 |
|
39 | 39 | See Also |
40 | 40 | -------- |
@@ -87,65 +87,71 @@ def _is_covered(path: str, referenced: set[str]) -> bool: |
87 | 87 |
|
88 | 88 | class GarbageCollector: |
89 | 89 | """ |
90 | | - Store-specific garbage collector — handles exactly one store. |
| 90 | + Store-specific garbage collector — one store, a fixed set of schemas. |
91 | 91 |
|
92 | | - Bound to a store name and the config that defines it at construction, so |
93 | | - neither is threaded through individual operations. Storage-side work (the |
94 | | - stored-file listings, deletion) uses this store/config; database metadata |
95 | | - is read through each scanned schema's own connection. |
| 92 | + The schemas to collect and the store are bound at construction, so neither |
| 93 | + is threaded through individual operations. The store is resolved eagerly |
| 94 | + (an unknown/misconfigured store raises immediately). Storage-side work (the |
| 95 | + stored-file listings, deletion) uses this store; database metadata is read |
| 96 | + through each schema's own connection. |
96 | 97 |
|
97 | 98 | Parameters |
98 | 99 | ---------- |
| 100 | + *schemas : Schema |
| 101 | + The schemas to collect. At least one is required. Every managed path |
| 102 | + embeds its schema, so collection is per-schema and confined to these |
| 103 | + schemas: objects of any other schema sharing the store are never listed |
| 104 | + and never at risk — pass any subset safely. |
99 | 105 | store : str, optional |
100 | | - Store name (None = the default store). Resolved at construction — an |
101 | | - unknown/misconfigured store raises immediately. |
| 106 | + Store name (None = the default store), keyword-only. |
102 | 107 | config : Config, optional |
103 | | - Config that defines the store. If None, the global ``settings.config`` |
104 | | - is used. |
| 108 | + Config that defines the store. Defaults to the first schema's |
| 109 | + connection config (``schemas[0].connection._config``). |
105 | 110 | """ |
106 | 111 |
|
107 | | - def __init__(self, store: str | None = None, *, config=None) -> None: |
108 | | - if config is None: |
109 | | - from .settings import config as _global_config |
110 | | - |
111 | | - config = _global_config |
| 112 | + def __init__(self, *schemas: "Schema", store: str | None = None, config=None) -> None: |
| 113 | + if not schemas: |
| 114 | + raise DataJointError("At least one schema must be provided") |
| 115 | + self.schemas = schemas |
112 | 116 | self.store = store |
113 | | - self.config = config |
| 117 | + self.config = config if config is not None else schemas[0].connection._config |
114 | 118 | # Resolve the store eagerly: validates it exists and pins the backend |
115 | 119 | # and section prefixes for this collector's lifetime (one store only). |
116 | | - self.backend = get_store_backend(store, config=config) |
117 | | - spec = config.get_store_spec(store) |
| 120 | + self.backend = get_store_backend(store, config=self.config) |
| 121 | + spec = self.config.get_store_spec(store) |
118 | 122 | self._hash_prefix = spec["hash_prefix"].strip("/") # settings applies the "_hash" default |
119 | 123 | self._schema_prefix = spec["schema_prefix"].strip("/") # ... "_schema" default |
120 | 124 |
|
121 | 125 | # ------------------------------------------------------------------ # |
122 | 126 | # References — extracted from database metadata (per-schema connection) |
123 | 127 | # ------------------------------------------------------------------ # |
124 | | - def hash_references(self, *schemas: "Schema", verbose: bool = False) -> set[str]: |
| 128 | + def hash_references(self, verbose: bool = False) -> set[str]: |
125 | 129 | """ |
126 | | - Full relative store paths of hash-addressed objects referenced by live rows. |
| 130 | + Full relative store paths of hash-addressed objects referenced by live |
| 131 | + rows across this collector's schemas. |
127 | 132 |
|
128 | 133 | Reads columns classified by ``Heading.hash_objects`` (``<hash@>``, and |
129 | 134 | external ``<blob@>``/``<attach@>``). Paths are taken verbatim from each |
130 | 135 | row's metadata (independent of the store's current prefixes) and |
131 | 136 | filtered to this collector's store. |
132 | 137 | """ |
133 | | - return self._references("hash_objects", schemas, verbose) |
| 138 | + return self._references("hash_objects", verbose) |
134 | 139 |
|
135 | | - def schema_references(self, *schemas: "Schema", verbose: bool = False) -> set[str]: |
| 140 | + def schema_references(self, verbose: bool = False) -> set[str]: |
136 | 141 | """ |
137 | | - Full relative store paths of schema-addressed objects referenced by live rows. |
| 142 | + Full relative store paths of schema-addressed objects referenced by live |
| 143 | + rows across this collector's schemas. |
138 | 144 |
|
139 | 145 | Reads columns classified by ``Heading.schema_objects`` — any |
140 | 146 | ``SchemaCodec`` (``<object@>``, ``<npy@>``, custom subclasses, |
141 | 147 | recognized by type per #1469). For a directory-valued object the |
142 | 148 | recorded path is the directory prefix, not its individual files. |
143 | 149 | """ |
144 | | - return self._references("schema_objects", schemas, verbose) |
| 150 | + return self._references("schema_objects", verbose) |
145 | 151 |
|
146 | | - def _references(self, heading_attr: str, schemas, verbose: bool) -> set[str]: |
| 152 | + def _references(self, heading_attr: str, verbose: bool) -> set[str]: |
147 | 153 | referenced: set[str] = set() |
148 | | - for schema in schemas: |
| 154 | + for schema in self.schemas: |
149 | 155 | if verbose: |
150 | 156 | logger.info(f"Scanning schema {schema.database} for {heading_attr}") |
151 | 157 | for table_name in schema.list_tables(): |
@@ -294,133 +300,98 @@ def delete_schema_path(self, path: str) -> bool: |
294 | 300 | # ------------------------------------------------------------------ # |
295 | 301 | # Orchestration |
296 | 302 | # ------------------------------------------------------------------ # |
297 | | - def scan(self, *schemas: "Schema", verbose: bool = False) -> dict[str, Any]: |
| 303 | + def collect(self, dry_run: bool = True, verbose: bool = False) -> dict[str, Any]: |
298 | 304 | """ |
299 | | - Scan for orphaned storage items without deleting. |
| 305 | + Report — and, unless ``dry_run``, remove — orphaned storage. |
300 | 306 |
|
301 | | - Scans both hash-addressed and schema-addressed storage. Each schema is |
302 | | - scanned against ITS OWN subtree, so orphan detection is confined to the |
303 | | - schemas passed — objects of other schemas sharing the store are never |
304 | | - listed and never at risk; any subset may be passed safely. Objects |
305 | | - under a *former* prefix stay readable via their metadata paths but are |
306 | | - not reclamation candidates until the setting is restored. |
| 307 | + ``dry_run=True`` (the default) is a **read-only scan**: it reports what |
| 308 | + would be removed and deletes nothing. ``dry_run=False`` additionally |
| 309 | + deletes the orphans. The report is the same either way, so the safe |
| 310 | + default doubles as the inspection call. |
307 | 311 |
|
308 | | - Returns a dict with per-section statistics: |
| 312 | + Every managed path embeds its schema, so a schema's stored files can |
| 313 | + only match that schema's references — orphan detection is confined to |
| 314 | + this collector's schemas; objects of any other schema on the store are |
| 315 | + never listed, deleted, or at risk. Objects under a *former* prefix stay |
| 316 | + readable via their metadata paths but are not reclamation candidates |
| 317 | + until the setting is restored. |
| 318 | +
|
| 319 | + Returns a dict with per-section stats and the deletion outcome: |
309 | 320 |
|
310 | 321 | - hash_referenced / hash_stored / hash_orphaned / hash_orphaned_bytes |
311 | 322 | - orphaned_hashes: orphaned hash items as full relative store paths |
312 | 323 | (NOT bare hashes — passed as-is to the deleter) |
313 | 324 | - schema_paths_referenced / schema_paths_stored / schema_paths_orphaned |
314 | 325 | / schema_paths_orphaned_bytes |
315 | 326 | - orphaned_paths: orphaned schema files as full relative store paths |
| 327 | + - hash_deleted / schema_paths_deleted / deleted / bytes_freed (0 when |
| 328 | + dry_run) / errors / dry_run |
316 | 329 | """ |
317 | | - if not schemas: |
318 | | - raise DataJointError("At least one schema must be provided") |
| 330 | + # References can be gathered across all schemas at once: because paths |
| 331 | + # embed the schema, a file under schema X is only ever covered by an X |
| 332 | + # reference, so combined coverage equals per-schema coverage. |
| 333 | + hash_referenced = self.hash_references(verbose=verbose) |
| 334 | + schema_paths_referenced = self.schema_references(verbose=verbose) |
319 | 335 |
|
320 | | - hash_referenced: set[str] = set() |
321 | 336 | hash_stored: dict[str, int] = {} |
322 | | - orphaned_hashes: set[str] = set() |
323 | | - schema_paths_referenced: set[str] = set() |
324 | 337 | schema_paths_stored: dict[str, int] = {} |
325 | | - orphaned_paths: set[str] = set() |
326 | | - |
327 | | - for schema in schemas: |
328 | | - # --- Hash-addressed storage (this schema's subtree) --- |
329 | | - h_ref = self.hash_references(schema, verbose=verbose) |
330 | | - h_stored = self.list_hash_paths(schema.database) |
331 | | - orphaned_hashes |= set(h_stored.keys()) - h_ref |
332 | | - |
333 | | - # --- Schema-addressed storage (this schema's section) --- |
334 | | - s_ref = self.schema_references(schema, verbose=verbose) |
335 | | - s_stored = self.list_schema_paths(schema.database) |
336 | | - # Coverage, not exact set difference: a referenced path may be a |
337 | | - # directory-valued object (many files + a manifest). The walk is |
338 | | - # confined to this schema's section, so coverage against s_ref alone |
339 | | - # is complete (no hash objects can appear here). |
340 | | - orphaned_paths |= {p for p in s_stored if not _is_covered(p, s_ref)} |
341 | | - |
342 | | - hash_referenced |= h_ref |
343 | | - hash_stored.update(h_stored) |
344 | | - schema_paths_referenced |= s_ref |
345 | | - schema_paths_stored.update(s_stored) |
| 338 | + for schema in self.schemas: |
| 339 | + hash_stored.update(self.list_hash_paths(schema.database)) |
| 340 | + schema_paths_stored.update(self.list_schema_paths(schema.database)) |
| 341 | + |
| 342 | + orphaned_hashes = sorted(set(hash_stored.keys()) - hash_referenced) |
| 343 | + # Coverage, not exact set difference: a referenced path may be a |
| 344 | + # directory-valued object (many files + a manifest sidecar). |
| 345 | + orphaned_paths = sorted(p for p in schema_paths_stored if not _is_covered(p, schema_paths_referenced)) |
| 346 | + |
| 347 | + hash_deleted = 0 |
| 348 | + schema_paths_deleted = 0 |
| 349 | + bytes_freed = 0 |
| 350 | + errors = 0 |
| 351 | + |
| 352 | + if not dry_run: |
| 353 | + # The size maps from the scan above are reused for the byte tally — |
| 354 | + # no second listing pass. |
| 355 | + for path in orphaned_hashes: |
| 356 | + try: |
| 357 | + if delete_path(path, self.store, config=self.config): |
| 358 | + hash_deleted += 1 |
| 359 | + bytes_freed += hash_stored.get(path, 0) |
| 360 | + if verbose: |
| 361 | + logger.info(f"Deleted: {path} ({hash_stored.get(path, 0)} bytes)") |
| 362 | + except Exception as e: |
| 363 | + errors += 1 |
| 364 | + logger.warning(f"Failed to delete {path}: {e}") |
| 365 | + |
| 366 | + for path in orphaned_paths: |
| 367 | + try: |
| 368 | + if self.delete_schema_path(path): |
| 369 | + schema_paths_deleted += 1 |
| 370 | + bytes_freed += schema_paths_stored.get(path, 0) |
| 371 | + if verbose: |
| 372 | + logger.info(f"Deleted schema path: {path} ({schema_paths_stored.get(path, 0)} bytes)") |
| 373 | + except Exception as e: |
| 374 | + errors += 1 |
| 375 | + logger.warning(f"Failed to delete schema path {path}: {e}") |
346 | 376 |
|
347 | 377 | return { |
348 | 378 | # Hash-addressed storage stats |
349 | 379 | "hash_referenced": len(hash_referenced), |
350 | 380 | "hash_stored": len(hash_stored), |
351 | 381 | "hash_orphaned": len(orphaned_hashes), |
352 | 382 | "hash_orphaned_bytes": sum(hash_stored.get(h, 0) for h in orphaned_hashes), |
353 | | - "orphaned_hashes": sorted(orphaned_hashes), |
| 383 | + "orphaned_hashes": orphaned_hashes, |
354 | 384 | # Schema-addressed storage stats |
355 | 385 | "schema_paths_referenced": len(schema_paths_referenced), |
356 | 386 | "schema_paths_stored": len(schema_paths_stored), |
357 | 387 | "schema_paths_orphaned": len(orphaned_paths), |
358 | 388 | "schema_paths_orphaned_bytes": sum(schema_paths_stored.get(p, 0) for p in orphaned_paths), |
359 | | - "orphaned_paths": sorted(orphaned_paths), |
360 | | - } |
361 | | - |
362 | | - def collect(self, *schemas: "Schema", dry_run: bool = True, verbose: bool = False) -> dict[str, Any]: |
363 | | - """ |
364 | | - Remove orphaned storage items. |
365 | | -
|
366 | | - Scans the given schemas (via :meth:`scan`), then removes the orphans it |
367 | | - identifies. Orphan identity comes entirely from :meth:`scan`, so live |
368 | | - objects outside the current section (after a prefix change), other |
369 | | - schemas' objects, and the ``filepath_prefix`` namespace are never |
370 | | - deleted. |
371 | | -
|
372 | | - Returns a dict with: hash_deleted, schema_paths_deleted, deleted, |
373 | | - bytes_freed (0 if dry_run), errors, dry_run, and the per-section orphan |
374 | | - counts (hash_orphaned, schema_paths_orphaned). |
375 | | - """ |
376 | | - stats = self.scan(*schemas, verbose=verbose) |
377 | | - |
378 | | - hash_deleted = 0 |
379 | | - schema_paths_deleted = 0 |
380 | | - bytes_freed = 0 |
381 | | - errors = 0 |
382 | | - |
383 | | - if not dry_run: |
384 | | - if stats["hash_orphaned"] > 0: |
385 | | - # Size map for bytes_freed, merged across the passed schemas. |
386 | | - hash_stored: dict[str, int] = {} |
387 | | - for schema in schemas: |
388 | | - hash_stored.update(self.list_hash_paths(schema.database)) |
389 | | - for path in stats["orphaned_hashes"]: |
390 | | - try: |
391 | | - size = hash_stored.get(path, 0) |
392 | | - if delete_path(path, self.store, config=self.config): |
393 | | - hash_deleted += 1 |
394 | | - bytes_freed += size |
395 | | - if verbose: |
396 | | - logger.info(f"Deleted: {path} ({size} bytes)") |
397 | | - except Exception as e: |
398 | | - errors += 1 |
399 | | - logger.warning(f"Failed to delete {path}: {e}") |
400 | | - |
401 | | - if stats["schema_paths_orphaned"] > 0: |
402 | | - schema_paths_stored: dict[str, int] = {} |
403 | | - for schema in schemas: |
404 | | - schema_paths_stored.update(self.list_schema_paths(schema.database)) |
405 | | - for path in stats["orphaned_paths"]: |
406 | | - try: |
407 | | - size = schema_paths_stored.get(path, 0) |
408 | | - if self.delete_schema_path(path): |
409 | | - schema_paths_deleted += 1 |
410 | | - bytes_freed += size |
411 | | - if verbose: |
412 | | - logger.info(f"Deleted schema path: {path} ({size} bytes)") |
413 | | - except Exception as e: |
414 | | - errors += 1 |
415 | | - logger.warning(f"Failed to delete schema path {path}: {e}") |
416 | | - |
417 | | - return { |
| 389 | + "orphaned_paths": orphaned_paths, |
| 390 | + # Deletion outcome (all zero when dry_run) |
418 | 391 | "hash_deleted": hash_deleted, |
419 | 392 | "schema_paths_deleted": schema_paths_deleted, |
420 | 393 | "deleted": hash_deleted + schema_paths_deleted, |
421 | 394 | "bytes_freed": bytes_freed, |
422 | 395 | "errors": errors, |
423 | 396 | "dry_run": dry_run, |
424 | | - "hash_orphaned": stats["hash_orphaned"], |
425 | | - "schema_paths_orphaned": stats["schema_paths_orphaned"], |
426 | 397 | } |
0 commit comments