From 005fdd1efab8331e9ae9ce8e5cae6dfaf91278c3 Mon Sep 17 00:00:00 2001 From: MilagrosMarin Date: Fri, 17 Jul 2026 06:41:08 +0200 Subject: [PATCH] docs(codecs): fix stale storage-path and hash-algorithm docstrings Three built-in codec class docstrings described pre-2.3.1 storage layouts (and HashCodec claimed the wrong hash algorithm entirely): - HashCodec: said 'SHA256'; actually MD5 + Base32 (hash_registry.py:65-67). Layout said '_hash/{hash[:2]}/{hash[2:4]}/{hash}'; actually '{hash_prefix}/{schema}/{hash}' with hash_prefix defaulting to '_hash' and no [:2]/[:4] split anywhere in code. - NpyCodec: layout said '{schema}/{table}/{pk}/{attribute}.npy'; actually '{schema_prefix}/{schema}/{table}/{pk}/{attribute}_{token}.npy' after #1479 added the schema_prefix section and the random token suffix (via storage.build_object_path). - ObjectCodec: same drift as NpyCodec, as a directory rather than a file. All three docstrings had their 'Deletion:' line updated in v2.3.1 (dj.gc.collect() -> dj.gc.GarbageCollector) but the surrounding path claims were left stale. Docs-only. --- src/datajoint/builtin_codecs/hash.py | 10 ++++++---- src/datajoint/builtin_codecs/npy.py | 6 ++++-- src/datajoint/builtin_codecs/object.py | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/datajoint/builtin_codecs/hash.py b/src/datajoint/builtin_codecs/hash.py index 4873c073f..db9de7a3d 100644 --- a/src/datajoint/builtin_codecs/hash.py +++ b/src/datajoint/builtin_codecs/hash.py @@ -1,5 +1,5 @@ """ -Hash-addressed storage codec with SHA256 deduplication. +Hash-addressed storage codec with MD5-based deduplication. """ from __future__ import annotations @@ -12,11 +12,13 @@ class HashCodec(Codec): """ - Hash-addressed storage with SHA256 deduplication. + Hash-addressed storage with content-addressed deduplication. The ```` codec stores raw bytes using hash-addressed storage. - Data is identified by its SHA256 hash and stored in a hierarchical directory: - ``_hash/{hash[:2]}/{hash[2:4]}/{hash}`` + Data is identified by a 26-character Base32-encoded MD5 digest of the + content, and stored at ``{hash_prefix}/{schema}/{hash}`` (``hash_prefix`` + defaults to ``_hash``). Stores with subfolding configured insert + additional path segments: ``{hash_prefix}/{schema}/{fold1}/{fold2}/{hash}``. The database column stores JSON metadata: ``{hash, store, size}``. Duplicate content is automatically deduplicated across all tables. diff --git a/src/datajoint/builtin_codecs/npy.py b/src/datajoint/builtin_codecs/npy.py index 3094dfb90..d9315322b 100644 --- a/src/datajoint/builtin_codecs/npy.py +++ b/src/datajoint/builtin_codecs/npy.py @@ -226,7 +226,9 @@ class NpyCodec(SchemaCodec): Schema-addressed storage for numpy arrays as .npy files. The ```` codec stores numpy arrays as standard ``.npy`` files - using schema-addressed paths: ``{schema}/{table}/{pk}/{attribute}.npy``. + using schema-addressed paths: ``{schema_prefix}/{schema}/{table}/{pk}/{attribute}_{token}.npy`` + (``schema_prefix`` defaults to ``_schema``; ``{token}`` is a random + per-write suffix). Arrays are fetched lazily via ``NpyRef``, which provides metadata access without I/O and transparent numpy integration via ``__array__``. @@ -269,7 +271,7 @@ class Recording(dj.Manual): Storage Details: - File format: NumPy .npy (version 1.0 or 2.0) - - Path: ``{schema}/{table}/{pk}/{attribute}.npy`` + - Path: ``{schema_prefix}/{schema}/{table}/{pk}/{attribute}_{token}.npy`` - Database column: JSON with ``{path, store, dtype, shape}`` Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``. diff --git a/src/datajoint/builtin_codecs/object.py b/src/datajoint/builtin_codecs/object.py index 08ef0e188..edff4bede 100644 --- a/src/datajoint/builtin_codecs/object.py +++ b/src/datajoint/builtin_codecs/object.py @@ -15,7 +15,9 @@ class ObjectCodec(SchemaCodec): Schema-addressed storage for files and folders. The ```` codec provides managed file/folder storage using - schema-addressed paths: ``{schema}/{table}/{pk}/{field}/``. This creates + schema-addressed paths: ``{schema_prefix}/{schema}/{table}/{pk}/{field}_{token}/`` + (``schema_prefix`` defaults to ``_schema``; ``{token}`` is a random + per-write suffix). This creates a browsable organization in object storage that mirrors the database schema. Unlike hash-addressed storage (````), each row has its own unique path @@ -51,7 +53,7 @@ def make(self, key): Storage Structure: Objects are stored at:: - {store_root}/{schema}/{table}/{pk}/{field}/ + {store_root}/{schema_prefix}/{schema}/{table}/{pk}/{field}_{token}/ Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.