From f70b28bfc2e0180b6c729885167ef6918ccb3df3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 02:48:20 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(storage):=20governed=20download=20for?= =?UTF-8?q?=20field-owned=20files=20=E2=80=94=20ADR-0104=20D3=20wave=202?= =?UTF-8?q?=20(PR-4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file owned by a record's field (sys_file.ref_object / ref_id, set by PR-3) is now authorized on download the same way an attachment is: the caller must be able to READ the file's parent record, or be its uploader. Previously only attachments-scope files were gated and every field file kept an anonymous capability URL. Parent resolution differs by surface, and that asymmetry is the point of the ownership model. An attachment may hang off many records, so its readable-by set is the union over its sys_attachment join rows. A field-owned file belongs to exactly one record, so its readable-by set is that one record's and nothing more. Under a shared reference model the field case would have had to union too — which is exactly how copying a file id into a more public record would silently widen access. Denials report FILE_DOWNLOAD_DENIED, distinct from the attachments path's ATTACHMENT_DOWNLOAD_DENIED: the file BELONGS TO one record rather than being ATTACHED TO several, and the message should not claim otherwise. acl: 'public_read' is the opt-out, and now an explicit declaration rather than the silent default every field file used to get. Genuinely public images — anything embedded in an , which cannot carry a bearer token — must declare it. Gates nothing that is open today. A pre-cutover field holds an inline blob or an external URL, never a sys_file id, so no existing file has an owner recorded and none of them start being gated; the gate engages only for files a field has actually claimed, and disengages when ownership is released. Both are covered by regression tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SHpGw3GBA9aFpfwVArRWfd --- .../adr-0104-d3w2-pr4-governed-download.md | 33 +++++++++ .../src/storage-routes.test.ts | 71 ++++++++++++++++++- .../service-storage/src/storage-routes.ts | 57 ++++++++++----- .../src/storage-service-plugin.ts | 36 ++++++++-- 4 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 .changeset/adr-0104-d3w2-pr4-governed-download.md diff --git a/.changeset/adr-0104-d3w2-pr4-governed-download.md b/.changeset/adr-0104-d3w2-pr4-governed-download.md new file mode 100644 index 0000000000..1c68413b3a --- /dev/null +++ b/.changeset/adr-0104-d3w2-pr4-governed-download.md @@ -0,0 +1,33 @@ +--- +"@objectstack/service-storage": minor +--- + +feat(storage): governed download for field-owned files — ADR-0104 D3 wave 2 (PR-4) + +A file owned by a record's field (`sys_file.ref_object` / `ref_id`, set by +PR-3) is now authorized on download the same way an attachment is: the caller +must be able to READ the file's parent record, or be its uploader. Previously +only `attachments`-scope files were gated and every field file kept an +anonymous capability URL. + +**Parent resolution differs by surface, and that asymmetry is the point.** An +attachment may hang off many records, so its readable-by set is the union over +its `sys_attachment` join rows. A field-owned file belongs to exactly one +record, so its readable-by set is that one record's — nothing more. Under a +shared reference model the field case would have had to union too, which is +what makes copying a file id into a more public record silently widen access. + +Denials are reported as `FILE_DOWNLOAD_DENIED` (403), distinct from the +attachments path's `ATTACHMENT_DOWNLOAD_DENIED`, since the file *belongs to* one +record rather than being *attached to* several. + +**`acl: 'public_read'` is the opt-out**, and now an explicit declaration rather +than the silent default every field file used to get. Genuinely public images — +anything embedded in an ``, which cannot carry a bearer token — must +declare it. + +**Dual-mode safe, gates nothing that is open today.** A pre-cutover field holds +an inline blob or an external URL, never a `sys_file` id, so no existing file +has an owner recorded and none of them start being gated. The gate engages only +for files a record's field has actually claimed, and disengages again when +ownership is released. diff --git a/packages/services/service-storage/src/storage-routes.test.ts b/packages/services/service-storage/src/storage-routes.test.ts index ebd140f3f1..e63ddf923e 100644 --- a/packages/services/service-storage/src/storage-routes.test.ts +++ b/packages/services/service-storage/src/storage-routes.test.ts @@ -320,7 +320,7 @@ describe('Storage REST Routes', () => { expect(authorizeFileRead).not.toHaveBeenCalled(); }); - it('never gates a non-attachments file (avatars / field files stay open)', async () => { + it('never gates a file with neither an attachments scope nor a field owner', async () => { const { server, store: s, authorizeFileRead } = serverWith('deny'); await commit(s, { id: 'a5', scope: 'user', key: 'user/a5.png' }); const res = await hit(server, '/api/v1/storage/files/:fileId', 'a5'); @@ -334,6 +334,75 @@ describe('Storage REST Routes', () => { const res = await hit(server, '/api/v1/storage/files/:fileId/url', 'a6'); expect(res._status).toBe(200); }); + + // ── Field-owned files (ADR-0104 D3 wave 2) ───────────────────── + describe('field-owned files', () => { + const owned = (id: string, over: any = {}) => ({ + id, + scope: 'user', + key: `user/${id}.png`, + ref_object: 'product', + ref_id: 'p1', + ref_field: 'image', + ...over, + }); + + it('gates a field-owned file even though its scope is not attachments', async () => { + const { server, store: s, authorizeFileRead } = serverWith('deny'); + await commit(s, owned('fo1')); + const res = await hit(server, '/api/v1/storage/files/:fileId/url', 'fo1'); + expect(res._status).toBe(403); + // Distinct from the attachments code — this file BELONGS to one + // record, it is not attached to several. + expect(res._json?.code).toBe('FILE_DOWNLOAD_DENIED'); + expect(authorizeFileRead).toHaveBeenCalledOnce(); + }); + + it('401s an unauthenticated field-owned download', async () => { + const { server, store: s } = serverWith('unauthenticated'); + await commit(s, owned('fo2')); + const res = await hit(server, '/api/v1/storage/files/:fileId', 'fo2'); + expect(res._status).toBe(401); + expect(res._json?.code).toBe('AUTH_REQUIRED'); + }); + + it('allows an authorized field-owned download', async () => { + const { server, store: s } = serverWith('allow', { downloadTtl: 120 }); + await commit(s, owned('fo3')); + const res = await hit(server, '/api/v1/storage/files/:fileId/url', 'fo3'); + expect(res._status).toBe(200); + }); + + it('opts out via acl: public_read (the embedding escape hatch)', async () => { + const { server, store: s, authorizeFileRead } = serverWith('deny'); + await commit(s, owned('fo4', { acl: 'public_read' })); + const res = await hit(server, '/api/v1/storage/files/:fileId/url', 'fo4'); + expect(res._status).toBe(200); + expect(authorizeFileRead).not.toHaveBeenCalled(); + }); + + /** + * DUAL-MODE REGRESSION. A pre-cutover field holds an inline blob or an + * external URL, never a sys_file id — so no legacy file is ever claimed, + * `ref_object` stays unset, and this change gates nothing that used to + * be open. An unclaimed upload behaves the same way. + */ + it('does not gate an unclaimed file (no ref_object) — legacy fields keep working', async () => { + const { server, store: s, authorizeFileRead } = serverWith('deny'); + await commit(s, { id: 'fo5', scope: 'user', key: 'user/fo5.png' }); + const res = await hit(server, '/api/v1/storage/files/:fileId', 'fo5'); + expect(res._status).toBe(302); + expect(authorizeFileRead).not.toHaveBeenCalled(); + }); + + it('does not gate a file whose ownership was released', async () => { + const { server, store: s, authorizeFileRead } = serverWith('deny'); + await commit(s, owned('fo6', { ref_object: null, ref_id: null, ref_field: null })); + const res = await hit(server, '/api/v1/storage/files/:fileId', 'fo6'); + expect(res._status).toBe(302); + expect(authorizeFileRead).not.toHaveBeenCalled(); + }); + }); }); describe('PUT/GET /_local/raw/:token', () => { diff --git a/packages/services/service-storage/src/storage-routes.ts b/packages/services/service-storage/src/storage-routes.ts index 77745b8799..f9a66db99f 100644 --- a/packages/services/service-storage/src/storage-routes.ts +++ b/packages/services/service-storage/src/storage-routes.ts @@ -27,17 +27,20 @@ export interface StorageRoutesOptions { * is a tracked follow-up needing cookie sessions or signed links). */ resolveSession?: (req: IHttpRequest) => Promise<{ userId?: string } | null | undefined>; - /** - * Authorize a DOWNLOAD of an `attachments`-scope file (#2970 item 2). When - * wired, the download endpoints (`/files/:fileId` and `/files/:fileId/url`) - * consult this for `scope==='attachments'`, non-`public_read` files only: + /** + * Authorize a DOWNLOAD of a parent-governed file (#2970 item 2, extended by + * ADR-0104 D3 wave 2). When wired, the download endpoints + * (`/files/:fileId` and `/files/:fileId/url`) consult this for + * `scope==='attachments'` files AND for field-owned files (`ref_object` set), + * excluding `public_read` ones: * - `unauthenticated` → 401 (no session) - * - `deny` → 403 (session, but cannot read a parent record the file is - * attached to and is not the owner) + * - `deny` → 403 (session, but cannot read the parent record the file + * belongs to / is attached to, and is not the owner) * - `allow` → a short-lived signed URL is issued - * Non-attachments files (field files, avatars, org logos) keep the stable - * anonymous capability URL — they are embedded in `` which cannot - * carry a bearer token, and are out of scope for the attachments leak. + * A file with neither an attachments scope nor a field owner — an unclaimed + * upload, an org logo — keeps the stable anonymous capability URL, as does + * any file explicitly marked `acl: 'public_read'` (the opt-in for genuinely + * public embedding, since `` cannot carry a bearer token). * When absent (bare kernels, tests), all downloads stay open (back-compat). */ authorizeFileRead?: (file: FileRecord, req: IHttpRequest) => Promise; @@ -80,9 +83,20 @@ export function registerStorageRoutes( const sessionTtl = opts.sessionTtl ?? 86400; const downloadTtl = opts.downloadTtl ?? 300; - // ── Download authorization gate (#2970 item 2) ─────────────────────── - // Only `attachments`-scope, non-public files are gated; everything else - // keeps the stable anonymous capability URL (image/avatar embedding). + // ── Download authorization gate (#2970 item 2, ADR-0104 D3 wave 2) ─── + // Two kinds of file are gated, both deriving access from a PARENT record: + // - `attachments`-scope files, via their sys_attachment join rows; + // - field-owned files, via the single record whose field owns them + // (`ref_object`/`ref_id`, ADR-0104 D3 wave 2). + // `acl: 'public_read'` opts a file back out to the stable anonymous + // capability URL — needed for genuinely public embedding (`` + // cannot carry a bearer token), and now an explicit declaration rather + // than the silent default it used to be for every field file. + // + // Dual-mode safe: a legacy field holds an inline blob or an external URL, + // never a `sys_file` id, so no legacy file has `ref_object` set and none of + // them start being gated by this change. + // // Returns the signed-URL TTL to use, or `false` if a response was already // sent (401/403) and the handler must stop. const authorizeDownload = async ( @@ -90,7 +104,9 @@ export function registerStorageRoutes( req: IHttpRequest, res: IHttpResponse, ): Promise => { - if (file.scope !== 'attachments' || file.acl === 'public_read' || !opts.authorizeFileRead) { + const fieldOwned = !!file.ref_object && file.ref_id != null && file.ref_id !== ''; + const gated = file.scope === 'attachments' || fieldOwned; + if (!gated || file.acl === 'public_read' || !opts.authorizeFileRead) { return presignedTtl; } let verdict: FileReadVerdict; @@ -104,10 +120,17 @@ export function registerStorageRoutes( return false; } if (verdict === 'deny') { - res.status(403).json({ - error: 'You do not have access to a record this file is attached to', - code: 'ATTACHMENT_DOWNLOAD_DENIED', - }); + res.status(403).json( + fieldOwned + ? { + error: 'You do not have access to the record this file belongs to', + code: 'FILE_DOWNLOAD_DENIED', + } + : { + error: 'You do not have access to a record this file is attached to', + code: 'ATTACHMENT_DOWNLOAD_DENIED', + }, + ); return false; } return downloadTtl; diff --git a/packages/services/service-storage/src/storage-service-plugin.ts b/packages/services/service-storage/src/storage-service-plugin.ts index d397bbdc05..79af9ff8c3 100644 --- a/packages/services/service-storage/src/storage-service-plugin.ts +++ b/packages/services/service-storage/src/storage-service-plugin.ts @@ -463,12 +463,20 @@ function buildAuthSessionResolver( } /** - * Authorize an `attachments`-scope download (#2970 item 2). Builds the FULL - * caller ExecutionContext via `resolveAuthzContext` (the same shared resolver - * rest-server uses — a bare `{ userId }` would lack the resolved permissions - * the parent RLS needs), then allows when the caller is the file's owner or - * can READ a record the file is attached to. Returns `undefined` (routes stay - * open) when the auth service or engine is absent. + * Authorize a parent-governed download (#2970 item 2; extended for field-owned + * files by ADR-0104 D3 wave 2). Builds the FULL caller ExecutionContext via + * `resolveAuthzContext` (the same shared resolver rest-server uses — a bare + * `{ userId }` would lack the resolved permissions the parent RLS needs), then + * allows when the caller is the file's owner or can READ the file's parent + * record. Returns `undefined` (routes stay open) when the auth service or + * engine is absent. + * + * "Parent" resolves differently for the two surfaces, and that asymmetry is the + * point of the ownership model: an attachment may hang off MANY records, so its + * readable-by set is the union over its join rows; a field-owned file belongs + * to exactly ONE record, so its readable-by set is that record's and nothing + * more. A shared model would have had to union field references too, silently + * widening access whenever one file id was copied into a more public record. */ function buildFileReadAuthorizer( ctx: PluginContext, @@ -487,6 +495,22 @@ function buildFileReadAuthorizer( // Uploader / owner may always download. if (file.owner_id && String(file.owner_id) === String(authz.userId)) return 'allow'; + // Field-owned (ADR-0104 D3 wave 2): exactly ONE record's field holds + // this file, so access is that record's READ access — never a union. + if (file.ref_object && file.ref_id != null && file.ref_id !== '') { + try { + const visible = (await (engine as any).find(String(file.ref_object), { + where: { id: file.ref_id }, + fields: ['id'], + limit: 1, + context: authz, + })) as Array>; + return visible?.length ? 'allow' : 'deny'; + } catch { + return 'deny'; // unknown/failing owner object — fail closed + } + } + // Otherwise: readable via any parent record this file is attached to. const links = (await (engine as any).find('sys_attachment', { where: { file_id: file.id }, From 6295f02407c86550fedb46a6a7625e91eb330563 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 02:52:26 +0000 Subject: [PATCH 2/2] =?UTF-8?q?feat(storage):=20verifyFileReferences=20?= =?UTF-8?q?=E2=80=94=20the=20executable=20R4=20acceptance=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0104 D3 wave 2 planned to gate enabling collection on "reference counting is verified", which is aspirational rather than checkable. This makes it runnable. verifyFileReferences() compares GROUND TRUTH (what records' file-class fields actually hold) against RECORDED OWNERSHIP (sys_file.ref_object/ref_id/ref_field) and classifies every disagreement by whether it can cause DATA LOSS once collection is on, because only that class should block a release: blocking unowned_reference a record holds a file nothing owns — with collection on it looks free and its bytes go foreign_owner a record holds a file owned by a different slot; that reference is invisible to the lifecycle, so the bytes go when the recorded owner releases shared_reference one file held by two slots — exclusivity was violated, so copy-on-claim did not run, and one file's ACL is serving two parents advisory stale_owner owned but no longer held; fails toward retention unreferenced_file committed and unpointed-at — storage cost, not risk The scan is read-only: it never writes, tombstones or deletes. The point is that a ledger may not be handed authority over an irreversible delete until it has been shown to agree with reality, so this is meant to be run repeatedly, on real tenant data, and to report zero blocking discrepancies on consecutive runs before the gated collection change may merge. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SHpGw3GBA9aFpfwVArRWfd --- .../adr-0104-d3w2-pr4-governed-download.md | 19 + .../services/service-storage/src/index.ts | 12 + .../src/verify-file-references.test.ts | 237 +++++++++++ .../src/verify-file-references.ts | 367 ++++++++++++++++++ 4 files changed, 635 insertions(+) create mode 100644 packages/services/service-storage/src/verify-file-references.test.ts create mode 100644 packages/services/service-storage/src/verify-file-references.ts diff --git a/.changeset/adr-0104-d3w2-pr4-governed-download.md b/.changeset/adr-0104-d3w2-pr4-governed-download.md index 1c68413b3a..4b77cfc56f 100644 --- a/.changeset/adr-0104-d3w2-pr4-governed-download.md +++ b/.changeset/adr-0104-d3w2-pr4-governed-download.md @@ -31,3 +31,22 @@ an inline blob or an external URL, never a `sys_file` id, so no existing file has an owner recorded and none of them start being gated. The gate engages only for files a record's field has actually claimed, and disengages again when ownership is released. + +--- + +Also adds `verifyFileReferences()` — the executable form of ADR-0104's R4 +acceptance gate. It compares ground truth (what records' file fields actually +hold) against recorded ownership, and classifies disagreements by whether they +could cause data loss once collection is enabled: + +- **blocking** — `unowned_reference` (a held file nothing owns), `foreign_owner` + (a record holds a file owned by another slot), `shared_reference` (one file + held by two slots, i.e. exclusivity was violated). Each would let a later reap + delete bytes a record still points at. +- **advisory** — `stale_owner` (owned but no longer held; fails toward + retention) and `unreferenced_file` (storage cost, not a correctness problem). + +The scan is read-only — it never writes, tombstones, or deletes. A ledger may +not be given authority over irreversible deletes until it has been shown to +agree with reality, so this must report zero blocking discrepancies on real +tenant data, on consecutive runs, before the gated collection change may merge. diff --git a/packages/services/service-storage/src/index.ts b/packages/services/service-storage/src/index.ts index 49e18a808a..b599af5bbb 100644 --- a/packages/services/service-storage/src/index.ts +++ b/packages/services/service-storage/src/index.ts @@ -16,5 +16,17 @@ export { installAttachmentLifecycleHooks, createSysFileReapGuard, createUploadSe export type { AttachmentLifecycleEngine, AttachmentLifecycleLogger } from './attachment-lifecycle.js'; export { installFileReferenceHooks, FileReferenceCopyError } from './file-reference-lifecycle.js'; export type { FileReferenceEngine, FileReferenceLogger } from './file-reference-lifecycle.js'; +export { + verifyFileReferences, + formatFileReferenceReport, + BLOCKING_ISSUE_KINDS, +} from './verify-file-references.js'; +export type { + FileReferenceIssue, + FileReferenceIssueKind, + FileReferenceReport, + VerifyReferencesEngine, + VerifyReferencesOptions, +} from './verify-file-references.js'; export { installAttachmentAccessHooks, installAttachmentReadVisibility } from './attachment-access-hooks.js'; export type { AttachmentSharingLike } from './attachment-access-hooks.js'; diff --git a/packages/services/service-storage/src/verify-file-references.test.ts b/packages/services/service-storage/src/verify-file-references.test.ts new file mode 100644 index 0000000000..76d5be555b --- /dev/null +++ b/packages/services/service-storage/src/verify-file-references.test.ts @@ -0,0 +1,237 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect } from 'vitest'; +import { + verifyFileReferences, + formatFileReferenceReport, + BLOCKING_ISSUE_KINDS, + type VerifyReferencesEngine, +} from './verify-file-references.js'; + +const REGISTRY: Record = { + sys_file: { fields: { id: { type: 'text' } } }, + product: { + fields: { + id: { type: 'text' }, + name: { type: 'text' }, + image: { type: 'image' }, + gallery: { type: 'image', multiple: true }, + }, + }, + tag: { fields: { id: { type: 'text' }, label: { type: 'text' } } }, +}; + +function fakeEngine(tables: Record>>): VerifyReferencesEngine { + const matches = (row: Record, where: Record) => + Object.entries(where).every(([k, v]) => { + if (v && typeof v === 'object' && '$ne' in (v as any)) return row[k] !== (v as any).$ne; + return row[k] === v; + }); + return { + getObject: (name) => REGISTRY[name], + getConfigs: () => REGISTRY, + async find(object, options: any) { + const rows = (tables[object] ?? []).filter((r) => matches(r, options?.where ?? {})); + const start = typeof options?.offset === 'number' ? options.offset : 0; + const end = typeof options?.limit === 'number' ? start + options.limit : undefined; + return rows.slice(start, end); + }, + }; +} + +const file = (id: string, owner?: { object: string; recordId: string; field: string }) => ({ + id, + status: 'committed', + scope: 'user', + ref_object: owner?.object ?? null, + ref_id: owner?.recordId ?? null, + ref_field: owner?.field ?? null, +}); + +describe('verifyFileReferences (ADR-0104 D3 wave 2 — R4 gate)', () => { + it('reports a clean bill when ownership matches what records hold', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'file_a' }], + sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })], + }); + + const report = await verifyFileReferences(engine); + + expect(report.ok).toBe(true); + expect(report.blocking).toBe(0); + expect(report.issues).toEqual([]); + expect(report.scannedObjects).toEqual(['product']); + expect(report.heldReferences).toBe(1); + expect(report.ownedFiles).toBe(1); + expect(formatFileReferenceReport(report)).toContain('No discrepancies'); + }); + + it('only scans objects that declare a file-class field', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1' }], + tag: [{ id: 't1', label: 'x' }], + sys_file: [], + }); + + const report = await verifyFileReferences(engine); + + expect(report.scannedObjects).toEqual(['product']); + expect(report.scannedRecords).toBe(1); + }); + + // ── Blocking: these would delete bytes something still holds ────── + it('BLOCKS on a held file with no recorded owner', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'file_a' }], + sys_file: [file('file_a')], // uploaded, never claimed + }); + + const report = await verifyFileReferences(engine); + + expect(report.ok).toBe(false); + expect(report.counts.unowned_reference).toBe(1); + expect(report.issues[0]).toMatchObject({ + kind: 'unowned_reference', + fileId: 'file_a', + object: 'product', + recordId: 'p1', + field: 'image', + }); + expect(formatFileReferenceReport(report)).toContain('BLOCKING'); + }); + + it('BLOCKS when a record holds a file owned by a different slot', async () => { + const engine = fakeEngine({ + product: [ + { id: 'p1', image: 'file_a' }, + { id: 'p2', image: 'file_a' }, + ], + sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })], + }); + + const report = await verifyFileReferences(engine); + + expect(report.ok).toBe(false); + // p2's reference is invisible to the lifecycle: when p1 releases, the + // bytes go while p2 still points at them. + expect(report.counts.foreign_owner).toBe(1); + expect(report.issues.find((i) => i.kind === 'foreign_owner')).toMatchObject({ recordId: 'p2' }); + // …and the exclusivity violation itself is reported too. + expect(report.counts.shared_reference).toBe(1); + }); + + it('BLOCKS on a file held by two slots (exclusivity violated)', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'file_a', gallery: ['file_a'] }], + sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })], + }); + + const report = await verifyFileReferences(engine); + + expect(report.counts.shared_reference).toBe(1); + expect(report.ok).toBe(false); + }); + + // ── Advisory: safe directions ───────────────────────────────────── + it('reports a stale owner as advisory, not blocking', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1' }], // field cleared without the release landing + sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })], + }); + + const report = await verifyFileReferences(engine); + + expect(report.counts.stale_owner).toBe(1); + // Fails toward retention — the file is simply never collected. + expect(report.blocking).toBe(0); + expect(report.ok).toBe(true); + }); + + it('reports unreferenced committed files only when asked, and never as blocking', async () => { + const tables = { + product: [{ id: 'p1' }], + sys_file: [file('file_orphan')], + }; + + const without = await verifyFileReferences(fakeEngine(tables)); + expect(without.counts.unreferenced_file).toBe(0); + + const withSweep = await verifyFileReferences(fakeEngine(tables), { includeUnreferenced: true }); + expect(withSweep.counts.unreferenced_file).toBe(1); + expect(withSweep.ok).toBe(true); + }); + + it('ignores attachments-scope files, which sys_attachment governs', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1' }], + sys_file: [{ ...file('file_att'), scope: 'attachments' }], + }); + + const report = await verifyFileReferences(engine, { includeUnreferenced: true }); + + expect(report.counts.unreferenced_file).toBe(0); + }); + + // ── Dual-mode ───────────────────────────────────────────────────── + it('ignores inline blobs and URL values — a pre-cutover tenant reads clean', async () => { + const engine = fakeEngine({ + product: [ + { id: 'p1', image: { url: 'https://cdn.example.com/a.png', name: 'a.png' } }, + { id: 'p2', image: 'https://cdn.example.com/b.png' }, + { id: 'p3', image: 'data:image/svg+xml,' }, + ], + sys_file: [], + }); + + const report = await verifyFileReferences(engine); + + expect(report.heldReferences).toBe(0); + expect(report.issues).toEqual([]); + expect(report.ok).toBe(true); + }); + + // ── Mechanics ───────────────────────────────────────────────────── + it('pages through records rather than reading one unbounded page', async () => { + const many = Array.from({ length: 1200 }, (_, i) => ({ id: `p${i}`, image: `file_${i}` })); + const engine = fakeEngine({ + product: many, + sys_file: many.map((r, i) => file(`file_${i}`, { object: 'product', recordId: `p${i}`, field: 'image' })), + }); + + const report = await verifyFileReferences(engine); + + expect(report.scannedRecords).toBe(1200); + expect(report.ownedFiles).toBe(1200); + expect(report.ok).toBe(true); + }); + + it('marks the verdict truncated when a scan bound is hit', async () => { + const many = Array.from({ length: 1200 }, (_, i) => ({ id: `p${i}` })); + const engine = fakeEngine({ product: many, sys_file: [] }); + + const report = await verifyFileReferences(engine, { maxRecordsPerObject: 500 }); + + expect(report.truncated).toBe(true); + expect(formatFileReferenceReport(report)).toContain('truncated'); + }); + + it('can be scoped to specific objects', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'file_a' }], + sys_file: [file('file_a')], + }); + + const report = await verifyFileReferences(engine, { objects: ['tag'] }); + + expect(report.scannedObjects).toEqual([]); + expect(report.ok).toBe(true); + }); + + it('classifies exactly the data-loss kinds as blocking', () => { + expect([...BLOCKING_ISSUE_KINDS].sort()).toEqual([ + 'foreign_owner', + 'shared_reference', + 'unowned_reference', + ]); + }); +}); diff --git a/packages/services/service-storage/src/verify-file-references.ts b/packages/services/service-storage/src/verify-file-references.ts new file mode 100644 index 0000000000..a56a416595 --- /dev/null +++ b/packages/services/service-storage/src/verify-file-references.ts @@ -0,0 +1,367 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { FILE_REFERENCE_TYPES, isFileIdToken } from '@objectstack/spec/data'; + +/** + * File-reference reconciliation (ADR-0104 D3 wave 2) — the executable form of + * the ADR's R4 acceptance gate. + * + * Compares GROUND TRUTH (what records' file-class fields actually hold) against + * RECORDED OWNERSHIP (`sys_file.ref_object` / `ref_id` / `ref_field`). The + * ownership columns are what a later, gated change will let authorise deleting + * file bytes, and a ledger may not be given that power until it has been shown + * to agree with reality — so this exists to be run, repeatedly, before that + * change may merge. + * + * Discrepancies are split by whether they can cause DATA LOSS once collection + * is enabled, because only that class blocks: + * + * - `unowned_reference` (**blocking**) — a record holds a file that nothing + * owns. With collection on, the file looks free and its bytes go. + * - `foreign_owner` (**blocking**) — a record holds a file owned by a + * different slot. That reference is invisible to the lifecycle, so when the + * recorded owner releases, the bytes go while this record still points at + * them. + * - `shared_reference` (**blocking**) — one file held by two or more slots, + * i.e. exclusivity was violated and copy-on-claim did not run. Same failure + * as above, and it also means one file's ACL is serving two parents. + * - `stale_owner` (advisory) — a file is owned by a slot that no longer holds + * it. Fails toward retention: the file is simply never collected. + * - `unreferenced_file` (advisory) — a committed file nothing points at. + * Storage cost, not a correctness problem. + * + * The scan is read-only. It never writes, never tombstones, and never deletes. + */ + +const SYSTEM_CTX = { isSystem: true } as const; + +/** Records read per page while scanning an object. */ +const SCAN_PAGE_SIZE = 500; + +export type FileReferenceIssueKind = + | 'unowned_reference' + | 'foreign_owner' + | 'shared_reference' + | 'stale_owner' + | 'unreferenced_file'; + +/** Issue kinds that would let a later reap delete bytes something still holds. */ +export const BLOCKING_ISSUE_KINDS: ReadonlySet = new Set([ + 'unowned_reference', + 'foreign_owner', + 'shared_reference', +] as const); + +export interface FileReferenceIssue { + kind: FileReferenceIssueKind; + fileId: string; + /** The slot this issue is reported against, when it has one. */ + object?: string; + recordId?: string; + field?: string; + detail: string; +} + +export interface FileReferenceReport { + /** Objects that declare at least one file-class field. */ + scannedObjects: string[]; + scannedRecords: number; + /** Distinct (file, slot) references actually held by records. */ + heldReferences: number; + /** `sys_file` rows carrying an owner. */ + ownedFiles: number; + issues: FileReferenceIssue[]; + counts: Record; + blocking: number; + /** + * Zero blocking issues. The gate: this must hold on real tenant data, on + * consecutive runs, before collection may be enabled. + */ + ok: boolean; + /** True when a bound was hit, so `ok` covers only what was scanned. */ + truncated: boolean; +} + +/** Engine surface the scan needs — duck-typed, like the other storage seams. */ +export interface VerifyReferencesEngine { + getObject(name: string): any | undefined; + getConfigs?(): Record; + find(object: string, options: Record): Promise>>; +} + +export interface VerifyReferencesOptions { + /** Restrict the scan to these objects (default: every object with a file field). */ + objects?: string[]; + /** Safety bound on records read per object. Exceeding it sets `truncated`. */ + maxRecordsPerObject?: number; + /** Include the advisory `unreferenced_file` sweep (an extra full sys_file read). */ + includeUnreferenced?: boolean; +} + +function slotKey(object: string, recordId: string, field: string): string { + return `${object}${recordId}${field}`; +} + +function fileFieldsOf(engine: VerifyReferencesEngine, objectName: string): string[] { + const schema: any = engine.getObject(objectName); + if (!schema?.fields) return []; + return Object.entries(schema.fields) + .filter(([, def]: [string, any]) => def && FILE_REFERENCE_TYPES.has(def.type)) + .map(([name]) => name); +} + +function idTokensIn(value: unknown): string[] { + if (Array.isArray(value)) return value.filter((v) => isFileIdToken(v)) as string[]; + return isFileIdToken(value) ? [value] : []; +} + +/** + * Scan records and `sys_file` ownership, and report where they disagree. + */ +export async function verifyFileReferences( + engine: VerifyReferencesEngine, + options: VerifyReferencesOptions = {}, +): Promise { + const maxPerObject = options.maxRecordsPerObject ?? 100_000; + const issues: FileReferenceIssue[] = []; + let truncated = false; + + const candidates = + options.objects ?? + (typeof engine.getConfigs === 'function' ? Object.keys(engine.getConfigs()) : []); + const scannedObjects = candidates.filter( + (name) => name !== 'sys_file' && fileFieldsOf(engine, name).length > 0, + ); + + // ── 1. Ground truth: what do records actually hold? ──────────────── + /** fileId → the slots holding it. */ + const held = new Map>(); + let scannedRecords = 0; + + for (const object of scannedObjects) { + const fileFields = fileFieldsOf(engine, object); + let offset = 0; + for (;;) { + let page: Array>; + try { + page = await engine.find(object, { + fields: ['id', ...fileFields], + limit: SCAN_PAGE_SIZE, + offset, + context: { ...SYSTEM_CTX }, + }); + } catch { + break; // unreadable object — skip rather than abort the whole scan + } + if (!page || page.length === 0) break; + for (const record of page) { + const recordId = record?.id; + if (recordId == null) continue; + scannedRecords++; + for (const field of fileFields) { + for (const fileId of new Set(idTokensIn(record[field]))) { + const list = held.get(fileId) ?? []; + list.push({ object, recordId: String(recordId), field }); + held.set(fileId, list); + } + } + } + offset += page.length; + if (page.length < SCAN_PAGE_SIZE) break; + if (offset >= maxPerObject) { + truncated = true; + break; + } + } + } + + // ── 2. Recorded ownership ───────────────────────────────────────── + /** fileId → the slot recorded as its owner. */ + const owned = new Map(); + let ownerOffset = 0; + for (;;) { + let page: Array>; + try { + page = await engine.find('sys_file', { + where: { ref_object: { $ne: null } }, + fields: ['id', 'ref_object', 'ref_id', 'ref_field', 'status'], + limit: SCAN_PAGE_SIZE, + offset: ownerOffset, + context: { ...SYSTEM_CTX }, + }); + } catch { + break; + } + if (!page || page.length === 0) break; + for (const row of page) { + // A driver without `$ne` support may hand back unowned rows too — filter + // here rather than trusting the predicate round-tripped. + if (row?.id == null || !row.ref_object || row.ref_id == null) continue; + owned.set(String(row.id), { + object: String(row.ref_object), + recordId: String(row.ref_id), + field: String(row.ref_field ?? ''), + }); + } + ownerOffset += page.length; + if (page.length < SCAN_PAGE_SIZE) break; + } + + // ── 3. Diff ─────────────────────────────────────────────────────── + let heldReferences = 0; + for (const [fileId, slots] of held) { + heldReferences += slots.length; + + if (slots.length > 1) { + issues.push({ + kind: 'shared_reference', + fileId, + detail: + `held by ${slots.length} slots (` + + slots.map((s) => `${s.object}/${s.recordId}.${s.field}`).join(', ') + + ') — field references must be exclusive; copy-on-claim did not run', + }); + } + + const owner = owned.get(fileId); + if (!owner) { + const s = slots[0]; + issues.push({ + kind: 'unowned_reference', + fileId, + object: s.object, + recordId: s.recordId, + field: s.field, + detail: `held by ${s.object}/${s.recordId}.${s.field} but no sys_file owner is recorded`, + }); + continue; + } + for (const s of slots) { + if (slotKey(s.object, s.recordId, s.field) !== slotKey(owner.object, owner.recordId, owner.field)) { + issues.push({ + kind: 'foreign_owner', + fileId, + object: s.object, + recordId: s.recordId, + field: s.field, + detail: + `held by ${s.object}/${s.recordId}.${s.field} but owned by ` + + `${owner.object}/${owner.recordId}.${owner.field}`, + }); + } + } + } + + for (const [fileId, owner] of owned) { + const slots = held.get(fileId); + const stillHeld = slots?.some( + (s) => slotKey(s.object, s.recordId, s.field) === slotKey(owner.object, owner.recordId, owner.field), + ); + if (!stillHeld) { + issues.push({ + kind: 'stale_owner', + fileId, + object: owner.object, + recordId: owner.recordId, + field: owner.field, + detail: + `owned by ${owner.object}/${owner.recordId}.${owner.field}, which no longer holds it — ` + + 'retained rather than collected (safe)', + }); + } + } + + // ── 4. Advisory: committed files nothing points at ──────────────── + if (options.includeUnreferenced) { + let offset = 0; + for (;;) { + let page: Array>; + try { + page = await engine.find('sys_file', { + where: { status: 'committed' }, + fields: ['id', 'ref_object', 'scope'], + limit: SCAN_PAGE_SIZE, + offset, + context: { ...SYSTEM_CTX }, + }); + } catch { + break; + } + if (!page || page.length === 0) break; + for (const row of page) { + const id = row?.id == null ? null : String(row.id); + // attachments-scope files are governed by sys_attachment, not by the + // ownership columns — not this scan's business. + if (!id || row.ref_object || row.scope === 'attachments') continue; + if (held.has(id)) continue; + issues.push({ + kind: 'unreferenced_file', + fileId: id, + detail: 'committed but neither owned nor held by any field — storage cost, not data risk', + }); + } + offset += page.length; + if (page.length < SCAN_PAGE_SIZE) break; + } + } + + const counts = { + unowned_reference: 0, + foreign_owner: 0, + shared_reference: 0, + stale_owner: 0, + unreferenced_file: 0, + } as Record; + for (const issue of issues) counts[issue.kind]++; + const blocking = issues.filter((i) => BLOCKING_ISSUE_KINDS.has(i.kind)).length; + + return { + scannedObjects, + scannedRecords, + heldReferences, + ownedFiles: owned.size, + issues, + counts, + blocking, + ok: blocking === 0, + truncated, + }; +} + +/** Render a report as human-readable lines (CLI / CI log output). */ +export function formatFileReferenceReport(report: FileReferenceReport): string { + const lines: string[] = []; + lines.push( + `Scanned ${report.scannedRecords} record(s) across ${report.scannedObjects.length} object(s) ` + + `with file fields; ${report.heldReferences} reference(s) held, ${report.ownedFiles} file(s) owned.`, + ); + if (report.truncated) { + lines.push('⚠ Scan was truncated by maxRecordsPerObject — this verdict covers only what was read.'); + } + if (report.issues.length === 0) { + lines.push('✅ No discrepancies: recorded ownership matches what records hold.'); + return lines.join('\n'); + } + for (const kind of [ + 'unowned_reference', + 'foreign_owner', + 'shared_reference', + 'stale_owner', + 'unreferenced_file', + ] as FileReferenceIssueKind[]) { + const of = report.issues.filter((i) => i.kind === kind); + if (of.length === 0) continue; + lines.push( + `\n${BLOCKING_ISSUE_KINDS.has(kind) ? '❌ BLOCKING' : '• advisory'} — ${kind} (${of.length}):`, + ); + for (const i of of.slice(0, 20)) lines.push(` ${i.fileId}: ${i.detail}`); + if (of.length > 20) lines.push(` … and ${of.length - 20} more`); + } + lines.push( + report.ok + ? '\n✅ No blocking discrepancies. Advisory items above cost storage, not data.' + : `\n❌ ${report.blocking} blocking discrepancy(ies). Enabling collection now could delete ` + + 'bytes a record still references.', + ); + return lines.join('\n'); +}