diff --git a/.changeset/adr-0104-d3w2-pr4-governed-download.md b/.changeset/adr-0104-d3w2-pr4-governed-download.md new file mode 100644 index 0000000000..4b77cfc56f --- /dev/null +++ b/.changeset/adr-0104-d3w2-pr4-governed-download.md @@ -0,0 +1,52 @@ +--- +"@objectstack/service-storage": minor +--- + +feat(storage): governed download for field-owned files — ADR-0104 D3 wave 2 (PR-4) + +A file owned by a record's field (`sys_file.ref_object` / `ref_id`, set by +PR-3) is now authorized on download the same way an attachment is: the caller +must be able to READ the file's parent record, or be its uploader. Previously +only `attachments`-scope files were gated and every field file kept an +anonymous capability URL. + +**Parent resolution differs by surface, and that asymmetry is the point.** An +attachment may hang off many records, so its readable-by set is the union over +its `sys_attachment` join rows. A field-owned file belongs to exactly one +record, so its readable-by set is that one record's — nothing more. Under a +shared reference model the field case would have had to union too, which is +what makes copying a file id into a more public record silently widen access. + +Denials are reported as `FILE_DOWNLOAD_DENIED` (403), distinct from the +attachments path's `ATTACHMENT_DOWNLOAD_DENIED`, since the file *belongs to* one +record rather than being *attached to* several. + +**`acl: 'public_read'` is the opt-out**, and now an explicit declaration rather +than the silent default every field file used to get. Genuinely public images — +anything embedded in an ``, which cannot carry a bearer token — must +declare it. + +**Dual-mode safe, gates nothing that is open today.** A pre-cutover field holds +an inline blob or an external URL, never a `sys_file` id, so no existing file +has an owner recorded and none of them start being gated. The gate engages only +for files a record's field has actually claimed, and disengages again when +ownership is released. + +--- + +Also adds `verifyFileReferences()` — the executable form of ADR-0104's R4 +acceptance gate. It compares ground truth (what records' file fields actually +hold) against recorded ownership, and classifies disagreements by whether they +could cause data loss once collection is enabled: + +- **blocking** — `unowned_reference` (a held file nothing owns), `foreign_owner` + (a record holds a file owned by another slot), `shared_reference` (one file + held by two slots, i.e. exclusivity was violated). Each would let a later reap + delete bytes a record still points at. +- **advisory** — `stale_owner` (owned but no longer held; fails toward + retention) and `unreferenced_file` (storage cost, not a correctness problem). + +The scan is read-only — it never writes, tombstones, or deletes. A ledger may +not be given authority over irreversible deletes until it has been shown to +agree with reality, so this must report zero blocking discrepancies on real +tenant data, on consecutive runs, before the gated collection change may merge. diff --git a/packages/services/service-storage/src/index.ts b/packages/services/service-storage/src/index.ts index 49e18a808a..b599af5bbb 100644 --- a/packages/services/service-storage/src/index.ts +++ b/packages/services/service-storage/src/index.ts @@ -16,5 +16,17 @@ export { installAttachmentLifecycleHooks, createSysFileReapGuard, createUploadSe export type { AttachmentLifecycleEngine, AttachmentLifecycleLogger } from './attachment-lifecycle.js'; export { installFileReferenceHooks, FileReferenceCopyError } from './file-reference-lifecycle.js'; export type { FileReferenceEngine, FileReferenceLogger } from './file-reference-lifecycle.js'; +export { + verifyFileReferences, + formatFileReferenceReport, + BLOCKING_ISSUE_KINDS, +} from './verify-file-references.js'; +export type { + FileReferenceIssue, + FileReferenceIssueKind, + FileReferenceReport, + VerifyReferencesEngine, + VerifyReferencesOptions, +} from './verify-file-references.js'; export { installAttachmentAccessHooks, installAttachmentReadVisibility } from './attachment-access-hooks.js'; export type { AttachmentSharingLike } from './attachment-access-hooks.js'; diff --git a/packages/services/service-storage/src/storage-routes.test.ts b/packages/services/service-storage/src/storage-routes.test.ts index ebd140f3f1..e63ddf923e 100644 --- a/packages/services/service-storage/src/storage-routes.test.ts +++ b/packages/services/service-storage/src/storage-routes.test.ts @@ -320,7 +320,7 @@ describe('Storage REST Routes', () => { expect(authorizeFileRead).not.toHaveBeenCalled(); }); - it('never gates a non-attachments file (avatars / field files stay open)', async () => { + it('never gates a file with neither an attachments scope nor a field owner', async () => { const { server, store: s, authorizeFileRead } = serverWith('deny'); await commit(s, { id: 'a5', scope: 'user', key: 'user/a5.png' }); const res = await hit(server, '/api/v1/storage/files/:fileId', 'a5'); @@ -334,6 +334,75 @@ describe('Storage REST Routes', () => { const res = await hit(server, '/api/v1/storage/files/:fileId/url', 'a6'); expect(res._status).toBe(200); }); + + // ── Field-owned files (ADR-0104 D3 wave 2) ───────────────────── + describe('field-owned files', () => { + const owned = (id: string, over: any = {}) => ({ + id, + scope: 'user', + key: `user/${id}.png`, + ref_object: 'product', + ref_id: 'p1', + ref_field: 'image', + ...over, + }); + + it('gates a field-owned file even though its scope is not attachments', async () => { + const { server, store: s, authorizeFileRead } = serverWith('deny'); + await commit(s, owned('fo1')); + const res = await hit(server, '/api/v1/storage/files/:fileId/url', 'fo1'); + expect(res._status).toBe(403); + // Distinct from the attachments code — this file BELONGS to one + // record, it is not attached to several. + expect(res._json?.code).toBe('FILE_DOWNLOAD_DENIED'); + expect(authorizeFileRead).toHaveBeenCalledOnce(); + }); + + it('401s an unauthenticated field-owned download', async () => { + const { server, store: s } = serverWith('unauthenticated'); + await commit(s, owned('fo2')); + const res = await hit(server, '/api/v1/storage/files/:fileId', 'fo2'); + expect(res._status).toBe(401); + expect(res._json?.code).toBe('AUTH_REQUIRED'); + }); + + it('allows an authorized field-owned download', async () => { + const { server, store: s } = serverWith('allow', { downloadTtl: 120 }); + await commit(s, owned('fo3')); + const res = await hit(server, '/api/v1/storage/files/:fileId/url', 'fo3'); + expect(res._status).toBe(200); + }); + + it('opts out via acl: public_read (the embedding escape hatch)', async () => { + const { server, store: s, authorizeFileRead } = serverWith('deny'); + await commit(s, owned('fo4', { acl: 'public_read' })); + const res = await hit(server, '/api/v1/storage/files/:fileId/url', 'fo4'); + expect(res._status).toBe(200); + expect(authorizeFileRead).not.toHaveBeenCalled(); + }); + + /** + * DUAL-MODE REGRESSION. A pre-cutover field holds an inline blob or an + * external URL, never a sys_file id — so no legacy file is ever claimed, + * `ref_object` stays unset, and this change gates nothing that used to + * be open. An unclaimed upload behaves the same way. + */ + it('does not gate an unclaimed file (no ref_object) — legacy fields keep working', async () => { + const { server, store: s, authorizeFileRead } = serverWith('deny'); + await commit(s, { id: 'fo5', scope: 'user', key: 'user/fo5.png' }); + const res = await hit(server, '/api/v1/storage/files/:fileId', 'fo5'); + expect(res._status).toBe(302); + expect(authorizeFileRead).not.toHaveBeenCalled(); + }); + + it('does not gate a file whose ownership was released', async () => { + const { server, store: s, authorizeFileRead } = serverWith('deny'); + await commit(s, owned('fo6', { ref_object: null, ref_id: null, ref_field: null })); + const res = await hit(server, '/api/v1/storage/files/:fileId', 'fo6'); + expect(res._status).toBe(302); + expect(authorizeFileRead).not.toHaveBeenCalled(); + }); + }); }); describe('PUT/GET /_local/raw/:token', () => { diff --git a/packages/services/service-storage/src/storage-routes.ts b/packages/services/service-storage/src/storage-routes.ts index 77745b8799..f9a66db99f 100644 --- a/packages/services/service-storage/src/storage-routes.ts +++ b/packages/services/service-storage/src/storage-routes.ts @@ -27,17 +27,20 @@ export interface StorageRoutesOptions { * is a tracked follow-up needing cookie sessions or signed links). */ resolveSession?: (req: IHttpRequest) => Promise<{ userId?: string } | null | undefined>; - /** - * Authorize a DOWNLOAD of an `attachments`-scope file (#2970 item 2). When - * wired, the download endpoints (`/files/:fileId` and `/files/:fileId/url`) - * consult this for `scope==='attachments'`, non-`public_read` files only: + /** + * Authorize a DOWNLOAD of a parent-governed file (#2970 item 2, extended by + * ADR-0104 D3 wave 2). When wired, the download endpoints + * (`/files/:fileId` and `/files/:fileId/url`) consult this for + * `scope==='attachments'` files AND for field-owned files (`ref_object` set), + * excluding `public_read` ones: * - `unauthenticated` → 401 (no session) - * - `deny` → 403 (session, but cannot read a parent record the file is - * attached to and is not the owner) + * - `deny` → 403 (session, but cannot read the parent record the file + * belongs to / is attached to, and is not the owner) * - `allow` → a short-lived signed URL is issued - * Non-attachments files (field files, avatars, org logos) keep the stable - * anonymous capability URL — they are embedded in `` which cannot - * carry a bearer token, and are out of scope for the attachments leak. + * A file with neither an attachments scope nor a field owner — an unclaimed + * upload, an org logo — keeps the stable anonymous capability URL, as does + * any file explicitly marked `acl: 'public_read'` (the opt-in for genuinely + * public embedding, since `` cannot carry a bearer token). * When absent (bare kernels, tests), all downloads stay open (back-compat). */ authorizeFileRead?: (file: FileRecord, req: IHttpRequest) => Promise; @@ -80,9 +83,20 @@ export function registerStorageRoutes( const sessionTtl = opts.sessionTtl ?? 86400; const downloadTtl = opts.downloadTtl ?? 300; - // ── Download authorization gate (#2970 item 2) ─────────────────────── - // Only `attachments`-scope, non-public files are gated; everything else - // keeps the stable anonymous capability URL (image/avatar embedding). + // ── Download authorization gate (#2970 item 2, ADR-0104 D3 wave 2) ─── + // Two kinds of file are gated, both deriving access from a PARENT record: + // - `attachments`-scope files, via their sys_attachment join rows; + // - field-owned files, via the single record whose field owns them + // (`ref_object`/`ref_id`, ADR-0104 D3 wave 2). + // `acl: 'public_read'` opts a file back out to the stable anonymous + // capability URL — needed for genuinely public embedding (`` + // cannot carry a bearer token), and now an explicit declaration rather + // than the silent default it used to be for every field file. + // + // Dual-mode safe: a legacy field holds an inline blob or an external URL, + // never a `sys_file` id, so no legacy file has `ref_object` set and none of + // them start being gated by this change. + // // Returns the signed-URL TTL to use, or `false` if a response was already // sent (401/403) and the handler must stop. const authorizeDownload = async ( @@ -90,7 +104,9 @@ export function registerStorageRoutes( req: IHttpRequest, res: IHttpResponse, ): Promise => { - if (file.scope !== 'attachments' || file.acl === 'public_read' || !opts.authorizeFileRead) { + const fieldOwned = !!file.ref_object && file.ref_id != null && file.ref_id !== ''; + const gated = file.scope === 'attachments' || fieldOwned; + if (!gated || file.acl === 'public_read' || !opts.authorizeFileRead) { return presignedTtl; } let verdict: FileReadVerdict; @@ -104,10 +120,17 @@ export function registerStorageRoutes( return false; } if (verdict === 'deny') { - res.status(403).json({ - error: 'You do not have access to a record this file is attached to', - code: 'ATTACHMENT_DOWNLOAD_DENIED', - }); + res.status(403).json( + fieldOwned + ? { + error: 'You do not have access to the record this file belongs to', + code: 'FILE_DOWNLOAD_DENIED', + } + : { + error: 'You do not have access to a record this file is attached to', + code: 'ATTACHMENT_DOWNLOAD_DENIED', + }, + ); return false; } return downloadTtl; diff --git a/packages/services/service-storage/src/storage-service-plugin.ts b/packages/services/service-storage/src/storage-service-plugin.ts index d397bbdc05..79af9ff8c3 100644 --- a/packages/services/service-storage/src/storage-service-plugin.ts +++ b/packages/services/service-storage/src/storage-service-plugin.ts @@ -463,12 +463,20 @@ function buildAuthSessionResolver( } /** - * Authorize an `attachments`-scope download (#2970 item 2). Builds the FULL - * caller ExecutionContext via `resolveAuthzContext` (the same shared resolver - * rest-server uses — a bare `{ userId }` would lack the resolved permissions - * the parent RLS needs), then allows when the caller is the file's owner or - * can READ a record the file is attached to. Returns `undefined` (routes stay - * open) when the auth service or engine is absent. + * Authorize a parent-governed download (#2970 item 2; extended for field-owned + * files by ADR-0104 D3 wave 2). Builds the FULL caller ExecutionContext via + * `resolveAuthzContext` (the same shared resolver rest-server uses — a bare + * `{ userId }` would lack the resolved permissions the parent RLS needs), then + * allows when the caller is the file's owner or can READ the file's parent + * record. Returns `undefined` (routes stay open) when the auth service or + * engine is absent. + * + * "Parent" resolves differently for the two surfaces, and that asymmetry is the + * point of the ownership model: an attachment may hang off MANY records, so its + * readable-by set is the union over its join rows; a field-owned file belongs + * to exactly ONE record, so its readable-by set is that record's and nothing + * more. A shared model would have had to union field references too, silently + * widening access whenever one file id was copied into a more public record. */ function buildFileReadAuthorizer( ctx: PluginContext, @@ -487,6 +495,22 @@ function buildFileReadAuthorizer( // Uploader / owner may always download. if (file.owner_id && String(file.owner_id) === String(authz.userId)) return 'allow'; + // Field-owned (ADR-0104 D3 wave 2): exactly ONE record's field holds + // this file, so access is that record's READ access — never a union. + if (file.ref_object && file.ref_id != null && file.ref_id !== '') { + try { + const visible = (await (engine as any).find(String(file.ref_object), { + where: { id: file.ref_id }, + fields: ['id'], + limit: 1, + context: authz, + })) as Array>; + return visible?.length ? 'allow' : 'deny'; + } catch { + return 'deny'; // unknown/failing owner object — fail closed + } + } + // Otherwise: readable via any parent record this file is attached to. const links = (await (engine as any).find('sys_attachment', { where: { file_id: file.id }, diff --git a/packages/services/service-storage/src/verify-file-references.test.ts b/packages/services/service-storage/src/verify-file-references.test.ts new file mode 100644 index 0000000000..76d5be555b --- /dev/null +++ b/packages/services/service-storage/src/verify-file-references.test.ts @@ -0,0 +1,237 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect } from 'vitest'; +import { + verifyFileReferences, + formatFileReferenceReport, + BLOCKING_ISSUE_KINDS, + type VerifyReferencesEngine, +} from './verify-file-references.js'; + +const REGISTRY: Record = { + sys_file: { fields: { id: { type: 'text' } } }, + product: { + fields: { + id: { type: 'text' }, + name: { type: 'text' }, + image: { type: 'image' }, + gallery: { type: 'image', multiple: true }, + }, + }, + tag: { fields: { id: { type: 'text' }, label: { type: 'text' } } }, +}; + +function fakeEngine(tables: Record>>): VerifyReferencesEngine { + const matches = (row: Record, where: Record) => + Object.entries(where).every(([k, v]) => { + if (v && typeof v === 'object' && '$ne' in (v as any)) return row[k] !== (v as any).$ne; + return row[k] === v; + }); + return { + getObject: (name) => REGISTRY[name], + getConfigs: () => REGISTRY, + async find(object, options: any) { + const rows = (tables[object] ?? []).filter((r) => matches(r, options?.where ?? {})); + const start = typeof options?.offset === 'number' ? options.offset : 0; + const end = typeof options?.limit === 'number' ? start + options.limit : undefined; + return rows.slice(start, end); + }, + }; +} + +const file = (id: string, owner?: { object: string; recordId: string; field: string }) => ({ + id, + status: 'committed', + scope: 'user', + ref_object: owner?.object ?? null, + ref_id: owner?.recordId ?? null, + ref_field: owner?.field ?? null, +}); + +describe('verifyFileReferences (ADR-0104 D3 wave 2 — R4 gate)', () => { + it('reports a clean bill when ownership matches what records hold', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'file_a' }], + sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })], + }); + + const report = await verifyFileReferences(engine); + + expect(report.ok).toBe(true); + expect(report.blocking).toBe(0); + expect(report.issues).toEqual([]); + expect(report.scannedObjects).toEqual(['product']); + expect(report.heldReferences).toBe(1); + expect(report.ownedFiles).toBe(1); + expect(formatFileReferenceReport(report)).toContain('No discrepancies'); + }); + + it('only scans objects that declare a file-class field', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1' }], + tag: [{ id: 't1', label: 'x' }], + sys_file: [], + }); + + const report = await verifyFileReferences(engine); + + expect(report.scannedObjects).toEqual(['product']); + expect(report.scannedRecords).toBe(1); + }); + + // ── Blocking: these would delete bytes something still holds ────── + it('BLOCKS on a held file with no recorded owner', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'file_a' }], + sys_file: [file('file_a')], // uploaded, never claimed + }); + + const report = await verifyFileReferences(engine); + + expect(report.ok).toBe(false); + expect(report.counts.unowned_reference).toBe(1); + expect(report.issues[0]).toMatchObject({ + kind: 'unowned_reference', + fileId: 'file_a', + object: 'product', + recordId: 'p1', + field: 'image', + }); + expect(formatFileReferenceReport(report)).toContain('BLOCKING'); + }); + + it('BLOCKS when a record holds a file owned by a different slot', async () => { + const engine = fakeEngine({ + product: [ + { id: 'p1', image: 'file_a' }, + { id: 'p2', image: 'file_a' }, + ], + sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })], + }); + + const report = await verifyFileReferences(engine); + + expect(report.ok).toBe(false); + // p2's reference is invisible to the lifecycle: when p1 releases, the + // bytes go while p2 still points at them. + expect(report.counts.foreign_owner).toBe(1); + expect(report.issues.find((i) => i.kind === 'foreign_owner')).toMatchObject({ recordId: 'p2' }); + // …and the exclusivity violation itself is reported too. + expect(report.counts.shared_reference).toBe(1); + }); + + it('BLOCKS on a file held by two slots (exclusivity violated)', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'file_a', gallery: ['file_a'] }], + sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })], + }); + + const report = await verifyFileReferences(engine); + + expect(report.counts.shared_reference).toBe(1); + expect(report.ok).toBe(false); + }); + + // ── Advisory: safe directions ───────────────────────────────────── + it('reports a stale owner as advisory, not blocking', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1' }], // field cleared without the release landing + sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })], + }); + + const report = await verifyFileReferences(engine); + + expect(report.counts.stale_owner).toBe(1); + // Fails toward retention — the file is simply never collected. + expect(report.blocking).toBe(0); + expect(report.ok).toBe(true); + }); + + it('reports unreferenced committed files only when asked, and never as blocking', async () => { + const tables = { + product: [{ id: 'p1' }], + sys_file: [file('file_orphan')], + }; + + const without = await verifyFileReferences(fakeEngine(tables)); + expect(without.counts.unreferenced_file).toBe(0); + + const withSweep = await verifyFileReferences(fakeEngine(tables), { includeUnreferenced: true }); + expect(withSweep.counts.unreferenced_file).toBe(1); + expect(withSweep.ok).toBe(true); + }); + + it('ignores attachments-scope files, which sys_attachment governs', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1' }], + sys_file: [{ ...file('file_att'), scope: 'attachments' }], + }); + + const report = await verifyFileReferences(engine, { includeUnreferenced: true }); + + expect(report.counts.unreferenced_file).toBe(0); + }); + + // ── Dual-mode ───────────────────────────────────────────────────── + it('ignores inline blobs and URL values — a pre-cutover tenant reads clean', async () => { + const engine = fakeEngine({ + product: [ + { id: 'p1', image: { url: 'https://cdn.example.com/a.png', name: 'a.png' } }, + { id: 'p2', image: 'https://cdn.example.com/b.png' }, + { id: 'p3', image: 'data:image/svg+xml,' }, + ], + sys_file: [], + }); + + const report = await verifyFileReferences(engine); + + expect(report.heldReferences).toBe(0); + expect(report.issues).toEqual([]); + expect(report.ok).toBe(true); + }); + + // ── Mechanics ───────────────────────────────────────────────────── + it('pages through records rather than reading one unbounded page', async () => { + const many = Array.from({ length: 1200 }, (_, i) => ({ id: `p${i}`, image: `file_${i}` })); + const engine = fakeEngine({ + product: many, + sys_file: many.map((r, i) => file(`file_${i}`, { object: 'product', recordId: `p${i}`, field: 'image' })), + }); + + const report = await verifyFileReferences(engine); + + expect(report.scannedRecords).toBe(1200); + expect(report.ownedFiles).toBe(1200); + expect(report.ok).toBe(true); + }); + + it('marks the verdict truncated when a scan bound is hit', async () => { + const many = Array.from({ length: 1200 }, (_, i) => ({ id: `p${i}` })); + const engine = fakeEngine({ product: many, sys_file: [] }); + + const report = await verifyFileReferences(engine, { maxRecordsPerObject: 500 }); + + expect(report.truncated).toBe(true); + expect(formatFileReferenceReport(report)).toContain('truncated'); + }); + + it('can be scoped to specific objects', async () => { + const engine = fakeEngine({ + product: [{ id: 'p1', image: 'file_a' }], + sys_file: [file('file_a')], + }); + + const report = await verifyFileReferences(engine, { objects: ['tag'] }); + + expect(report.scannedObjects).toEqual([]); + expect(report.ok).toBe(true); + }); + + it('classifies exactly the data-loss kinds as blocking', () => { + expect([...BLOCKING_ISSUE_KINDS].sort()).toEqual([ + 'foreign_owner', + 'shared_reference', + 'unowned_reference', + ]); + }); +}); diff --git a/packages/services/service-storage/src/verify-file-references.ts b/packages/services/service-storage/src/verify-file-references.ts new file mode 100644 index 0000000000..a56a416595 --- /dev/null +++ b/packages/services/service-storage/src/verify-file-references.ts @@ -0,0 +1,367 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { FILE_REFERENCE_TYPES, isFileIdToken } from '@objectstack/spec/data'; + +/** + * File-reference reconciliation (ADR-0104 D3 wave 2) — the executable form of + * the ADR's R4 acceptance gate. + * + * Compares GROUND TRUTH (what records' file-class fields actually hold) against + * RECORDED OWNERSHIP (`sys_file.ref_object` / `ref_id` / `ref_field`). The + * ownership columns are what a later, gated change will let authorise deleting + * file bytes, and a ledger may not be given that power until it has been shown + * to agree with reality — so this exists to be run, repeatedly, before that + * change may merge. + * + * Discrepancies are split by whether they can cause DATA LOSS once collection + * is enabled, because only that class blocks: + * + * - `unowned_reference` (**blocking**) — a record holds a file that nothing + * owns. With collection on, the file looks free and its bytes go. + * - `foreign_owner` (**blocking**) — a record holds a file owned by a + * different slot. That reference is invisible to the lifecycle, so when the + * recorded owner releases, the bytes go while this record still points at + * them. + * - `shared_reference` (**blocking**) — one file held by two or more slots, + * i.e. exclusivity was violated and copy-on-claim did not run. Same failure + * as above, and it also means one file's ACL is serving two parents. + * - `stale_owner` (advisory) — a file is owned by a slot that no longer holds + * it. Fails toward retention: the file is simply never collected. + * - `unreferenced_file` (advisory) — a committed file nothing points at. + * Storage cost, not a correctness problem. + * + * The scan is read-only. It never writes, never tombstones, and never deletes. + */ + +const SYSTEM_CTX = { isSystem: true } as const; + +/** Records read per page while scanning an object. */ +const SCAN_PAGE_SIZE = 500; + +export type FileReferenceIssueKind = + | 'unowned_reference' + | 'foreign_owner' + | 'shared_reference' + | 'stale_owner' + | 'unreferenced_file'; + +/** Issue kinds that would let a later reap delete bytes something still holds. */ +export const BLOCKING_ISSUE_KINDS: ReadonlySet = new Set([ + 'unowned_reference', + 'foreign_owner', + 'shared_reference', +] as const); + +export interface FileReferenceIssue { + kind: FileReferenceIssueKind; + fileId: string; + /** The slot this issue is reported against, when it has one. */ + object?: string; + recordId?: string; + field?: string; + detail: string; +} + +export interface FileReferenceReport { + /** Objects that declare at least one file-class field. */ + scannedObjects: string[]; + scannedRecords: number; + /** Distinct (file, slot) references actually held by records. */ + heldReferences: number; + /** `sys_file` rows carrying an owner. */ + ownedFiles: number; + issues: FileReferenceIssue[]; + counts: Record; + blocking: number; + /** + * Zero blocking issues. The gate: this must hold on real tenant data, on + * consecutive runs, before collection may be enabled. + */ + ok: boolean; + /** True when a bound was hit, so `ok` covers only what was scanned. */ + truncated: boolean; +} + +/** Engine surface the scan needs — duck-typed, like the other storage seams. */ +export interface VerifyReferencesEngine { + getObject(name: string): any | undefined; + getConfigs?(): Record; + find(object: string, options: Record): Promise>>; +} + +export interface VerifyReferencesOptions { + /** Restrict the scan to these objects (default: every object with a file field). */ + objects?: string[]; + /** Safety bound on records read per object. Exceeding it sets `truncated`. */ + maxRecordsPerObject?: number; + /** Include the advisory `unreferenced_file` sweep (an extra full sys_file read). */ + includeUnreferenced?: boolean; +} + +function slotKey(object: string, recordId: string, field: string): string { + return `${object}${recordId}${field}`; +} + +function fileFieldsOf(engine: VerifyReferencesEngine, objectName: string): string[] { + const schema: any = engine.getObject(objectName); + if (!schema?.fields) return []; + return Object.entries(schema.fields) + .filter(([, def]: [string, any]) => def && FILE_REFERENCE_TYPES.has(def.type)) + .map(([name]) => name); +} + +function idTokensIn(value: unknown): string[] { + if (Array.isArray(value)) return value.filter((v) => isFileIdToken(v)) as string[]; + return isFileIdToken(value) ? [value] : []; +} + +/** + * Scan records and `sys_file` ownership, and report where they disagree. + */ +export async function verifyFileReferences( + engine: VerifyReferencesEngine, + options: VerifyReferencesOptions = {}, +): Promise { + const maxPerObject = options.maxRecordsPerObject ?? 100_000; + const issues: FileReferenceIssue[] = []; + let truncated = false; + + const candidates = + options.objects ?? + (typeof engine.getConfigs === 'function' ? Object.keys(engine.getConfigs()) : []); + const scannedObjects = candidates.filter( + (name) => name !== 'sys_file' && fileFieldsOf(engine, name).length > 0, + ); + + // ── 1. Ground truth: what do records actually hold? ──────────────── + /** fileId → the slots holding it. */ + const held = new Map>(); + let scannedRecords = 0; + + for (const object of scannedObjects) { + const fileFields = fileFieldsOf(engine, object); + let offset = 0; + for (;;) { + let page: Array>; + try { + page = await engine.find(object, { + fields: ['id', ...fileFields], + limit: SCAN_PAGE_SIZE, + offset, + context: { ...SYSTEM_CTX }, + }); + } catch { + break; // unreadable object — skip rather than abort the whole scan + } + if (!page || page.length === 0) break; + for (const record of page) { + const recordId = record?.id; + if (recordId == null) continue; + scannedRecords++; + for (const field of fileFields) { + for (const fileId of new Set(idTokensIn(record[field]))) { + const list = held.get(fileId) ?? []; + list.push({ object, recordId: String(recordId), field }); + held.set(fileId, list); + } + } + } + offset += page.length; + if (page.length < SCAN_PAGE_SIZE) break; + if (offset >= maxPerObject) { + truncated = true; + break; + } + } + } + + // ── 2. Recorded ownership ───────────────────────────────────────── + /** fileId → the slot recorded as its owner. */ + const owned = new Map(); + let ownerOffset = 0; + for (;;) { + let page: Array>; + try { + page = await engine.find('sys_file', { + where: { ref_object: { $ne: null } }, + fields: ['id', 'ref_object', 'ref_id', 'ref_field', 'status'], + limit: SCAN_PAGE_SIZE, + offset: ownerOffset, + context: { ...SYSTEM_CTX }, + }); + } catch { + break; + } + if (!page || page.length === 0) break; + for (const row of page) { + // A driver without `$ne` support may hand back unowned rows too — filter + // here rather than trusting the predicate round-tripped. + if (row?.id == null || !row.ref_object || row.ref_id == null) continue; + owned.set(String(row.id), { + object: String(row.ref_object), + recordId: String(row.ref_id), + field: String(row.ref_field ?? ''), + }); + } + ownerOffset += page.length; + if (page.length < SCAN_PAGE_SIZE) break; + } + + // ── 3. Diff ─────────────────────────────────────────────────────── + let heldReferences = 0; + for (const [fileId, slots] of held) { + heldReferences += slots.length; + + if (slots.length > 1) { + issues.push({ + kind: 'shared_reference', + fileId, + detail: + `held by ${slots.length} slots (` + + slots.map((s) => `${s.object}/${s.recordId}.${s.field}`).join(', ') + + ') — field references must be exclusive; copy-on-claim did not run', + }); + } + + const owner = owned.get(fileId); + if (!owner) { + const s = slots[0]; + issues.push({ + kind: 'unowned_reference', + fileId, + object: s.object, + recordId: s.recordId, + field: s.field, + detail: `held by ${s.object}/${s.recordId}.${s.field} but no sys_file owner is recorded`, + }); + continue; + } + for (const s of slots) { + if (slotKey(s.object, s.recordId, s.field) !== slotKey(owner.object, owner.recordId, owner.field)) { + issues.push({ + kind: 'foreign_owner', + fileId, + object: s.object, + recordId: s.recordId, + field: s.field, + detail: + `held by ${s.object}/${s.recordId}.${s.field} but owned by ` + + `${owner.object}/${owner.recordId}.${owner.field}`, + }); + } + } + } + + for (const [fileId, owner] of owned) { + const slots = held.get(fileId); + const stillHeld = slots?.some( + (s) => slotKey(s.object, s.recordId, s.field) === slotKey(owner.object, owner.recordId, owner.field), + ); + if (!stillHeld) { + issues.push({ + kind: 'stale_owner', + fileId, + object: owner.object, + recordId: owner.recordId, + field: owner.field, + detail: + `owned by ${owner.object}/${owner.recordId}.${owner.field}, which no longer holds it — ` + + 'retained rather than collected (safe)', + }); + } + } + + // ── 4. Advisory: committed files nothing points at ──────────────── + if (options.includeUnreferenced) { + let offset = 0; + for (;;) { + let page: Array>; + try { + page = await engine.find('sys_file', { + where: { status: 'committed' }, + fields: ['id', 'ref_object', 'scope'], + limit: SCAN_PAGE_SIZE, + offset, + context: { ...SYSTEM_CTX }, + }); + } catch { + break; + } + if (!page || page.length === 0) break; + for (const row of page) { + const id = row?.id == null ? null : String(row.id); + // attachments-scope files are governed by sys_attachment, not by the + // ownership columns — not this scan's business. + if (!id || row.ref_object || row.scope === 'attachments') continue; + if (held.has(id)) continue; + issues.push({ + kind: 'unreferenced_file', + fileId: id, + detail: 'committed but neither owned nor held by any field — storage cost, not data risk', + }); + } + offset += page.length; + if (page.length < SCAN_PAGE_SIZE) break; + } + } + + const counts = { + unowned_reference: 0, + foreign_owner: 0, + shared_reference: 0, + stale_owner: 0, + unreferenced_file: 0, + } as Record; + for (const issue of issues) counts[issue.kind]++; + const blocking = issues.filter((i) => BLOCKING_ISSUE_KINDS.has(i.kind)).length; + + return { + scannedObjects, + scannedRecords, + heldReferences, + ownedFiles: owned.size, + issues, + counts, + blocking, + ok: blocking === 0, + truncated, + }; +} + +/** Render a report as human-readable lines (CLI / CI log output). */ +export function formatFileReferenceReport(report: FileReferenceReport): string { + const lines: string[] = []; + lines.push( + `Scanned ${report.scannedRecords} record(s) across ${report.scannedObjects.length} object(s) ` + + `with file fields; ${report.heldReferences} reference(s) held, ${report.ownedFiles} file(s) owned.`, + ); + if (report.truncated) { + lines.push('⚠ Scan was truncated by maxRecordsPerObject — this verdict covers only what was read.'); + } + if (report.issues.length === 0) { + lines.push('✅ No discrepancies: recorded ownership matches what records hold.'); + return lines.join('\n'); + } + for (const kind of [ + 'unowned_reference', + 'foreign_owner', + 'shared_reference', + 'stale_owner', + 'unreferenced_file', + ] as FileReferenceIssueKind[]) { + const of = report.issues.filter((i) => i.kind === kind); + if (of.length === 0) continue; + lines.push( + `\n${BLOCKING_ISSUE_KINDS.has(kind) ? '❌ BLOCKING' : '• advisory'} — ${kind} (${of.length}):`, + ); + for (const i of of.slice(0, 20)) lines.push(` ${i.fileId}: ${i.detail}`); + if (of.length > 20) lines.push(` … and ${of.length - 20} more`); + } + lines.push( + report.ok + ? '\n✅ No blocking discrepancies. Advisory items above cost storage, not data.' + : `\n❌ ${report.blocking} blocking discrepancy(ies). Enabling collection now could delete ` + + 'bytes a record still references.', + ); + return lines.join('\n'); +}