From f905943c5da754dd9d2d4b691ede5750fdd32a4f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 02:34:38 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(storage):=20exclusive=20field-referenc?= =?UTF-8?q?e=20file=20ownership=20=E2=80=94=20ADR-0104=20D3=20wave=202=20(?= =?UTF-8?q?PR-3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file/image/avatar/video/audio field holding a sys_file id now records its owner on the file itself: sys_file.ref_object / ref_id / ref_field name the single (object, record, field) slot that references it, maintained on the engine write path — claimed on insert, reconciled on update, released when the owning record is deleted. Field references are EXCLUSIVE, unlike attachments. The attachments surface deliberately shares one file across many sys_attachment join rows; a field reference is owned by at most one slot, and writing an already-owned id into a second slot copies the bytes into a fresh sys_file rather than sharing the row. Two properties follow, and both exist so a mistake cannot become unrecoverable: 1. A file's read authorisation derives from exactly one parent record instead of the union of every referrer's — so copying a private record's file id into a world-readable record, which generated code does without any visible error, cannot silently widen who can read the bytes. 2. Release is an OBSERVED transition ("my one owner let go"), never an inferred absence ("I counted and found nobody"). With no count, no miscount can authorise a delete — and a freshly-built ledger's ignorance of pre-existing references cannot either. Deletes nothing. This records and releases ownership; it never tombstones, and the scope==='attachments' guardrail that keeps field-referenced files out of the reap entirely is untouched, so shipping it cannot reclaim any bytes. Enabling collection must extend releaseOwnership AND the sys_file reap guard's sweep-time re-verify in the same change — half of it is worse than none, since tombstoning while the guard still re-verifies only sys_attachment (always empty for a field file) makes every release a guaranteed byte delete. Also lifts isFileIdToken into @objectstack/spec/data as the single arbiter of "is this stored string an opaque file id or a legacy/external URL?", replacing the private copy in the engine read resolver so the reader and the writer cannot drift into claiming files nobody expands, or vice versa. Dormant until a field actually holds an id token: objects without file-class fields, inline-blob values and URL-shaped values all exit before any I/O. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SHpGw3GBA9aFpfwVArRWfd --- .../adr-0104-d3w2-pr3-file-ownership.md | 34 ++ ...0104-field-runtime-value-shape-contract.md | 118 +++- packages/objectql/src/engine.ts | 12 +- .../src/file-reference-lifecycle.test.ts | 508 +++++++++++++++++ .../src/file-reference-lifecycle.ts | 539 ++++++++++++++++++ .../services/service-storage/src/index.ts | 2 + .../service-storage/src/metadata-store.ts | 6 + .../src/objects/system-file.object.ts | 45 ++ .../src/storage-service-plugin.test.ts | 19 + .../src/storage-service-plugin.ts | 8 + packages/spec/api-surface.json | 1 + packages/spec/src/data/field-value.zod.ts | 19 + 12 files changed, 1296 insertions(+), 15 deletions(-) create mode 100644 .changeset/adr-0104-d3w2-pr3-file-ownership.md create mode 100644 packages/services/service-storage/src/file-reference-lifecycle.test.ts create mode 100644 packages/services/service-storage/src/file-reference-lifecycle.ts diff --git a/.changeset/adr-0104-d3w2-pr3-file-ownership.md b/.changeset/adr-0104-d3w2-pr3-file-ownership.md new file mode 100644 index 0000000000..68b3d1f114 --- /dev/null +++ b/.changeset/adr-0104-d3w2-pr3-file-ownership.md @@ -0,0 +1,34 @@ +--- +"@objectstack/service-storage": minor +"@objectstack/spec": minor +--- + +feat(storage): exclusive field-reference file ownership — ADR-0104 D3 wave 2 (PR-3) + +A `file`/`image`/`avatar`/`video`/`audio` field that holds a `sys_file` id now +records its owner on the file: `sys_file.ref_object` / `ref_id` / `ref_field` +name the single `(object, record, field)` slot that references it, maintained on +the engine write path — claimed on insert, reconciled on update, released when +the owning record is deleted. + +**Field references are exclusive, unlike attachments.** The attachments surface +deliberately shares one file across many `sys_attachment` join rows; a field +reference is owned by at most one slot, and writing an already-owned id into a +second slot **copies the bytes into a fresh `sys_file`** rather than sharing the +row. That keeps a file's read authorisation derived from exactly one parent +record instead of the union of every referrer's — so copying a private record's +file id into a world-readable one cannot silently widen access — and it removes +reference counting from the lifecycle entirely: a file is released because its +one owner let go, never because a count came back zero. + +**Deletes nothing.** This records and releases ownership; it never tombstones, +and the `scope === 'attachments'` guardrail that keeps field-referenced files +out of the reap is untouched. Collection is a separate, gated change that must +also extend the reap guard's sweep-time re-verify in the same commit. + +Also exports `isFileIdToken` from `@objectstack/spec/data` as the single arbiter +of "is this stored string an opaque file id, or a legacy/external URL?", now +shared by the read resolver and the write claimer so the two cannot drift. + +Dormant until a field actually holds an id token: objects without file-class +fields, inline-blob values and URL-shaped values all exit before any I/O. diff --git a/docs/adr/0104-field-runtime-value-shape-contract.md b/docs/adr/0104-field-runtime-value-shape-contract.md index 59f32ecce8..456323a71a 100644 --- a/docs/adr/0104-field-runtime-value-shape-contract.md +++ b/docs/adr/0104-field-runtime-value-shape-contract.md @@ -1,8 +1,9 @@ # ADR-0104: Field runtime value-shape as a first-class contract — spec-owned value schemas, typed action handlers, file-as-reference - **Status**: Accepted (2026-07-22) — staged per D4. D1 landed (#3429), D2 - landed (#3432); D3 refined into two waves by the 2026-07-24 addendum below. - Strict-default flip of D1/D2 tracked in #3438. + landed (#3432); D3 refined into two waves by the 2026-07-24 addendum below, + and wave 2's ownership model settled by the 2026-07-27 addendum. + Strict-default flip of D1/D2 tracked in #3438; wave 2 sequence in #3459. - **Date**: 2026-07-22 - **Issue**: design follow-up generalizing #3405 / #3406 (inline lookup param silently stripped); relates #3407 (silently dropped writes), #1878 / #1891 @@ -481,12 +482,119 @@ rode v16/v17) to amortise the migration cost: under AI authoring this is *desirable*: it disambiguates "managed file" from "external link" so the AI cannot conflate them. +## Addendum (2026-07-27) — wave 2's ownership model: field references are exclusive + +The 2026-07-24 addendum left one thing open, and it turned out to be the +decision the rest of wave 2 hangs off: **when a field holds a `sys_file` id, +what is the relationship between file and record?** Implementing the reference +rows forced the answer. + +### Two lineages, and why we take one of each + +Enterprise platforms solve file lifecycle two ways: + +- **Junction-for-everything** (Salesforce `ContentDocumentLink`, SAP GOS): every + reference is a row in one polymorphic link table; a file is shared, and dies + when the last link does. Sharing is the feature. +- **Column-as-reference** (Dataverse File/Image columns): the column *is* the + reference; lifecycle follows the row; no sharing. + +ObjectStack already runs the first for its attachments surface (`sys_attachment` ++ the ADR-0057 tombstone/reap), and that is right — attaching one document to +several records is exactly what that surface is for, and the user performs the +attach explicitly. + +**Field references take the second model.** A `product.image` is a *property of +that product*, and properties are copied, not shared. Concretely: at most one +`(object, record, field)` slot owns a given `sys_file`; the owner is recorded on +`sys_file.ref_object` / `ref_id` / `ref_field`; writing an already-owned id into +a second slot **copies the bytes into a fresh `sys_file`** rather than sharing +the row. + +### Why exclusivity, decided on failure modes rather than elegance + +Sharing is more storage-efficient and matches the better-known lineage. We take +exclusivity anyway, because under AI-authored metadata the two models fail +differently and the difference is not symmetric. + +The naive generated clone — `insert({...src, id: undefined})` — spreads a file +id into a second record. Every model must let that code *work*; rejecting it +would force AI authors to learn a bespoke file API, which is precisely the kind +of special case that produces mistakes. So the real question is what happens +*after* it works: + +| | worst case | recoverable? | +|---|---|---| +| Shared + reference-counted | file's readable-by set becomes the **union of every referrer's** — copying a private record's id into a world-readable record silently widens access; and a single miscount authorises an **irreversible byte delete** | no | +| Exclusive + copy-on-claim | duplicated bytes | yes — it costs money, not data | + +Read authorisation for attachment files already derives from the parent record. +Under sharing, "the parent record" is ambiguous, and the safe reading of an +ambiguous ACL is the union — which means generated code can leak data while +containing no visible error. Exclusivity removes the ambiguity by construction: +one file, one parent, one ACL. + +The storage cost this concedes belongs at a different layer anyway: **dedup the +bytes, not the rows.** Content-hash dedup behind the storage adapter (`etag` is +already carried on `sys_file`) recovers the savings while leaving every row its +own owner and its own ACL — the lifecycle never has to reason about sharing. +A genuinely shared library asset should be modelled as its own object with a +lookup, or attached through the attachments surface; steering AI authors there +is the correct outcome, not a limitation. + +### The safety principle this buys: observed transition, never inferred absence + +> A file becomes collectable only because the platform **observed its one owner +> let go** — never because a scan **inferred** that nobody references it. + +The existing attachment path already obeys this (the tombstone is set by the +hook that watched the join row die, not by a sweep that found none). Exclusive +ownership extends the same discipline to fields, and eliminates counting +entirely: with no count, no miscount can authorise a delete. It also defuses the +cold-start hazard a new ledger would otherwise carry — a freshly-built reference +table knows nothing of references that predate it, so trusting its zeroes would +mass-delete history. + +### Consequent resequencing of wave 2 + +The 2026-07-24 plan packaged the write cutover and the GC enable together, and +placed the backfill after. That order is wrong: **a ledger may not authorise +deletion until it has been backfilled and reconciled.** Wave 2 therefore +sequences as: + +1. **Ownership bookkeeping** — `ref_*` columns, claim/release on the write path, + copy-on-claim. Records ownership; deletes nothing. Dormant until a field + actually holds an id, so it lands safely ahead of the cutover. +2. **Governed download** — read authorisation derived from the one owning + record; anonymous capability URL demoted to opt-in `acl: 'public_read'`. +3. **Write cutover** (protocol major) — stored form narrows to an id; + `accept` / `maxSize` enforced. Still deletes nothing. +4. **Backfill** — `os migrate` converts legacy inline blobs to `sys_file` rows + and claims them; external URLs reported, not converted. +5. **Reconciliation soak** — `os storage verify-references` compares what + records actually hold against recorded ownership, reporting *over-claim* + (safe: file retained longer than needed), *under-claim* (**blocking**: a held + file with no owner would be treated as free), and unclaimed orphans. +6. **GC enable** — *gated, irreversible*. Relaxing the `scope === 'attachments'` + tombstone guardrail and extending the `sys_file` reap guard's sweep-time + re-verify to the ownership columns **must ship in the same change**. Half of + it is worse than none: tombstoning released files while the guard still + re-verifies only `sys_attachment` — always empty for a field file — turns + every release into a guaranteed byte delete rather than a risky one. + +R4's acceptance gate is now executable rather than aspirational: step 6 may not +merge until step 5 reports **zero under-claims for ≥7 consecutive days** on real +tenant data. R5 (public-posture inventory) and R6 (sub-key read scan) stand. + ### Why this stays inside ADR-0104 rather than a new ADR D3 is already this ADR's third phase; the two-wave split and the enforcement-point principle are a **refinement of the D4 rollout**, not a new decision, so they live here. Wave 2's migration mechanics (dual-read window, `os migrate` backfill, the R4/R5/R6 gates) are specified in §D3 above and need -no separate record. Should wave 2's implementation surface a genuinely new -decision (e.g. the reference-table shape, or a chunked-migration protocol), that -specific choice — not file-as-reference as a whole — would earn its own ADR. +no separate record. Wave 2's implementation did surface one genuinely new +decision — the exclusive-ownership model — and it is recorded as the 2026-07-27 +addendum above rather than a separate ADR, because it settles *how* D3's +already-accepted "field values point into `sys_file`" behaves rather than +revisiting whether to do it. A future choice that stands on its own (say, a +chunked-migration protocol, or byte-layer content dedup) would earn its own ADR. diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 88d6b35b1e..cb16ecf2c6 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -12,7 +12,7 @@ import { type DroppedFieldsEvent } from '@objectstack/spec/data'; import type { WriteObservabilityOptions } from '@objectstack/spec/contracts'; -import { parseAutonumberFormat, renderAutonumber, missingFieldValues, isTenancyDisabled, FILE_REFERENCE_TYPES } from '@objectstack/spec/data'; +import { parseAutonumberFormat, renderAutonumber, missingFieldValues, isTenancyDisabled, FILE_REFERENCE_TYPES, isFileIdToken } from '@objectstack/spec/data'; import { ExecutionContext, ExecutionContextSchema } from '@objectstack/spec/kernel'; import { IDataDriver, IDataEngine, Logger, createLogger, withTransientRetry, type RetryOptions } from '@objectstack/core'; import { SummaryRecomputeError, type SummaryRecomputeFailure } from './summary-errors.js'; @@ -2120,14 +2120,6 @@ export class ObjectQL implements IDataEngine { return records; } - /** - * Whether a value is an opaque `sys_file` id token — the minted uuid/nanoid - * form (letters, digits, `_`, `-`, bounded length), and crucially NOT a URL: - * a `https://…` / `/api/…` / `data:…` / `blob:…` file value carries `:`, `/` - * or `.` and is left untouched (it is a legacy/external URL, not a reference). - */ - private static readonly FILE_ID_RE = /^[A-Za-z0-9_-]{1,64}$/; - /** * Resolve file-field id references to their expanded `FileValueSchema` form * (ADR-0104 D3 wave 2). A `file`/`image`/`avatar`/`video`/`audio` value @@ -2168,7 +2160,7 @@ export class ObjectQL implements IDataEngine { // what keeps a seeded `data:`/CDN image value from firing a bogus lookup. const candidateIds: string[] = []; const addCandidate = (v: unknown) => { - if (typeof v === 'string' && ObjectQL.FILE_ID_RE.test(v)) candidateIds.push(v); + if (isFileIdToken(v)) candidateIds.push(v); }; for (const record of records) { for (const fieldName of fileFields) { diff --git a/packages/services/service-storage/src/file-reference-lifecycle.test.ts b/packages/services/service-storage/src/file-reference-lifecycle.test.ts new file mode 100644 index 0000000000..fec22208e4 --- /dev/null +++ b/packages/services/service-storage/src/file-reference-lifecycle.test.ts @@ -0,0 +1,508 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect, vi } from 'vitest'; +import { + installFileReferenceHooks, + FileReferenceCopyError, + type FileReferenceEngine, +} from './file-reference-lifecycle.js'; + +const silentLogger = () => ({ info: vi.fn(), warn: vi.fn(), debug: vi.fn() }); + +/** Object schemas the fake registry serves. `sys_file` must be present or the + * whole module is inert by design. */ +const REGISTRY: Record = { + sys_file: { name: 'sys_file', fields: { id: { type: 'text' }, key: { type: 'text' } } }, + product: { + name: 'product', + fields: { + id: { type: 'text' }, + name: { type: 'text' }, + image: { type: 'image' }, + gallery: { type: 'image', multiple: true }, + }, + }, + // No file-class field — every hook must exit before doing any I/O. + tag: { name: 'tag', fields: { id: { type: 'text' }, label: { type: 'text' } } }, +}; + +function fakeEngine(opts: { + files?: Array>; + records?: Record>>; + registry?: Record; +} = {}) { + const registry = opts.registry ?? REGISTRY; + const tables: Record>> = { + sys_file: [...(opts.files ?? [])], + ...Object.fromEntries(Object.entries(opts.records ?? {}).map(([k, v]) => [k, [...v]])), + }; + const hooks = new Map Promise | void>>(); + const calls: Array<{ op: string; object: string; arg: unknown }> = []; + + const matchValue = (actual: unknown, expected: unknown): boolean => { + if (expected && typeof expected === 'object' && Array.isArray((expected as any).$in)) { + return (expected as any).$in.some((v: unknown) => String(v) === String(actual)); + } + return actual === expected; + }; + const matches = (row: Record, where: Record) => + Object.entries(where).every(([k, v]) => matchValue(row[k], v)); + + const engine: FileReferenceEngine & { + tables: typeof tables; + calls: typeof calls; + trigger(event: string, ctx: any): Promise; + } = { + registerHook(event, handler, opts2) { + // Global registration — any object may declare a file-class field. + expect(opts2?.object).toBeUndefined(); + const list = hooks.get(event) ?? []; + list.push(handler); + hooks.set(event, list); + }, + getObject(name) { + return registry[name]; + }, + async find(object, options: any) { + calls.push({ op: 'find', object, arg: options?.where }); + const rows = (tables[object] ?? []).filter((r) => matches(r, options?.where ?? {})); + return typeof options?.limit === 'number' ? rows.slice(0, options.limit) : rows; + }, + async findOne(object, options: any) { + calls.push({ op: 'findOne', object, arg: options?.where }); + return (tables[object] ?? []).find((r) => matches(r, options?.where ?? {})) ?? null; + }, + async insert(object, data: any) { + calls.push({ op: 'insert', object, arg: data }); + (tables[object] ??= []).push({ ...data }); + return data; + }, + async update(object, data: any) { + calls.push({ op: 'update', object, arg: data }); + const row = (tables[object] ?? []).find((r) => String(r.id) === String(data.id)); + if (row) Object.assign(row, data); + return row; + }, + tables, + calls, + async trigger(event, ctx) { + for (const h of hooks.get(event) ?? []) await h(ctx); + }, + }; + return engine; +} + +function fakeStorage() { + return { + download: vi.fn(async () => Buffer.from('the-bytes')), + upload: vi.fn(async () => {}), + delete: vi.fn(async () => {}), + exists: vi.fn(async () => true), + getInfo: vi.fn(async () => ({ key: 'k', size: 9, contentType: 'image/png', lastModified: new Date() })), + } as any; +} + +type Engine = ReturnType; + +/** Drive an engine-shaped insert: beforeInsert → driver write → afterInsert, + * with the same ctx object throughout and `input.data` as the persisted row + * (exactly what engine.ts hands the driver). */ +async function driveInsert(engine: Engine, object: string, data: Record, id: string) { + const ctx: any = { object, event: 'beforeInsert', input: { data } }; + await engine.trigger('beforeInsert', ctx); + const row = { ...(ctx.input.data as Record), id }; + (engine.tables[object] ??= []).push(row); + ctx.event = 'afterInsert'; + ctx.result = row; + await engine.trigger('afterInsert', ctx); + return row; +} + +async function driveUpdate(engine: Engine, object: string, id: string, data: Record) { + const ctx: any = { object, event: 'beforeUpdate', input: { id, data } }; + await engine.trigger('beforeUpdate', ctx); + const row = (engine.tables[object] ?? []).find((r) => String(r.id) === String(id)); + if (row) Object.assign(row, ctx.input.data); + ctx.event = 'afterUpdate'; + ctx.result = row; + await engine.trigger('afterUpdate', ctx); + return row; +} + +async function driveDelete(engine: Engine, object: string, input: any) { + const ctx: any = { object, event: 'beforeDelete', input }; + await engine.trigger('beforeDelete', ctx); + const where = input?.options?.where; + if (input?.id != null) { + const ids = typeof input.id === 'object' ? input.id.$in : [input.id]; + engine.tables[object] = (engine.tables[object] ?? []).filter( + (r) => !ids.some((i: unknown) => String(i) === String(r.id)), + ); + } else if (where) { + engine.tables[object] = (engine.tables[object] ?? []).filter( + (r) => !Object.entries(where).every(([k, v]) => r[k] === v), + ); + } + ctx.event = 'afterDelete'; + await engine.trigger('afterDelete', ctx); +} + +function install(engine: Engine, storage: any = fakeStorage()) { + const logger = silentLogger(); + installFileReferenceHooks(engine, () => storage, logger); + return { logger, storage }; +} + +const file = (over: Record = {}) => ({ + id: 'file_a', + key: 'user/file_a.png', + name: 'a.png', + mime_type: 'image/png', + size: 10, + scope: 'user', + status: 'committed', + ...over, +}); + +describe('File Reference Ownership (ADR-0104 D3 wave 2)', () => { + // ── Claim ──────────────────────────────────────────────────────── + describe('claim', () => { + it('claims a file id written into a field on insert', async () => { + const engine = fakeEngine({ files: [file()] }); + install(engine); + + await driveInsert(engine, 'product', { name: 'Widget', image: 'file_a' }, 'p1'); + + expect(engine.tables.sys_file[0]).toMatchObject({ + id: 'file_a', + ref_object: 'product', + ref_id: 'p1', + ref_field: 'image', + }); + }); + + it('claims every id of a multiple:true field', async () => { + const engine = fakeEngine({ files: [file(), file({ id: 'file_b', key: 'user/file_b.png' })] }); + install(engine); + + await driveInsert(engine, 'product', { gallery: ['file_a', 'file_b'] }, 'p1'); + + expect(engine.tables.sys_file.map((f) => [f.id, f.ref_field])).toEqual([ + ['file_a', 'gallery'], + ['file_b', 'gallery'], + ]); + }); + + it('brings a tombstoned file back to life when it is referenced again', async () => { + const engine = fakeEngine({ + files: [file({ status: 'deleted', deleted_at: '2026-01-01T00:00:00.000Z' })], + }); + install(engine); + + await driveInsert(engine, 'product', { image: 'file_a' }, 'p1'); + + expect(engine.tables.sys_file[0]).toMatchObject({ + status: 'committed', + deleted_at: null, + ref_id: 'p1', + }); + }); + }); + + // ── Dormancy: the pre-v17 world must cost nothing ──────────────── + describe('dormancy (dual-mode)', () => { + it('ignores a legacy inline blob value — no sys_file access at all', async () => { + const engine = fakeEngine({ files: [file()] }); + install(engine); + + await driveInsert( + engine, + 'product', + { image: { url: 'https://cdn.example.com/a.png', name: 'a.png' } }, + 'p1', + ); + + expect(engine.calls.filter((c) => c.object === 'sys_file')).toHaveLength(0); + expect(engine.tables.sys_file[0].ref_id).toBeUndefined(); + }); + + it.each([ + ['https://cdn.example.com/a.png'], + ['/api/v1/storage/files/file_a'], + ['data:image/svg+xml,'], + ['blob:http://localhost/abc'], + ])('never treats the URL-shaped value %s as a reference', async (value) => { + const engine = fakeEngine({ files: [file()] }); + install(engine); + + await driveInsert(engine, 'product', { image: value }, 'p1'); + + expect(engine.calls.filter((c) => c.object === 'sys_file')).toHaveLength(0); + }); + + it('does nothing for an object with no file-class fields', async () => { + const engine = fakeEngine({ files: [file()] }); + install(engine); + + await driveInsert(engine, 'tag', { label: 'x' }, 't1'); + await driveDelete(engine, 'tag', { id: 't1' }); + + expect(engine.calls.filter((c) => c.object === 'sys_file')).toHaveLength(0); + }); + + it('stays inert when sys_file is not registered (storage plugin absent)', async () => { + const engine = fakeEngine({ + files: [file()], + registry: { product: REGISTRY.product }, + }); + install(engine); + + await driveInsert(engine, 'product', { image: 'file_a' }, 'p1'); + + expect(engine.calls.filter((c) => c.object === 'sys_file')).toHaveLength(0); + }); + }); + + // ── Release ────────────────────────────────────────────────────── + describe('release', () => { + it('releases ownership when the owning record is deleted', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + install(engine); + + await driveDelete(engine, 'product', { id: 'p1' }); + + expect(engine.tables.sys_file[0]).toMatchObject({ + ref_object: null, + ref_id: null, + ref_field: null, + }); + }); + + it('releases via the beforeDelete stash for a where-shaped multi delete', async () => { + const engine = fakeEngine({ + files: [ + file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' }), + file({ id: 'file_b', ref_object: 'product', ref_id: 'p2', ref_field: 'image' }), + ], + records: { + product: [ + { id: 'p1', image: 'file_a', archived: true }, + { id: 'p2', image: 'file_b', archived: true }, + ], + }, + }); + install(engine); + + await driveDelete(engine, 'product', { options: { where: { archived: true } } }); + + expect(engine.tables.sys_file.every((f) => f.ref_id === null)).toBe(true); + }); + + it('releases the old file and claims the new one when a field is swapped', async () => { + const engine = fakeEngine({ + files: [ + file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' }), + file({ id: 'file_b', key: 'user/file_b.png' }), + ], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + install(engine); + + await driveUpdate(engine, 'product', 'p1', { image: 'file_b' }); + + expect(engine.tables.sys_file[0]).toMatchObject({ id: 'file_a', ref_id: null }); + expect(engine.tables.sys_file[1]).toMatchObject({ id: 'file_b', ref_id: 'p1', ref_field: 'image' }); + }); + + it('releases when the field is cleared', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + install(engine); + + await driveUpdate(engine, 'product', 'p1', { image: null }); + + expect(engine.tables.sys_file[0]).toMatchObject({ ref_id: null }); + }); + + it('leaves the file alone when a PATCH does not touch the file field', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a', name: 'old' }] }, + }); + install(engine); + + await driveUpdate(engine, 'product', 'p1', { name: 'new' }); + + expect(engine.tables.sys_file[0]).toMatchObject({ ref_id: 'p1', ref_field: 'image' }); + expect(engine.calls.filter((c) => c.op === 'update' && c.object === 'sys_file')).toHaveLength(0); + }); + + it('is idempotent when an update rewrites the same value', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + const { storage } = install(engine); + + await driveUpdate(engine, 'product', 'p1', { image: 'file_a' }); + + expect(engine.tables.sys_file[0]).toMatchObject({ ref_id: 'p1', ref_field: 'image' }); + expect(engine.calls.filter((c) => c.op === 'update' && c.object === 'sys_file')).toHaveLength(0); + expect(storage.download).not.toHaveBeenCalled(); + }); + + /** + * R4 REGRESSION. Releasing must NOT tombstone: `deleted_at` is what makes a + * row a reap candidate, and the reap guard's sweep-time re-verify still + * only consults `sys_attachment` (always empty for a field file). Setting + * the tombstone here without extending that guard in the same change turns + * every released file into a guaranteed byte delete. + */ + it('never tombstones on release — the file stays as retained as it was', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + install(engine); + + await driveDelete(engine, 'product', { id: 'p1' }); + + const row = engine.tables.sys_file[0]; + expect(row.status).toBe('committed'); + expect(row.deleted_at).toBeUndefined(); + const writes = engine.calls.filter((c) => c.op === 'update' && c.object === 'sys_file'); + for (const w of writes) { + expect(w.arg).not.toHaveProperty('status'); + expect(w.arg).not.toHaveProperty('deleted_at'); + } + }); + }); + + // ── Exclusive ownership / copy-on-claim ────────────────────────── + describe('exclusive ownership', () => { + it('copies the bytes when a second record claims an already-owned file', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + const { storage } = install(engine); + + const row = await driveInsert(engine, 'product', { image: 'file_a' }, 'p2'); + + // The second record stores a DIFFERENT id — never the original. + expect(row.image).not.toBe('file_a'); + expect(typeof row.image).toBe('string'); + expect(storage.download).toHaveBeenCalledWith('user/file_a.png'); + expect(storage.upload).toHaveBeenCalledTimes(1); + + // The original owner is untouched… + expect(engine.tables.sys_file[0]).toMatchObject({ id: 'file_a', ref_id: 'p1' }); + // …and the copy is owned by the second record. + const copy = engine.tables.sys_file.find((f) => f.id === row.image)!; + expect(copy).toMatchObject({ + ref_object: 'product', + ref_id: 'p2', + ref_field: 'image', + status: 'committed', + name: 'a.png', + mime_type: 'image/png', + }); + expect(copy.key).not.toBe('user/file_a.png'); + }); + + it('copies when an UPDATE moves an owned id into a different slot', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }, { id: 'p2' }] }, + }); + const { storage } = install(engine); + + const row = await driveUpdate(engine, 'product', 'p2', { image: 'file_a' }); + + expect(row!.image).not.toBe('file_a'); + expect(storage.upload).toHaveBeenCalledTimes(1); + expect(engine.tables.sys_file[0]).toMatchObject({ id: 'file_a', ref_id: 'p1' }); + }); + + it('copies into the SAME record when a second field references its file', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + install(engine); + + const row = await driveUpdate(engine, 'product', 'p1', { gallery: ['file_a'] }); + + // Ownership is per (object, record, FIELD) — a sibling field is a + // different slot and gets its own copy. + expect((row!.gallery as string[])[0]).not.toBe('file_a'); + expect(engine.tables.sys_file[0]).toMatchObject({ ref_field: 'image' }); + }); + + it('does not copy when the same slot rewrites its own file', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + const { storage } = install(engine); + + await driveUpdate(engine, 'product', 'p1', { image: 'file_a', name: 'renamed' }); + + expect(storage.download).not.toHaveBeenCalled(); + expect(engine.tables.sys_file).toHaveLength(1); + }); + + it('fails the write when the copy cannot be made', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + const storage = fakeStorage(); + storage.download = vi.fn(async () => { + throw new Error('backend unavailable'); + }); + install(engine, storage); + + await expect(driveInsert(engine, 'product', { image: 'file_a' }, 'p2')).rejects.toThrow( + FileReferenceCopyError, + ); + // Nothing partially recorded: the original still owns its file and no + // half-built copy row was left behind. + expect(engine.tables.sys_file).toHaveLength(1); + expect(engine.tables.sys_file[0]).toMatchObject({ id: 'file_a', ref_id: 'p1' }); + }); + + it('never transfers ownership when copying is impossible (no storage service)', async () => { + const engine = fakeEngine({ + files: [file({ ref_object: 'product', ref_id: 'p1', ref_field: 'image' })], + records: { product: [{ id: 'p1', image: 'file_a' }] }, + }); + const logger = silentLogger(); + installFileReferenceHooks(engine, () => null, logger); + + await driveInsert(engine, 'product', { image: 'file_a' }, 'p2'); + + // The first record keeps it — a steal would silently re-home the file + // and hand its read authorisation to a different parent record. + expect(engine.tables.sys_file[0]).toMatchObject({ ref_id: 'p1', ref_field: 'image' }); + expect(logger.warn).toHaveBeenCalled(); + }); + }); + + // ── Unknown ids ────────────────────────────────────────────────── + it('leaves an id that matches no sys_file row untouched', async () => { + const engine = fakeEngine({ files: [file()] }); + const { storage } = install(engine); + + const row = await driveInsert(engine, 'product', { image: 'not_a_known_file' }, 'p1'); + + expect(row.image).toBe('not_a_known_file'); + expect(storage.download).not.toHaveBeenCalled(); + expect(engine.calls.filter((c) => c.op === 'update' && c.object === 'sys_file')).toHaveLength(0); + }); +}); diff --git a/packages/services/service-storage/src/file-reference-lifecycle.ts b/packages/services/service-storage/src/file-reference-lifecycle.ts new file mode 100644 index 0000000000..6cf5981c6b --- /dev/null +++ b/packages/services/service-storage/src/file-reference-lifecycle.ts @@ -0,0 +1,539 @@ +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. + +import { randomUUID } from 'node:crypto'; +import { FILE_REFERENCE_TYPES, isFileIdToken } from '@objectstack/spec/data'; +import type { IStorageService } from '@objectstack/spec/contracts'; + +/** + * Field-reference file ownership (ADR-0104 D3 wave 2). + * + * A `file`/`image`/`avatar`/`video`/`audio` FIELD holds an opaque `sys_file` + * id. This module keeps `sys_file.ref_object/ref_id/ref_field` in step with + * what records actually hold, so the platform always knows which record's + * field owns a given file. + * + * ## Exclusive ownership, not shared reference counting + * + * The attachments surface (`sys_attachment`) deliberately SHARES one file + * across many join rows — Salesforce ContentDocumentLink semantics, where + * attaching the same document to several records is the point. Field + * references take the opposite model: at most ONE (object, record, field) + * slot owns a file. Writing an already-owned id into a second slot COPIES the + * bytes into a fresh `sys_file` instead of sharing the row. + * + * Two properties fall out, and both exist to keep a mistake from becoming + * unrecoverable: + * + * 1. **Release is an observed transition, never an inferred absence.** The + * platform learns a file is free because its one owner let go — not + * because a count came back zero. A miscount cannot authorise a delete, + * because nothing is ever counted. + * 2. **Read authorisation derives from exactly one parent record**, not the + * union of every referrer's. Copying a private record's file id into a + * world-readable record — the sort of thing generated code does without + * any visible mistake — cannot widen who can read the bytes, because the + * second record gets its own copy. + * + * ## What this module does NOT do + * + * It records ownership and it releases ownership. It never tombstones and + * never deletes bytes. The `scope === 'attachments'` guardrail in + * `attachment-lifecycle.ts` — which is what keeps field-referenced files out + * of the reap entirely — is untouched here, so shipping this module cannot + * delete anything. Turning released files into reap candidates is a separate, + * gated change that must also extend the reap guard's sweep-time re-verify in + * the same commit; see {@link releaseOwnership}. + * + * ## Dormancy + * + * Every path exits before doing I/O unless a file-class field on the written + * object actually holds an id token. Objects without file-class fields pay a + * cached-metadata lookup and nothing else; a field holding a legacy inline + * blob or an external URL is never an id token and costs nothing. + */ + +const PACKAGE_ID = 'com.objectstack.service.storage'; +const SYSTEM_CTX = { isSystem: true } as const; + +/** Bound on records resolved per where-shaped multi-delete — matches the + * attachment lifecycle's posture: bound one pass, converge across sweeps. */ +const MULTI_DELETE_RESOLVE_LIMIT = 1_000; + +/** Bound on owned files released per record delete. */ +const RELEASE_BATCH_LIMIT = 1_000; + +/** Key under which beforeDelete stashes the ids of records about to die (the + * engine passes the SAME HookContext object to before/after delete). Only + * needed for where-shaped deletes, where afterDelete has no id list. */ +const STASH_KEY = '__fileRefDeletedIds'; + +/** Engine surface these installers need — duck-typed like the other + * service-storage seams so tests can fake it. */ +export interface FileReferenceEngine { + registerHook( + event: string, + handler: (ctx: any) => void | Promise, + options?: { object?: string | string[]; packageId?: string; priority?: number }, + ): void; + getObject(name: string): any | undefined; + find(object: string, options: Record): Promise>>; + findOne(object: string, options: Record): Promise | null>; + insert(object: string, data: Record, options?: Record): Promise; + update(object: string, data: Record, options: Record): Promise; +} + +export interface FileReferenceLogger { + info(msg: string, meta?: unknown): void; + warn(msg: string, meta?: unknown): void; + debug?(msg: string, meta?: unknown): void; +} + +/** Raised when a file id must be copied to preserve exclusive ownership but + * the copy fails. The write is failed rather than completed, because a field + * pointing at bytes it does not own is precisely the state the ownership + * model exists to make impossible — and the state a later reap would act on. */ +export class FileReferenceCopyError extends Error { + readonly code = 'ERR_FILE_REFERENCE_COPY'; + constructor(fileId: string, cause: unknown) { + super( + `Cannot copy file '${fileId}' for a second field reference — ` + + `refusing the write rather than storing a reference the record does not own ` + + `(${(cause as Error)?.message ?? cause})`, + ); + } +} + +function asIdList(id: unknown): Array | null { + if (typeof id === 'string' || typeof id === 'number') return [id]; + if (id && typeof id === 'object' && Array.isArray((id as any).$in)) { + return (id as any).$in.filter((v: unknown) => typeof v === 'string' || typeof v === 'number'); + } + return null; +} + +/** File-class field names on an object, or `[]`. */ +function fileFieldsOf(engine: FileReferenceEngine, objectName: string): string[] { + const schema: any = engine.getObject(objectName); + if (!schema?.fields) return []; + return Object.entries(schema.fields) + .filter(([, def]: [string, any]) => def && FILE_REFERENCE_TYPES.has(def.type)) + .map(([name]) => name); +} + +/** The id tokens a field value carries (a `multiple: true` field holds an + * array). Inline blobs and URL-shaped strings yield nothing. */ +function idTokensIn(value: unknown): string[] { + if (Array.isArray(value)) return value.filter((v) => isFileIdToken(v)) as string[]; + return isFileIdToken(value) ? [value] : []; +} + +/** + * Is this module live for `objectName`? Guards every entry point: + * - `sys_file` itself is excluded so the module's own bookkeeping writes + * cannot re-enter it, + * - a kernel without the storage objects registered stays inert, + * - an object with no file-class fields — nearly all of them — exits here. + */ +function activeFileFields(engine: FileReferenceEngine, objectName: string): string[] { + if (objectName === 'sys_file') return []; + if (!engine.getObject('sys_file')) return []; + return fileFieldsOf(engine, objectName); +} + +/** + * Copy a file's bytes into a brand-new `sys_file`, returning the new id. + * + * Reached only when a record writes an id already owned by a different field + * slot. Uses `download` + `upload` from the {@link IStorageService} contract; + * an adapter with a native server-side copy (S3 `CopyObject`) can shortcut + * this later without changing the caller. + */ +async function copyOwnedFile( + engine: FileReferenceEngine, + storage: IStorageService, + src: Record, +): Promise { + const srcKey = typeof src.key === 'string' ? src.key : ''; + if (!srcKey) throw new Error('source file has no storage key'); + + const bytes = await storage.download(srcKey); + const newId = randomUUID(); + const name = typeof src.name === 'string' && src.name ? src.name : newId; + const ext = name.includes('.') ? '.' + name.split('.').pop() : ''; + const scope = typeof src.scope === 'string' && src.scope ? src.scope : 'user'; + const newKey = `${scope}/${newId}${ext}`; + + await storage.upload(newKey, bytes, { + contentType: typeof src.mime_type === 'string' ? src.mime_type : undefined, + acl: typeof src.acl === 'string' ? (src.acl as any) : undefined, + } as any); + + const now = new Date().toISOString(); + // Ownership columns are deliberately left NULL — the after-hook claims the + // copy for the slot that triggered it, on the same path as any other file. + await engine.insert( + 'sys_file', + { + id: newId, + key: newKey, + name, + mime_type: src.mime_type ?? null, + size: src.size ?? null, + scope, + bucket: src.bucket ?? null, + acl: src.acl ?? null, + status: 'committed', + etag: null, + owner_id: src.owner_id ?? null, + created_at: now, + updated_at: now, + }, + { context: { ...SYSTEM_CTX } }, + ); + return newId; +} + +/** + * Rewrite, in place, any id in `data` that is already owned by a different + * slot so it points at a fresh copy instead. Runs in the BEFORE hooks, where + * mutating `input.data` is what the driver goes on to persist — so the record + * never transiently holds a reference it does not own. + * + * `recordId` is null on insert: a new record can never be the current owner, + * so every already-owned id is copied. + */ +async function applyCopyOnClaim( + engine: FileReferenceEngine, + getStorage: () => IStorageService | null | undefined, + logger: FileReferenceLogger, + object: string, + recordId: string | null, + data: Record, + fileFields: string[], +): Promise { + for (const field of fileFields) { + if (!(field in data)) continue; + const value = data[field]; + const tokens = idTokensIn(value); + if (tokens.length === 0) continue; + + const replacements = new Map(); + for (const token of [...new Set(tokens)]) { + const row = await engine.findOne('sys_file', { + where: { id: token }, + context: { ...SYSTEM_CTX }, + }); + // Unknown id: not a file this platform manages (external/legacy value). + // Left verbatim — the read resolver ignores it for the same reason. + if (!row) continue; + // Unclaimed: the after-hook will claim it for this slot. Nothing to copy. + if (row.ref_id == null) continue; + // Already ours (an update rewriting the same value) — no-op. + if ( + row.ref_object === object && + String(row.ref_id) === String(recordId ?? '') && + row.ref_field === field + ) { + continue; + } + + const storage = getStorage(); + if (!storage) { + // No storage service wired (bare kernel). Copying is impossible, so + // ownership cannot be maintained for this value — but failing user + // writes on a kernel that cannot have served an upload in the first + // place would be worse. Leave the value; `claimFile` refuses to steal, + // so the original owner is never disturbed. + logger.warn( + `[storage] file reference: no storage service — cannot copy already-owned file ${token} ` + + `for ${object}.${field}; the value is kept but this record will not own it`, + ); + continue; + } + try { + replacements.set(token, await copyOwnedFile(engine, storage, row)); + logger.debug?.( + `[storage] file reference: copied ${token} for ${object}.${field} (exclusive ownership)`, + ); + } catch (err) { + throw new FileReferenceCopyError(token, err); + } + } + + if (replacements.size === 0) continue; + data[field] = Array.isArray(value) + ? value.map((v) => (typeof v === 'string' && replacements.has(v) ? replacements.get(v)! : v)) + : (replacements.get(value as string) ?? value); + } +} + +/** + * Record that `fileId` is owned by `object`/`recordId`/`field`. + * + * NEVER steals: a file already owned by a different slot is left alone and + * logged. Reaching that branch means copy-on-claim could not run (no storage + * service), and the safe outcome is that the original owner keeps the file — + * a second record pointing at bytes it does not own is visible to + * `verify-references`, whereas a silently transferred owner would not be. + */ +async function claimFile( + engine: FileReferenceEngine, + logger: FileReferenceLogger, + fileId: string, + object: string, + recordId: string, + field: string, +): Promise { + const row = await engine.findOne('sys_file', { where: { id: fileId }, context: { ...SYSTEM_CTX } }); + if (!row) return; // not a platform-managed file — nothing to own + if ( + row.ref_id != null && + !(row.ref_object === object && String(row.ref_id) === String(recordId) && row.ref_field === field) + ) { + logger.warn( + `[storage] file reference: ${fileId} is already owned by ` + + `${row.ref_object}/${row.ref_id}.${row.ref_field} — not transferring it to ${object}/${recordId}.${field}`, + ); + return; + } + + const patch: Record = { + id: fileId, + ref_object: object, + ref_id: String(recordId), + ref_field: field, + }; + // A file re-referenced within its grace window comes back to life, mirroring + // the attachment re-attach path. + if (row.status === 'deleted') { + patch.status = 'committed'; + patch.deleted_at = null; + } + await engine.update('sys_file', patch, { context: { ...SYSTEM_CTX } }); +} + +/** + * Give up ownership of `rows` — the single seam through which a field-owned + * file becomes unreferenced. + * + * Today this only clears the ownership columns; the file becomes unowned and + * stays exactly as retained as it was before (no tombstone, so nothing makes + * it a reap candidate). Enabling collection means extending THIS function to + * also set `status='deleted'` + `deleted_at` — and that change is only safe in + * the same commit that teaches the `sys_file` reap guard to re-verify the + * ownership columns at sweep time. Doing the tombstone half alone would make + * every released file reapable while the guard still re-verifies only + * `sys_attachment` (always empty for field files), i.e. a guaranteed byte + * delete rather than a merely risky one. Keep both halves together. + */ +async function releaseOwnership( + engine: FileReferenceEngine, + logger: FileReferenceLogger, + rows: Array>, +): Promise { + for (const row of rows) { + const id = row?.id; + if (id == null) continue; + try { + await engine.update( + 'sys_file', + { id, ref_object: null, ref_id: null, ref_field: null }, + { context: { ...SYSTEM_CTX } }, + ); + logger.debug?.(`[storage] file reference: released ownership of sys_file ${String(id)}`); + } catch (err) { + // Bookkeeping must never break a user's write. A missed release only + // means the file stays owned — it lingers rather than being collected, + // which is the safe direction. + logger.warn( + `[storage] file reference: failed to release sys_file ${String(id)} ` + + `(${(err as Error)?.message ?? err})`, + ); + } + } +} + +/** Files currently owned by a specific slot. */ +async function ownedBySlot( + engine: FileReferenceEngine, + object: string, + recordId: string, + field: string, +): Promise>> { + return ( + (await engine.find('sys_file', { + where: { ref_object: object, ref_id: String(recordId), ref_field: field }, + limit: RELEASE_BATCH_LIMIT, + context: { ...SYSTEM_CTX }, + })) ?? [] + ); +} + +/** + * Install the ownership hooks. Global (no object filter): any object may + * declare a file-class field, and {@link activeFileFields} exits immediately + * for those that do not. + */ +export function installFileReferenceHooks( + engine: FileReferenceEngine, + getStorage: () => IStorageService | null | undefined, + logger: FileReferenceLogger, +): void { + // ── beforeInsert / beforeUpdate: copy-on-claim ──────────────────── + // Runs before the driver write so the persisted value is already the id + // this record will own. + engine.registerHook( + 'beforeInsert', + async (ctx: any) => { + const object: string = ctx?.object; + const data = ctx?.input?.data; + if (!object || !data || typeof data !== 'object') return; + const fileFields = activeFileFields(engine, object); + if (fileFields.length === 0) return; + await applyCopyOnClaim(engine, getStorage, logger, object, null, data, fileFields); + }, + { packageId: PACKAGE_ID }, + ); + + engine.registerHook( + 'beforeUpdate', + async (ctx: any) => { + const object: string = ctx?.object; + const data = ctx?.input?.data; + if (!object || !data || typeof data !== 'object') return; + const fileFields = activeFileFields(engine, object); + if (fileFields.length === 0) return; + const ids = asIdList(ctx?.input?.id); + // A multi-update writing a file id would give the same id to every + // matched record; only a single-record update can own one. Copy-on-claim + // needs a definite owner, so skip — `claimFile` then refuses to steal. + const recordId = ids && ids.length === 1 ? String(ids[0]) : null; + await applyCopyOnClaim(engine, getStorage, logger, object, recordId, data, fileFields); + }, + { packageId: PACKAGE_ID }, + ); + + // ── afterInsert: claim ──────────────────────────────────────────── + engine.registerHook( + 'afterInsert', + async (ctx: any) => { + const object: string = ctx?.object; + const row = ctx?.result; + if (!object || !row || typeof row !== 'object') return; + const fileFields = activeFileFields(engine, object); + if (fileFields.length === 0) return; + const recordId = row.id; + if (recordId == null) return; + for (const field of fileFields) { + for (const token of [...new Set(idTokensIn(row[field]))]) { + try { + await claimFile(engine, logger, token, object, String(recordId), field); + } catch (err) { + logger.warn( + `[storage] file reference: failed to claim ${token} for ${object}/${String(recordId)}.${field} ` + + `(${(err as Error)?.message ?? err})`, + ); + } + } + } + }, + { packageId: PACKAGE_ID }, + ); + + // ── afterUpdate: release what left the slot, claim what arrived ─── + engine.registerHook( + 'afterUpdate', + async (ctx: any) => { + const object: string = ctx?.object; + const data = ctx?.input?.data; + if (!object || !data || typeof data !== 'object') return; + const fileFields = activeFileFields(engine, object); + if (fileFields.length === 0) return; + const ids = asIdList(ctx?.input?.id); + if (!ids || ids.length !== 1) return; // see beforeUpdate — no definite owner + const recordId = String(ids[0]); + + for (const field of fileFields) { + // Only a field the write actually TOUCHED can have changed ownership; + // a PATCH that omits the field leaves its file alone. + if (!(field in data)) continue; + const incoming = new Set(idTokensIn(data[field])); + try { + const current = await ownedBySlot(engine, object, recordId, field); + const stale = current.filter((r) => !incoming.has(String(r.id))); + await releaseOwnership(engine, logger, stale); + const held = new Set(current.map((r) => String(r.id))); + for (const token of incoming) { + if (held.has(token)) continue; // already ours + await claimFile(engine, logger, token, object, recordId, field); + } + } catch (err) { + logger.warn( + `[storage] file reference: failed to reconcile ${object}/${recordId}.${field} ` + + `(${(err as Error)?.message ?? err})`, + ); + } + } + }, + { packageId: PACKAGE_ID }, + ); + + // ── beforeDelete: resolve ids for where-shaped deletes ──────────── + // A delete keyed by id needs nothing here; a `where` delete has no id list + // in afterDelete, and by then the records are gone. + engine.registerHook( + 'beforeDelete', + async (ctx: any) => { + const object: string = ctx?.object; + if (!object) return; + if (activeFileFields(engine, object).length === 0) return; + if (asIdList(ctx?.input?.id)) return; // afterDelete can read it directly + const where = ctx?.input?.options?.where; + if (!where) return; + try { + const rows = await engine.find(object, { + where, + limit: MULTI_DELETE_RESOLVE_LIMIT, + context: { ...SYSTEM_CTX }, + }); + ctx[STASH_KEY] = (rows ?? []).map((r) => r?.id).filter((id) => id != null); + } catch (err) { + logger.warn( + `[storage] file reference: failed to resolve ids before a where-delete on ${object} ` + + `(${(err as Error)?.message ?? err})`, + ); + ctx[STASH_KEY] = []; + } + }, + { packageId: PACKAGE_ID }, + ); + + // ── afterDelete: release everything the dead records owned ──────── + // Keyed off the ownership index, so the deleted records' field values never + // have to be read back (they are already gone). + engine.registerHook( + 'afterDelete', + async (ctx: any) => { + const object: string = ctx?.object; + if (!object) return; + if (activeFileFields(engine, object).length === 0) return; + const stashed = Array.isArray(ctx?.[STASH_KEY]) ? ctx[STASH_KEY] : null; + const ids = stashed ?? asIdList(ctx?.input?.id); + if (!ids || ids.length === 0) return; + try { + const owned = await engine.find('sys_file', { + where: { ref_object: object, ref_id: { $in: ids.map((id) => String(id)) } }, + limit: RELEASE_BATCH_LIMIT, + context: { ...SYSTEM_CTX }, + }); + if (owned?.length) await releaseOwnership(engine, logger, owned); + } catch (err) { + logger.warn( + `[storage] file reference: failed to release files owned by deleted ${object} records ` + + `(${(err as Error)?.message ?? err})`, + ); + } + }, + { packageId: PACKAGE_ID }, + ); +} diff --git a/packages/services/service-storage/src/index.ts b/packages/services/service-storage/src/index.ts index f40cd462cc..49e18a808a 100644 --- a/packages/services/service-storage/src/index.ts +++ b/packages/services/service-storage/src/index.ts @@ -14,5 +14,7 @@ export type { StorageRoutesOptions, FileReadVerdict } from './storage-routes.js' export { SystemFile, SystemUploadSession } from './objects/index.js'; export { installAttachmentLifecycleHooks, createSysFileReapGuard, createUploadSessionReapGuard } from './attachment-lifecycle.js'; export type { AttachmentLifecycleEngine, AttachmentLifecycleLogger } from './attachment-lifecycle.js'; +export { installFileReferenceHooks, FileReferenceCopyError } from './file-reference-lifecycle.js'; +export type { FileReferenceEngine, FileReferenceLogger } from './file-reference-lifecycle.js'; export { installAttachmentAccessHooks, installAttachmentReadVisibility } from './attachment-access-hooks.js'; export type { AttachmentSharingLike } from './attachment-access-hooks.js'; diff --git a/packages/services/service-storage/src/metadata-store.ts b/packages/services/service-storage/src/metadata-store.ts index 7e03552ab0..bafe3de728 100644 --- a/packages/services/service-storage/src/metadata-store.ts +++ b/packages/services/service-storage/src/metadata-store.ts @@ -23,6 +23,12 @@ export interface FileRecord { /** Orphan tombstone (#2755) — set when the last sys_attachment reference * is removed; the sys_file lifecycle TTL reaps after the grace window. */ deleted_at?: string | null; + /** Exclusive field-reference owner (ADR-0104 D3 wave 2) — the single + * (object, record, field) slot whose value is this file's id. NULL while + * unclaimed and for every attachments-surface file. */ + ref_object?: string | null; + ref_id?: string | null; + ref_field?: string | null; } /** diff --git a/packages/services/service-storage/src/objects/system-file.object.ts b/packages/services/service-storage/src/objects/system-file.object.ts index d01e1e8c9c..4917efba2c 100644 --- a/packages/services/service-storage/src/objects/system-file.object.ts +++ b/packages/services/service-storage/src/objects/system-file.object.ts @@ -99,6 +99,44 @@ export const SystemFile = ObjectSchema.create({ owner_id: Field.text({ label: 'Owner ID', + description: 'User who uploaded the file (authorship, not the field-reference owner)', + }), + + // ── Field-reference ownership (ADR-0104 D3 wave 2) ────────────── + // A `file`/`image`/`avatar`/`video`/`audio` FIELD value references a file + // by opaque id. Unlike the attachments surface — where one file is + // deliberately SHARED across many `sys_attachment` join rows — a field + // reference is EXCLUSIVE: at most one (object, record, field) slot owns a + // given file. Writing an already-owned id into a second slot copies the + // bytes into a fresh `sys_file` rather than sharing the row. + // + // Exclusivity is what makes the lifecycle safe to reason about: release is + // an OBSERVED transition ("my one owner let go"), never an inferred absence + // ("I counted and found nobody"), and a file's read authorization derives + // from exactly one parent record instead of the union of every referrer's. + // See ADR-0104 §"D3 wave 2 — ownership model". + // + // NULL on: freshly uploaded but not yet claimed by a record write, and on + // every attachments-surface file (those are governed by sys_attachment). + ref_object: Field.text({ + label: 'Referencing Object', + maxLength: 128, + description: 'Short object name of the record whose field owns this file', + group: 'Field Reference', + }), + + ref_id: Field.text({ + label: 'Referencing Record', + maxLength: 64, + description: 'Primary key of the record whose field owns this file', + group: 'Field Reference', + }), + + ref_field: Field.text({ + label: 'Referencing Field', + maxLength: 128, + description: 'Field name on the owning record that holds this file id', + group: 'Field Reference', }), metadata: Field.text({ @@ -120,6 +158,13 @@ export const SystemFile = ObjectSchema.create({ }), }, + indexes: [ + // "which file does this record's field own" / "release everything this + // record owned" — the release path is a single keyed write against this + // index, so a delete never has to re-read the record's field values. + { fields: ['ref_object', 'ref_id'] }, + ], + // ADR-0057 (#2755): sys_file rows are mostly permanent business truth, but // two terminal states are garbage that would otherwise grow forever: // - tombstoned attachment orphans (status='deleted', deleted_at set by diff --git a/packages/services/service-storage/src/storage-service-plugin.test.ts b/packages/services/service-storage/src/storage-service-plugin.test.ts index e7a678ec39..79bd0bd39a 100644 --- a/packages/services/service-storage/src/storage-service-plugin.test.ts +++ b/packages/services/service-storage/src/storage-service-plugin.test.ts @@ -197,12 +197,21 @@ describe('StorageServicePlugin: sys_file orphan lifecycle wiring (#2755)', () => const ctx = makeCtx(); const hookEvents: string[] = []; + const globalHookEvents: string[] = []; const middlewares: Array<{ object?: string }> = []; ctx.registerService('objectql', { registerHook: (event: string, _fn: unknown, opts: any) => { + // File-reference ownership (ADR-0104 D3 wave 2) registers GLOBALLY — + // any object may declare a file-class field. Everything else is + // scoped to sys_attachment. + if (opts?.object === undefined) { + globalHookEvents.push(event); + return; + } expect(opts?.object).toBe('sys_attachment'); hookEvents.push(event); }, + getObject: () => undefined, registerMiddleware: (_fn: unknown, opts: any) => { middlewares.push({ object: opts?.object }); }, @@ -229,6 +238,16 @@ describe('StorageServicePlugin: sys_file orphan lifecycle wiring (#2755)', () => 'beforeDelete', 'beforeInsert', ]); + // Field-reference ownership: claim/copy on write, release on delete + // (file-reference-lifecycle.ts). Registered without an object filter. + expect(globalHookEvents.sort()).toEqual([ + 'afterDelete', + 'afterInsert', + 'afterUpdate', + 'beforeDelete', + 'beforeInsert', + 'beforeUpdate', + ]); // Two reap guards: sys_file (#2755) + sys_upload_session multipart-abort (#2970). expect(guards.map((g) => g.object).sort()).toEqual(['sys_file', 'sys_upload_session']); expect(guards.every((g) => typeof g.guard === 'function')).toBe(true); diff --git a/packages/services/service-storage/src/storage-service-plugin.ts b/packages/services/service-storage/src/storage-service-plugin.ts index b10a14aaae..d397bbdc05 100644 --- a/packages/services/service-storage/src/storage-service-plugin.ts +++ b/packages/services/service-storage/src/storage-service-plugin.ts @@ -17,6 +17,7 @@ import type { FileRecord } from './metadata-store.js'; import { registerStorageRoutes } from './storage-routes.js'; import type { FileReadVerdict } from './storage-routes.js'; import { installAttachmentLifecycleHooks, createSysFileReapGuard, createUploadSessionReapGuard } from './attachment-lifecycle.js'; +import { installFileReferenceHooks } from './file-reference-lifecycle.js'; import { installAttachmentAccessHooks, installAttachmentReadVisibility } from './attachment-access-hooks.js'; import { SystemFile, SystemUploadSession } from './objects/index.js'; // ADR-0052 §3 ownership: `sys_attachment` (a file↔record link) belongs with the @@ -238,6 +239,13 @@ export class StorageServicePlugin implements Plugin { if (typeof (engine as any).registerMiddleware === 'function') { installAttachmentReadVisibility(engine as any, ctx.logger); } + // Field-reference ownership (ADR-0104 D3 wave 2) — keeps + // sys_file.ref_object/ref_id/ref_field in step with what records hold, + // and copies bytes rather than sharing a row when a second field slot + // writes an already-owned id. Records ownership only: it never + // tombstones, so the `scope==='attachments'` reap guardrail above + // still keeps field-referenced files out of collection entirely. + installFileReferenceHooks(engine as any, () => this.storage, ctx.logger); try { const lifecycle = ctx.getService('lifecycle'); if (lifecycle && typeof lifecycle.registerReapGuard === 'function') { diff --git a/packages/spec/api-surface.json b/packages/spec/api-surface.json index e10cd9d3c6..d4d617ed62 100644 --- a/packages/spec/api-surface.json +++ b/packages/spec/api-surface.json @@ -539,6 +539,7 @@ "hookForm (const)", "isCompatible (function)", "isDateMacroToken (function)", + "isFileIdToken (function)", "isFilterAST (function)", "isIncoherentAggregate (function)", "isMultiValueField (function)", diff --git a/packages/spec/src/data/field-value.zod.ts b/packages/spec/src/data/field-value.zod.ts index f7a95c3c45..80e97c3d7b 100644 --- a/packages/spec/src/data/field-value.zod.ts +++ b/packages/spec/src/data/field-value.zod.ts @@ -103,6 +103,25 @@ export const FILE_REFERENCE_TYPES: ReadonlySet = new Set([ 'image', 'file', 'avatar', 'video', 'audio', ] as const satisfies readonly FieldType[]); +/** + * Does a stored file-field string look like an opaque `sys_file` id, rather + * than the URL a file field legitimately holds in the legacy/dual-mode world? + * + * ADR-0104 D3 wave 2 makes this the SINGLE arbiter for both directions of the + * reference path — the read resolver (which ids does the engine expand?) and + * the write claimer (which ids does storage take ownership of?). Two + * hand-copied predicates that drift by one character would silently claim + * files nobody expands, or expand files nobody owns; there is exactly one + * definition so that cannot happen. + * + * A minted id is uuid/nanoid-shaped: word characters and `-`, nothing else. A + * URL — `https://…`, `/api/…`, `data:…`, `blob:…` — always carries a `:`, `/` + * or `.` and so can never match. + */ +export function isFileIdToken(value: unknown): value is string { + return typeof value === 'string' && /^[A-Za-z0-9_-]{1,64}$/.test(value); +} + /** Structured JSON payloads persisted in JSON columns. */ export const STRUCTURED_JSON_TYPES: ReadonlySet = new Set([ 'json', 'composite', 'repeater', 'record', 'location', 'address', 'vector', From 8fd264bf5e50f6a23e76fbe5b30705b24c683194 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 02:39:27 +0000 Subject: [PATCH 2/2] fix(storage): write \u0000-free same-slot check check:nul-bytes caught a raw NUL used as an impossible-record-id sentinel in the copy-on-claim same-slot comparison. A raw NUL makes ripgrep treat the whole file as binary, so it silently drops out of code search and every grep-based lint. Replaced with an explicit null check plus a named isSameSlot helper, which says what it means and is reused by claimFile. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SHpGw3GBA9aFpfwVArRWfd --- .../src/file-reference-lifecycle.ts | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/services/service-storage/src/file-reference-lifecycle.ts b/packages/services/service-storage/src/file-reference-lifecycle.ts index 6cf5981c6b..ddc088e4bd 100644 --- a/packages/services/service-storage/src/file-reference-lifecycle.ts +++ b/packages/services/service-storage/src/file-reference-lifecycle.ts @@ -127,6 +127,18 @@ function idTokensIn(value: unknown): string[] { return isFileIdToken(value) ? [value] : []; } +/** Does `row`'s recorded owner name exactly this slot? */ +function isSameSlot( + row: Record, + object: string, + recordId: string, + field: string, +): boolean { + return ( + row.ref_object === object && String(row.ref_id) === String(recordId) && row.ref_field === field + ); +} + /** * Is this module live for `objectName`? Guards every entry point: * - `sys_file` itself is excluded so the module's own bookkeeping writes @@ -229,13 +241,9 @@ async function applyCopyOnClaim( // Unclaimed: the after-hook will claim it for this slot. Nothing to copy. if (row.ref_id == null) continue; // Already ours (an update rewriting the same value) — no-op. - if ( - row.ref_object === object && - String(row.ref_id) === String(recordId ?? '') && - row.ref_field === field - ) { - continue; - } + // `recordId` is null on insert, where a brand-new record can never be + // the current owner — so there, every already-owned id is a copy. + if (recordId !== null && isSameSlot(row, object, recordId, field)) continue; const storage = getStorage(); if (!storage) { @@ -286,10 +294,7 @@ async function claimFile( ): Promise { const row = await engine.findOne('sys_file', { where: { id: fileId }, context: { ...SYSTEM_CTX } }); if (!row) return; // not a platform-managed file — nothing to own - if ( - row.ref_id != null && - !(row.ref_object === object && String(row.ref_id) === String(recordId) && row.ref_field === field) - ) { + if (row.ref_id != null && !isSameSlot(row, object, recordId, field)) { logger.warn( `[storage] file reference: ${fileId} is already owned by ` + `${row.ref_object}/${row.ref_id}.${row.ref_field} — not transferring it to ${object}/${recordId}.${field}`,