diff --git a/packages/function-resolution/__tests__/default-bucket.test.ts b/packages/function-resolution/__tests__/default-bucket.test.ts new file mode 100644 index 00000000..60977c4f --- /dev/null +++ b/packages/function-resolution/__tests__/default-bucket.test.ts @@ -0,0 +1,191 @@ +import { getConnections, PgTestClient } from 'pgsql-test'; + +let pg: PgTestClient; +let teardown: () => Promise; + +// Deterministic fixture ids. +const PLATFORM_DB = '11111111-1111-1111-1111-111111111111'; +const TENANT_DB = '22222222-2222-2222-2222-222222222222'; +// A tenant that never labelled a default bucket. +const BARE_DB = '33333333-3333-3333-3333-333333333333'; +// A tenant that labelled two, which is a tagging mistake and not a coin flip. +const AMBIGUOUS_DB = '44444444-4444-4444-4444-444444444444'; +// A tenant whose own default must never answer somebody else's probe. +const OTHER_DB = '55555555-5555-5555-5555-555555555555'; + +const ids: Record = {}; + +// The database's default bucket: the answer to "store this file" when no client +// named a bucket. It is a labelled bucket resolved by the same exactly-one rule +// capabilities use — never a literal, never an environment setting, never a +// fallback to another tenant's storage. +describe('default bucket resolution', () => { + beforeAll(async () => { + ({ pg, teardown } = await getConnections()); + + await pg.query( + `INSERT INTO metaschema_public.database (id, name, platform) + VALUES ($1, 'platform_db', true), ($2, 'tenant_db', false), + ($3, 'bare_db', false), ($4, 'ambiguous_db', false), + ($5, 'other_db', false)`, + [PLATFORM_DB, TENANT_DB, BARE_DB, AMBIGUOUS_DB, OTHER_DB] + ); + + // The published bucket plane, as the catalog-sync triggers maintain it. + await pg.query(`CREATE SCHEMA catalog_private`); + await pg.query( + `CREATE TABLE catalog_private.buckets ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + owner_scope text NOT NULL, + owner_key uuid, + is_visible boolean NOT NULL DEFAULT false, + database_id uuid NOT NULL, + key text NOT NULL, + type text NOT NULL, + physical_name text, + tags text[] + )` + ); + + const bucket = async ( + database: string, + key: string, + type: string, + tags: string[] + ) => { + const row = await pg.one( + `INSERT INTO catalog_private.buckets + (owner_scope, owner_key, is_visible, database_id, key, type, physical_name, tags) + VALUES ('database', $1, false, $1, $2, $3, $4, $5) RETURNING id`, + [database, key, type, `phys-${key}`, tags] + ); + return row.id; + }; + + ids.default = await bucket(TENANT_DB, 'default', 'private', ['default']); + ids.defaultPublic = await bucket(TENANT_DB, 'default-public', 'public', [ + 'default-public', + ]); + ids.avatars = await bucket(TENANT_DB, 'avatars', 'public', ['avatars']); + + ids.otherDefault = await bucket(OTHER_DB, 'default', 'private', ['default']); + + ids.ambiguousOne = await bucket(AMBIGUOUS_DB, 'documents', 'private', [ + 'default', + ]); + ids.ambiguousTwo = await bucket(AMBIGUOUS_DB, 'uploads', 'private', [ + 'default', + ]); + }); + + afterAll(async () => { + await teardown(); + }); + + it('the reserved tag vocabulary is one fact, not a literal per call site', async () => { + const [tags] = await pg.any( + `SELECT function_resolution.default_bucket_tag(false) AS private_tag, + function_resolution.default_bucket_tag(true) AS public_tag` + ); + expect(tags).toEqual({ private_tag: 'default', public_tag: 'default-public' }); + }); + + it('resolves the database default without anyone naming a bucket', async () => { + const [row] = await pg.any( + `SELECT bucket_id, resolved_key, bucket_type, physical_name, owner_database_id + FROM function_resolution.resolve_default_bucket($1, 'database', $1, false)`, + [TENANT_DB] + ); + expect(row).toEqual({ + bucket_id: ids.default, + resolved_key: 'default', + bucket_type: 'private', + physical_name: 'phys-default', + owner_database_id: TENANT_DB, + }); + }); + + it('public access resolves the CDN-served default, not the private one', async () => { + const [row] = await pg.any( + `SELECT bucket_id, bucket_type + FROM function_resolution.resolve_default_bucket($1, 'database', $1, true)`, + [TENANT_DB] + ); + expect(row).toEqual({ bucket_id: ids.defaultPublic, bucket_type: 'public' }); + }); + + it('an explicit logical key overrides the default and resolves by the same rule', async () => { + const [row] = await pg.any( + `SELECT bucket_id, resolved_key + FROM function_resolution.resolve_default_bucket($1, 'database', $1, false, 'avatars')`, + [TENANT_DB] + ); + expect(row).toEqual({ bucket_id: ids.avatars, resolved_key: 'avatars' }); + }); + + it('an explicit key nothing carries raises rather than falling back to the default', async () => { + await expect( + pg.any( + `SELECT * FROM function_resolution.resolve_default_bucket($1, 'database', $1, false, 'nope')`, + [TENANT_DB] + ) + ).rejects.toThrow(/STORAGE_DEFAULT_BUCKET_NOT_FOUND/); + }); + + it('a blank key is a caller bug, not a request for the default', async () => { + await expect( + pg.any( + `SELECT * FROM function_resolution.resolve_default_bucket($1, 'database', $1, false, ' ')`, + [TENANT_DB] + ) + ).rejects.toThrow(/STORAGE_BUCKET_KEY_BLANK/); + }); + + it('a database with no default bucket raises', async () => { + await expect( + pg.any( + `SELECT * FROM function_resolution.resolve_default_bucket($1, 'database', $1, false)`, + [BARE_DB] + ) + ).rejects.toThrow(/STORAGE_DEFAULT_BUCKET_NOT_FOUND/); + }); + + it('another tenant default is never borrowed', async () => { + // OTHER_DB has a bucket tagged 'default'; BARE_DB still has none. + const [foreign] = await pg.any( + `SELECT bucket_id FROM function_resolution.resolve_default_bucket($1, 'database', $1, false)`, + [OTHER_DB] + ); + expect(foreign.bucket_id).toBe(ids.otherDefault); + + await expect( + pg.any( + `SELECT * FROM function_resolution.resolve_default_bucket($1, 'database', $1, false)`, + [BARE_DB] + ) + ).rejects.toThrow(/STORAGE_DEFAULT_BUCKET_NOT_FOUND/); + }); + + it('two default buckets raise, naming the candidates', async () => { + let failure: (Error & { detail?: string }) | null = null; + try { + await pg.any( + `SELECT * FROM function_resolution.resolve_default_bucket($1, 'database', $1, false)`, + [AMBIGUOUS_DB] + ); + } catch (error) { + failure = error as Error & { detail?: string }; + } + + expect(failure).not.toBeNull(); + expect(failure!.message).toMatch(/STORAGE_DEFAULT_BUCKET_AMBIGUOUS/); + + const detail = JSON.parse(failure!.detail!); + expect(detail.code).toBe('STORAGE_DEFAULT_BUCKET_AMBIGUOUS'); + expect(detail.context.tag).toBe('default'); + expect(detail.context.candidates).toEqual([ + { bucket_id: ids.ambiguousOne, key: 'documents', type: 'private' }, + { bucket_id: ids.ambiguousTwo, key: 'uploads', type: 'private' }, + ]); + }); +}); diff --git a/packages/function-resolution/deploy/schemas/function_resolution/procedures/bucket_matches.sql b/packages/function-resolution/deploy/schemas/function_resolution/procedures/bucket_matches.sql new file mode 100644 index 00000000..2d3dca9f --- /dev/null +++ b/packages/function-resolution/deploy/schemas/function_resolution/procedures/bucket_matches.sql @@ -0,0 +1,94 @@ +-- Deploy schemas/function_resolution/procedures/bucket_matches to pg + +-- requires: schemas/function_resolution/schema +-- requires: schemas/function_resolution/procedures/frame_candidates + +BEGIN; + +-- bucket_matches: every bucket the nearest answering frame offers for a +-- {tags, type} selector — the lookup half of bucket resolution, with no opinion +-- about how many rows are acceptable. +-- +-- Split out so that "which buckets match" has exactly one implementation while +-- each caller states its own arity rule: resolve_bucket demands one for a +-- capability declaration, resolve_default_bucket demands one for a database's +-- default, and both raise on zero or several. A second copy of this query would +-- be a second answer to "which bucket is this". +-- +-- One static set-based query answers every frame at once. The shared plane holds +-- every logical database's rows, so each candidate carries the row's expected +-- database_id — the candidate's own key at database scope, the frame's lookup +-- database otherwise. Without it one tenant's probe could be answered by another +-- tenant's row. +-- +-- Candidates are probed most-specific frame first and only the nearest frame +-- that answered contributes rows: a nearer frame outranks an outer one, and ties +-- within that frame are the ambiguity a caller raises on. +-- +-- Cross-scope reach follows the catalog's own visibility rule: a bucket owned by +-- another database matches only when is_visible (propagated from the source +-- row's is_public), so an outer frame's private bucket is unreachable. +-- +-- A frame database without a buckets catalog simply contributes no candidates: +-- storage is an optional module, so its absence is not a provisioning error the +-- way a missing functions catalog is. +-- +-- plpgsql, not sql: this module is portable and deploys into databases that host +-- no catalog module, so catalog_private must be resolved on first call rather +-- than at CREATE FUNCTION time. +CREATE FUNCTION function_resolution.bucket_matches( + database_id uuid, + scope text, + entity_id uuid, + tags text[], + type_filter text DEFAULT NULL +) RETURNS TABLE ( + bucket_id uuid, + bucket_key text, + bucket_type text, + physical_name text, + owner_database_id uuid, + owner_scope text, + owner_key uuid +) AS $$ +BEGIN + RETURN QUERY + WITH hits AS ( + SELECT b.id, + b.key, + b.type, + b.physical_name, + b.database_id, + b.owner_scope, + b.owner_key, + cand.ord + FROM function_resolution.frame_candidates( + bucket_matches.database_id, + bucket_matches.scope, + bucket_matches.entity_id + ) cand + JOIN catalog_private.buckets b + ON b.owner_scope = cand.owner_scope + AND b.owner_key IS NOT DISTINCT FROM cand.owner_key + AND b.database_id = CASE + WHEN cand.owner_scope = 'database' THEN cand.owner_key + ELSE cand.lookup_database_id + END + WHERE b.tags @> bucket_matches.tags + AND (bucket_matches.type_filter IS NULL OR b.type = bucket_matches.type_filter) + AND (b.database_id = bucket_matches.database_id OR b.is_visible) + ) + SELECT h.id, + h.key, + h.type, + h.physical_name, + h.database_id, + h.owner_scope, + h.owner_key + FROM hits h + WHERE h.ord = (SELECT min(hh.ord) FROM hits hh) + ORDER BY h.id; +END; +$$ LANGUAGE plpgsql STABLE SECURITY DEFINER; + +COMMIT; diff --git a/packages/function-resolution/deploy/schemas/function_resolution/procedures/default_bucket_tag.sql b/packages/function-resolution/deploy/schemas/function_resolution/procedures/default_bucket_tag.sql new file mode 100644 index 00000000..fc65fe6c --- /dev/null +++ b/packages/function-resolution/deploy/schemas/function_resolution/procedures/default_bucket_tag.sql @@ -0,0 +1,26 @@ +-- Deploy schemas/function_resolution/procedures/default_bucket_tag to pg + +-- requires: schemas/function_resolution/schema + +BEGIN; + +-- default_bucket_tag: the reserved tag vocabulary for "the bucket this database +-- uses when nobody named one", in one place. +-- +-- A database's default bucket is not a column, a setting or a flag: it is a +-- bucket a tenant labelled, resolved by the same tag rule capabilities use. +-- Two reserved labels, because "the default" is two questions: +-- default -- the private default (presigned GET) +-- default-public -- the CDN-served default (public reads) +-- +-- Reserved only by convention plus provisioning: storage bootstrap applies them, +-- and resolution reads them. A tenant that retags its own buckets changes which +-- bucket is default, and a tenant that applies a label twice gets a loud +-- ambiguity rather than a silent winner. +CREATE FUNCTION function_resolution.default_bucket_tag( + public_access boolean +) RETURNS text AS $$ + SELECT CASE WHEN default_bucket_tag.public_access THEN 'default-public' ELSE 'default' END; +$$ LANGUAGE sql IMMUTABLE; + +COMMIT; diff --git a/packages/function-resolution/deploy/schemas/function_resolution/procedures/resolve_bucket.sql b/packages/function-resolution/deploy/schemas/function_resolution/procedures/resolve_bucket.sql index 6a8afa17..ae0f58b8 100644 --- a/packages/function-resolution/deploy/schemas/function_resolution/procedures/resolve_bucket.sql +++ b/packages/function-resolution/deploy/schemas/function_resolution/procedures/resolve_bucket.sql @@ -2,6 +2,7 @@ -- requires: schemas/function_resolution/schema -- requires: schemas/function_resolution/procedures/frame_candidates +-- requires: schemas/function_resolution/procedures/bucket_matches BEGIN; @@ -16,25 +17,15 @@ BEGIN; -- uploads / variants / exports are documentation, not DDL: nothing in the -- schema constrains a tenant to one bucket per tag. -- --- Determinism is enforced here rather than by a unique index: candidates are --- probed most-specific frame first, and the winning frame must answer with --- exactly one bucket. Zero matches and several matches both raise, and the +-- Determinism is enforced here rather than by a unique index: bucket_matches +-- probes candidates most-specific frame first, and the winning frame must answer +-- with exactly one bucket. Zero matches and several matches both raise, and the -- ambiguous error names the candidates so the fix (retag, narrow by type, or -- write an explicit capability binding) is obvious. -- --- Cross-scope reach follows the catalog's own visibility rule: a bucket owned --- by another database resolves only when is_visible (propagated from the --- source row's is_public), so an outer frame's private bucket is unreachable. --- --- One static set-based query answers every frame at once. The shared plane holds --- every logical database's rows, so each candidate carries the row's expected --- database_id — the candidate's own key at database scope, the frame's lookup --- database otherwise — exactly as resolve() does for functions. Without it one --- tenant's probe could be answered by another tenant's row. --- --- A frame database without a buckets catalog simply contributes no candidates: --- storage is an optional module, so its absence is not a provisioning error the --- way a missing functions catalog is. +-- The match itself — frames, same-tenant proof, cross-scope visibility, and the +-- absent-catalog case — lives in bucket_matches, so a capability declaration and +-- a database's default bucket are answered by one query under one arity rule. CREATE FUNCTION function_resolution.resolve_bucket( database_id uuid, scope text, @@ -63,39 +54,15 @@ BEGIN -- Every frame in one indexed read, keeping only the matches of the most -- specific frame that answered: a nearer frame outranks an outer one, and -- ties within that frame are the ambiguity raised below. - WITH hits AS ( - SELECT b.id, - b.key, - b.type, - b.physical_name, - b.database_id, - b.owner_scope, - b.owner_key, - cand.ord - FROM function_resolution.frame_candidates( - resolve_bucket.database_id, - resolve_bucket.scope, - resolve_bucket.entity_id - ) cand - JOIN catalog_private.buckets b - ON b.owner_scope = cand.owner_scope - AND b.owner_key IS NOT DISTINCT FROM cand.owner_key - AND b.database_id = CASE - WHEN cand.owner_scope = 'database' THEN cand.owner_key - ELSE cand.lookup_database_id - END - WHERE b.tags @> resolve_bucket.tags - AND (resolve_bucket.type_filter IS NULL OR b.type = resolve_bucket.type_filter) - AND (b.database_id = resolve_bucket.database_id OR b.is_visible) - ), - nearest AS ( - SELECT h.* - FROM hits h - WHERE h.ord = (SELECT min(hh.ord) FROM hits hh) - ) - SELECT COALESCE(jsonb_agg(to_jsonb(n) ORDER BY n.id), '[]'::jsonb) + SELECT COALESCE(jsonb_agg(to_jsonb(m) ORDER BY m.bucket_id), '[]'::jsonb) INTO v_matches - FROM nearest n; + FROM function_resolution.bucket_matches( + resolve_bucket.database_id, + resolve_bucket.scope, + resolve_bucket.entity_id, + resolve_bucket.tags, + resolve_bucket.type_filter + ) m; IF jsonb_array_length(v_matches) = 0 THEN RAISE EXCEPTION 'CAPABILITY_BUCKET_NOT_FOUND: no bucket tagged % % resolves in the scope chain starting at scope "%" (database_id=%)', @@ -111,18 +78,18 @@ BEGIN jsonb_array_length(v_matches), resolve_bucket.tags, COALESCE('of type ' || resolve_bucket.type_filter, '(any type)'), - (SELECT string_agg(format('%s (%s)', m->>'key', m->>'id'), ', ' ORDER BY m->>'key') + (SELECT string_agg(format('%s (%s)', m->>'bucket_key', m->>'bucket_id'), ', ' ORDER BY m->>'bucket_key') FROM jsonb_array_elements(v_matches) m) USING ERRCODE = 'FR012'; END IF; v_match := v_matches->0; - resolve_bucket.bucket_id := (v_match->>'id')::uuid; - resolve_bucket.bucket_key := v_match->>'key'; - resolve_bucket.bucket_type := v_match->>'type'; + resolve_bucket.bucket_id := (v_match->>'bucket_id')::uuid; + resolve_bucket.bucket_key := v_match->>'bucket_key'; + resolve_bucket.bucket_type := v_match->>'bucket_type'; resolve_bucket.physical_name := v_match->>'physical_name'; - resolve_bucket.owner_database_id := (v_match->>'database_id')::uuid; + resolve_bucket.owner_database_id := (v_match->>'owner_database_id')::uuid; resolve_bucket.owner_scope := v_match->>'owner_scope'; resolve_bucket.owner_key := (v_match->>'owner_key')::uuid; diff --git a/packages/function-resolution/deploy/schemas/function_resolution/procedures/resolve_default_bucket.sql b/packages/function-resolution/deploy/schemas/function_resolution/procedures/resolve_default_bucket.sql new file mode 100644 index 00000000..e911199a --- /dev/null +++ b/packages/function-resolution/deploy/schemas/function_resolution/procedures/resolve_default_bucket.sql @@ -0,0 +1,129 @@ +-- Deploy schemas/function_resolution/procedures/resolve_default_bucket to pg + +-- requires: schemas/function_resolution/schema +-- requires: schemas/function_resolution/procedures/bucket_matches +-- requires: schemas/function_resolution/procedures/default_bucket_tag + +BEGIN; + +-- resolve_default_bucket: answer "which bucket does this database store into +-- when the caller did not name one", server-side. +-- +-- Bucket choice belongs to the database, never to a client: a client-chosen +-- string means a different bucket per tenant, and an environment-level bucket +-- means storage that does not belong to any tenant at all. So there is no +-- global fallback here and no default that resolves outside the execution's own +-- frame chain — a database either labelled a default bucket or it has none, and +-- the second case raises. +-- +-- Two routes, one rule: +-- * bucket_key given -- the per-field override. A logical key is a label, the +-- same vocabulary a function's required_buckets declares, so the override +-- resolves through the identical match: nothing here is key-specific. +-- * bucket_key omitted -- the reserved default tag for the requested access +-- (see default_bucket_tag): 'default' or 'default-public'. +-- +-- Either way the match must be exactly one bucket. Zero raises, several raise +-- naming the candidates, and nothing is guessed in between: an ambiguous default +-- is a tenant's tagging mistake, and picking one would silently write a tenant's +-- files into whichever bucket sorted first. +CREATE FUNCTION function_resolution.resolve_default_bucket( + database_id uuid, + scope text, + entity_id uuid, + public_access boolean, + bucket_key text DEFAULT NULL +) RETURNS TABLE ( + bucket_id uuid, + resolved_key text, + bucket_type text, + physical_name text, + owner_database_id uuid, + owner_scope text, + owner_key uuid +) AS $$ +DECLARE + v_tag text; + v_matches jsonb; + v_match jsonb; +BEGIN + -- A blank override is a caller bug, not a request for the default: falling + -- through to tag resolution would turn a broken field binding into a + -- silently different bucket. + IF resolve_default_bucket.bucket_key IS NOT NULL + AND btrim(resolve_default_bucket.bucket_key) = '' THEN + PERFORM errors.raise_error( + 'STORAGE_BUCKET_KEY_BLANK', + jsonb_build_object( + 'database_id', resolve_default_bucket.database_id, + 'scope', resolve_default_bucket.scope + ), + 'internal' + ); + END IF; + + v_tag := COALESCE( + resolve_default_bucket.bucket_key, + function_resolution.default_bucket_tag(resolve_default_bucket.public_access) + ); + + SELECT COALESCE(jsonb_agg(to_jsonb(m) ORDER BY m.bucket_id), '[]'::jsonb) + INTO v_matches + FROM function_resolution.bucket_matches( + resolve_default_bucket.database_id, + resolve_default_bucket.scope, + resolve_default_bucket.entity_id, + ARRAY[v_tag] + ) m; + + IF jsonb_array_length(v_matches) = 0 THEN + PERFORM errors.raise_error( + 'STORAGE_DEFAULT_BUCKET_NOT_FOUND', + jsonb_build_object( + 'database_id', resolve_default_bucket.database_id, + 'scope', resolve_default_bucket.scope, + 'entity_id', resolve_default_bucket.entity_id, + 'tag', v_tag, + 'explicit_key', resolve_default_bucket.bucket_key IS NOT NULL + ), + 'internal' + ); + END IF; + + IF jsonb_array_length(v_matches) > 1 THEN + PERFORM errors.raise_error( + 'STORAGE_DEFAULT_BUCKET_AMBIGUOUS', + jsonb_build_object( + 'database_id', resolve_default_bucket.database_id, + 'scope', resolve_default_bucket.scope, + 'entity_id', resolve_default_bucket.entity_id, + 'tag', v_tag, + 'explicit_key', resolve_default_bucket.bucket_key IS NOT NULL, + 'candidates', ( + SELECT jsonb_agg(jsonb_build_object( + 'bucket_id', c->>'bucket_id', + 'key', c->>'bucket_key', + 'type', c->>'bucket_type' + ) ORDER BY c->>'bucket_key') + FROM jsonb_array_elements(v_matches) c + ) + ), + 'internal' + ); + END IF; + + v_match := v_matches->0; + + resolve_default_bucket.bucket_id := (v_match->>'bucket_id')::uuid; + resolve_default_bucket.resolved_key := v_match->>'bucket_key'; + resolve_default_bucket.bucket_type := v_match->>'bucket_type'; + resolve_default_bucket.physical_name := v_match->>'physical_name'; + resolve_default_bucket.owner_database_id := (v_match->>'owner_database_id')::uuid; + resolve_default_bucket.owner_scope := v_match->>'owner_scope'; + resolve_default_bucket.owner_key := (v_match->>'owner_key')::uuid; + + RETURN NEXT; +END; +$$ LANGUAGE plpgsql STABLE SECURITY DEFINER; + +COMMIT; diff --git a/packages/function-resolution/package.json b/packages/function-resolution/package.json index fd23019b..1661733f 100644 --- a/packages/function-resolution/package.json +++ b/packages/function-resolution/package.json @@ -26,6 +26,7 @@ "dependencies": { "@pgpm/app-scope": "workspace:*", "@pgpm/database-jobs": "workspace:*", + "@pgpm/errors": "workspace:*", "@pgpm/jwt-claims": "workspace:*", "@pgpm/metaschema-modules": "workspace:*", "@pgpm/metaschema-schema": "workspace:*", diff --git a/packages/function-resolution/pgpm-function-resolution.control b/packages/function-resolution/pgpm-function-resolution.control index 78023aba..18a4722b 100644 --- a/packages/function-resolution/pgpm-function-resolution.control +++ b/packages/function-resolution/pgpm-function-resolution.control @@ -2,6 +2,6 @@ comment = 'pgpm-function-resolution extension' default_version = '0.39.0' module_pathname = '$libdir/pgpm-function-resolution' -requires = 'plpgsql,pgpm-verify,metaschema-schema,metaschema-modules,pgpm-app-scope,pgpm-database-jobs,pgpm-jwt-claims' +requires = 'plpgsql,errors,pgpm-verify,metaschema-schema,metaschema-modules,pgpm-app-scope,pgpm-database-jobs,pgpm-jwt-claims' relocatable = false superuser = false diff --git a/packages/function-resolution/pgpm.plan b/packages/function-resolution/pgpm.plan index b98f1fd8..3c9a1435 100644 --- a/packages/function-resolution/pgpm.plan +++ b/packages/function-resolution/pgpm.plan @@ -9,7 +9,10 @@ schemas/function_resolution/procedures/resolve [schemas/function_resolution/sche schemas/function_resolution/procedures/resolve_invocation [schemas/function_resolution/schema schemas/function_resolution/procedures/resolve] 2017-08-11T08:11:51Z constructive # invocation-lane resolver schemas/function_resolution/procedures/enqueue [schemas/function_resolution/schema schemas/function_resolution/procedures/routing schemas/function_resolution/procedures/resolve] 2017-08-11T08:11:51Z constructive # resolver-aware enqueue entry point schemas/function_resolution/procedures/frame_candidates [schemas/function_resolution/schema] 2017-08-11T08:11:51Z constructive # frames expanded into catalog probe candidates -schemas/function_resolution/procedures/resolve_bucket [schemas/function_resolution/schema schemas/function_resolution/procedures/frame_candidates] 2017-08-11T08:11:51Z constructive # bucket selector {tags,type} resolution +schemas/function_resolution/procedures/bucket_matches [schemas/function_resolution/schema schemas/function_resolution/procedures/frame_candidates] 2017-08-11T08:11:51Z constructive # buckets a selector matches in the nearest frame +schemas/function_resolution/procedures/resolve_bucket [schemas/function_resolution/schema schemas/function_resolution/procedures/frame_candidates schemas/function_resolution/procedures/bucket_matches] 2017-08-11T08:11:51Z constructive # bucket selector {tags,type} resolution +schemas/function_resolution/procedures/default_bucket_tag [schemas/function_resolution/schema] 2017-08-11T08:11:51Z constructive # reserved default-bucket tag vocabulary +schemas/function_resolution/procedures/resolve_default_bucket [schemas/function_resolution/schema schemas/function_resolution/procedures/bucket_matches schemas/function_resolution/procedures/default_bucket_tag] 2017-08-11T08:11:51Z constructive # the database's default bucket, or an explicit key schemas/function_resolution/procedures/bucket_catalog_row [schemas/function_resolution/schema schemas/function_resolution/procedures/frame_candidates] 2017-08-11T08:11:51Z constructive # reachable bucket by id (same-tenant proof) schemas/function_resolution/procedures/api_catalog_row [schemas/function_resolution/schema schemas/function_resolution/procedures/frame_candidates] 2017-08-11T08:11:51Z constructive # reachable api by id schemas/function_resolution/procedures/resolve_api [schemas/function_resolution/schema schemas/function_resolution/procedures/frame_candidates schemas/function_resolution/procedures/api_catalog_row] 2017-08-11T08:11:51Z constructive # api selector module:/name: resolution diff --git a/packages/function-resolution/revert/schemas/function_resolution/procedures/bucket_matches.sql b/packages/function-resolution/revert/schemas/function_resolution/procedures/bucket_matches.sql new file mode 100644 index 00000000..83052d28 --- /dev/null +++ b/packages/function-resolution/revert/schemas/function_resolution/procedures/bucket_matches.sql @@ -0,0 +1,7 @@ +-- Revert schemas/function_resolution/procedures/bucket_matches from pg + +BEGIN; + +DROP FUNCTION function_resolution.bucket_matches(uuid, text, uuid, text[], text); + +COMMIT; diff --git a/packages/function-resolution/revert/schemas/function_resolution/procedures/default_bucket_tag.sql b/packages/function-resolution/revert/schemas/function_resolution/procedures/default_bucket_tag.sql new file mode 100644 index 00000000..2f943a65 --- /dev/null +++ b/packages/function-resolution/revert/schemas/function_resolution/procedures/default_bucket_tag.sql @@ -0,0 +1,7 @@ +-- Revert schemas/function_resolution/procedures/default_bucket_tag from pg + +BEGIN; + +DROP FUNCTION function_resolution.default_bucket_tag(boolean); + +COMMIT; diff --git a/packages/function-resolution/revert/schemas/function_resolution/procedures/resolve_default_bucket.sql b/packages/function-resolution/revert/schemas/function_resolution/procedures/resolve_default_bucket.sql new file mode 100644 index 00000000..088579fd --- /dev/null +++ b/packages/function-resolution/revert/schemas/function_resolution/procedures/resolve_default_bucket.sql @@ -0,0 +1,7 @@ +-- Revert schemas/function_resolution/procedures/resolve_default_bucket from pg + +BEGIN; + +DROP FUNCTION function_resolution.resolve_default_bucket(uuid, text, uuid, boolean, text); + +COMMIT; diff --git a/packages/function-resolution/sql/pgpm-function-resolution--0.39.0.bundle.tar.gz b/packages/function-resolution/sql/pgpm-function-resolution--0.39.0.bundle.tar.gz index 150d8b98..6d42e8a5 100644 Binary files a/packages/function-resolution/sql/pgpm-function-resolution--0.39.0.bundle.tar.gz and b/packages/function-resolution/sql/pgpm-function-resolution--0.39.0.bundle.tar.gz differ diff --git a/packages/function-resolution/sql/pgpm-function-resolution--0.39.0.sql b/packages/function-resolution/sql/pgpm-function-resolution--0.39.0.sql index b11f8e16..8a74261a 100644 --- a/packages/function-resolution/sql/pgpm-function-resolution--0.39.0.sql +++ b/packages/function-resolution/sql/pgpm-function-resolution--0.39.0.sql @@ -427,7 +427,7 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql STABLE SECURITY DEFINER; -CREATE FUNCTION function_resolution.resolve_bucket( +CREATE FUNCTION function_resolution.bucket_matches( database_id uuid, scope text, entity_id uuid, @@ -442,19 +442,8 @@ CREATE FUNCTION function_resolution.resolve_bucket( owner_scope text, owner_key uuid ) AS $EOFCODE$ -DECLARE - v_matches jsonb; - v_match jsonb; BEGIN - IF resolve_bucket.tags IS NULL OR cardinality(resolve_bucket.tags) = 0 THEN - RAISE EXCEPTION 'CAPABILITY_BUCKET_SELECTOR_EMPTY: a bucket selector needs at least one tag (database_id=%, scope="%")', - resolve_bucket.database_id, resolve_bucket.scope - USING ERRCODE = 'FR010'; - END IF; - - -- Every frame in one indexed read, keeping only the matches of the most - -- specific frame that answered: a nearer frame outranks an outer one, and - -- ties within that frame are the ambiguity raised below. + RETURN QUERY WITH hits AS ( SELECT b.id, b.key, @@ -465,9 +454,9 @@ BEGIN b.owner_key, cand.ord FROM function_resolution.frame_candidates( - resolve_bucket.database_id, - resolve_bucket.scope, - resolve_bucket.entity_id + bucket_matches.database_id, + bucket_matches.scope, + bucket_matches.entity_id ) cand JOIN catalog_private.buckets b ON b.owner_scope = cand.owner_scope @@ -476,18 +465,60 @@ BEGIN WHEN cand.owner_scope = 'database' THEN cand.owner_key ELSE cand.lookup_database_id END - WHERE b.tags @> resolve_bucket.tags - AND (resolve_bucket.type_filter IS NULL OR b.type = resolve_bucket.type_filter) - AND (b.database_id = resolve_bucket.database_id OR b.is_visible) - ), - nearest AS ( - SELECT h.* - FROM hits h - WHERE h.ord = (SELECT min(hh.ord) FROM hits hh) + WHERE b.tags @> bucket_matches.tags + AND (bucket_matches.type_filter IS NULL OR b.type = bucket_matches.type_filter) + AND (b.database_id = bucket_matches.database_id OR b.is_visible) ) - SELECT COALESCE(jsonb_agg(to_jsonb(n) ORDER BY n.id), '[]'::jsonb) + SELECT h.id, + h.key, + h.type, + h.physical_name, + h.database_id, + h.owner_scope, + h.owner_key + FROM hits h + WHERE h.ord = (SELECT min(hh.ord) FROM hits hh) + ORDER BY h.id; +END; +$EOFCODE$ LANGUAGE plpgsql STABLE SECURITY DEFINER; + +CREATE FUNCTION function_resolution.resolve_bucket( + database_id uuid, + scope text, + entity_id uuid, + tags text[], + type_filter text DEFAULT NULL +) RETURNS TABLE ( + bucket_id uuid, + bucket_key text, + bucket_type text, + physical_name text, + owner_database_id uuid, + owner_scope text, + owner_key uuid +) AS $EOFCODE$ +DECLARE + v_matches jsonb; + v_match jsonb; +BEGIN + IF resolve_bucket.tags IS NULL OR cardinality(resolve_bucket.tags) = 0 THEN + RAISE EXCEPTION 'CAPABILITY_BUCKET_SELECTOR_EMPTY: a bucket selector needs at least one tag (database_id=%, scope="%")', + resolve_bucket.database_id, resolve_bucket.scope + USING ERRCODE = 'FR010'; + END IF; + + -- Every frame in one indexed read, keeping only the matches of the most + -- specific frame that answered: a nearer frame outranks an outer one, and + -- ties within that frame are the ambiguity raised below. + SELECT COALESCE(jsonb_agg(to_jsonb(m) ORDER BY m.bucket_id), '[]'::jsonb) INTO v_matches - FROM nearest n; + FROM function_resolution.bucket_matches( + resolve_bucket.database_id, + resolve_bucket.scope, + resolve_bucket.entity_id, + resolve_bucket.tags, + resolve_bucket.type_filter + ) m; IF jsonb_array_length(v_matches) = 0 THEN RAISE EXCEPTION 'CAPABILITY_BUCKET_NOT_FOUND: no bucket tagged % % resolves in the scope chain starting at scope "%" (database_id=%)', @@ -503,18 +534,18 @@ BEGIN jsonb_array_length(v_matches), resolve_bucket.tags, COALESCE('of type ' || resolve_bucket.type_filter, '(any type)'), - (SELECT string_agg(format('%s (%s)', m->>'key', m->>'id'), ', ' ORDER BY m->>'key') + (SELECT string_agg(format('%s (%s)', m->>'bucket_key', m->>'bucket_id'), ', ' ORDER BY m->>'bucket_key') FROM jsonb_array_elements(v_matches) m) USING ERRCODE = 'FR012'; END IF; v_match := v_matches->0; - resolve_bucket.bucket_id := (v_match->>'id')::uuid; - resolve_bucket.bucket_key := v_match->>'key'; - resolve_bucket.bucket_type := v_match->>'type'; + resolve_bucket.bucket_id := (v_match->>'bucket_id')::uuid; + resolve_bucket.bucket_key := v_match->>'bucket_key'; + resolve_bucket.bucket_type := v_match->>'bucket_type'; resolve_bucket.physical_name := v_match->>'physical_name'; - resolve_bucket.owner_database_id := (v_match->>'database_id')::uuid; + resolve_bucket.owner_database_id := (v_match->>'owner_database_id')::uuid; resolve_bucket.owner_scope := v_match->>'owner_scope'; resolve_bucket.owner_key := (v_match->>'owner_key')::uuid; @@ -522,6 +553,111 @@ BEGIN END; $EOFCODE$ LANGUAGE plpgsql STABLE SECURITY DEFINER; +CREATE FUNCTION function_resolution.default_bucket_tag( + public_access boolean +) RETURNS text AS $EOFCODE$ + SELECT CASE WHEN default_bucket_tag.public_access THEN 'default-public' ELSE 'default' END; +$EOFCODE$ LANGUAGE sql IMMUTABLE; + +CREATE FUNCTION function_resolution.resolve_default_bucket( + database_id uuid, + scope text, + entity_id uuid, + public_access boolean, + bucket_key text DEFAULT NULL +) RETURNS TABLE ( + bucket_id uuid, + resolved_key text, + bucket_type text, + physical_name text, + owner_database_id uuid, + owner_scope text, + owner_key uuid +) AS $EOFCODE$ +DECLARE + v_tag text; + v_matches jsonb; + v_match jsonb; +BEGIN + -- A blank override is a caller bug, not a request for the default: falling + -- through to tag resolution would turn a broken field binding into a + -- silently different bucket. + IF resolve_default_bucket.bucket_key IS NOT NULL + AND btrim(resolve_default_bucket.bucket_key) = '' THEN + PERFORM errors.raise_error( + 'STORAGE_BUCKET_KEY_BLANK', + jsonb_build_object( + 'database_id', resolve_default_bucket.database_id, + 'scope', resolve_default_bucket.scope + ), + 'internal' + ); + END IF; + + v_tag := COALESCE( + resolve_default_bucket.bucket_key, + function_resolution.default_bucket_tag(resolve_default_bucket.public_access) + ); + + SELECT COALESCE(jsonb_agg(to_jsonb(m) ORDER BY m.bucket_id), '[]'::jsonb) + INTO v_matches + FROM function_resolution.bucket_matches( + resolve_default_bucket.database_id, + resolve_default_bucket.scope, + resolve_default_bucket.entity_id, + ARRAY[v_tag] + ) m; + + IF jsonb_array_length(v_matches) = 0 THEN + PERFORM errors.raise_error( + 'STORAGE_DEFAULT_BUCKET_NOT_FOUND', + jsonb_build_object( + 'database_id', resolve_default_bucket.database_id, + 'scope', resolve_default_bucket.scope, + 'entity_id', resolve_default_bucket.entity_id, + 'tag', v_tag, + 'explicit_key', resolve_default_bucket.bucket_key IS NOT NULL + ), + 'internal' + ); + END IF; + + IF jsonb_array_length(v_matches) > 1 THEN + PERFORM errors.raise_error( + 'STORAGE_DEFAULT_BUCKET_AMBIGUOUS', + jsonb_build_object( + 'database_id', resolve_default_bucket.database_id, + 'scope', resolve_default_bucket.scope, + 'entity_id', resolve_default_bucket.entity_id, + 'tag', v_tag, + 'explicit_key', resolve_default_bucket.bucket_key IS NOT NULL, + 'candidates', ( + SELECT jsonb_agg(jsonb_build_object( + 'bucket_id', c->>'bucket_id', + 'key', c->>'bucket_key', + 'type', c->>'bucket_type' + ) ORDER BY c->>'bucket_key') + FROM jsonb_array_elements(v_matches) c + ) + ), + 'internal' + ); + END IF; + + v_match := v_matches->0; + + resolve_default_bucket.bucket_id := (v_match->>'bucket_id')::uuid; + resolve_default_bucket.resolved_key := v_match->>'bucket_key'; + resolve_default_bucket.bucket_type := v_match->>'bucket_type'; + resolve_default_bucket.physical_name := v_match->>'physical_name'; + resolve_default_bucket.owner_database_id := (v_match->>'owner_database_id')::uuid; + resolve_default_bucket.owner_scope := v_match->>'owner_scope'; + resolve_default_bucket.owner_key := (v_match->>'owner_key')::uuid; + + RETURN NEXT; +END; +$EOFCODE$ LANGUAGE plpgsql STABLE SECURITY DEFINER; + CREATE FUNCTION function_resolution.bucket_catalog_row( database_id uuid, scope text, diff --git a/packages/function-resolution/verify/schemas/function_resolution/procedures/bucket_matches.sql b/packages/function-resolution/verify/schemas/function_resolution/procedures/bucket_matches.sql new file mode 100644 index 00000000..ab922637 --- /dev/null +++ b/packages/function-resolution/verify/schemas/function_resolution/procedures/bucket_matches.sql @@ -0,0 +1,7 @@ +-- Verify schemas/function_resolution/procedures/bucket_matches on pg + +BEGIN; + +SELECT assert_function('function_resolution.bucket_matches(uuid, text, uuid, text[], text)'::regprocedure); + +ROLLBACK; diff --git a/packages/function-resolution/verify/schemas/function_resolution/procedures/default_bucket_tag.sql b/packages/function-resolution/verify/schemas/function_resolution/procedures/default_bucket_tag.sql new file mode 100644 index 00000000..ab4e8c08 --- /dev/null +++ b/packages/function-resolution/verify/schemas/function_resolution/procedures/default_bucket_tag.sql @@ -0,0 +1,7 @@ +-- Verify schemas/function_resolution/procedures/default_bucket_tag on pg + +BEGIN; + +SELECT assert_function('function_resolution.default_bucket_tag(boolean)'::regprocedure); + +ROLLBACK; diff --git a/packages/function-resolution/verify/schemas/function_resolution/procedures/resolve_default_bucket.sql b/packages/function-resolution/verify/schemas/function_resolution/procedures/resolve_default_bucket.sql new file mode 100644 index 00000000..3ed76ac8 --- /dev/null +++ b/packages/function-resolution/verify/schemas/function_resolution/procedures/resolve_default_bucket.sql @@ -0,0 +1,7 @@ +-- Verify schemas/function_resolution/procedures/resolve_default_bucket on pg + +BEGIN; + +SELECT assert_function('function_resolution.resolve_default_bucket(uuid, text, uuid, boolean, text)'::regprocedure); + +ROLLBACK; diff --git a/packages/object-store/deploy/schemas/object_store_public/procedures/insert_nodes_at_paths.sql b/packages/object-store/deploy/schemas/object_store_public/procedures/insert_nodes_at_paths.sql index 2bca0171..6df26acc 100644 --- a/packages/object-store/deploy/schemas/object_store_public/procedures/insert_nodes_at_paths.sql +++ b/packages/object-store/deploy/schemas/object_store_public/procedures/insert_nodes_at_paths.sql @@ -187,31 +187,38 @@ BEGIN max_depth; -- 3. resolve each dirty directory against the pre-existing tree, walking down - -- from the current root so untouched siblings survive the rebuild + -- from the current root so untouched siblings survive the rebuild. + -- + -- The descent joins a directory to its parent on the parent's key, which + -- every directory already carries: an equijoin the planner hashes once per + -- level. Recursing on depth instead and matching with a path-prefix filter + -- (`child.path[1:r.depth] = r.path`) is the same walk but not a join + -- condition, so every dirty directory is compared against every row of the + -- level above — 9.6M filtered comparisons per level on a 22k-directory + -- batch, each re-deriving the path from the key, which is 300s of a 307s + -- pass against 0.4s for this one. + -- + -- The root is the descent's seed, not a child: it is its own parent under + -- `to_jsonb(path[1:0])`, so leaving it in the recursive side joins it to + -- itself forever. WITH RECURSIVE dirs AS ( SELECT dr.node_key, - dr.depth, - ARRAY ( - SELECT - jsonb_array_elements_text(dr.node_key::jsonb))::text[] AS path - FROM unnest(d_key, d_depth) AS dr (node_key, depth) + dr.name, + dr.parent + FROM unnest(d_key, d_name, d_parent) AS dr (node_key, name, parent) + WHERE dr.node_key <> root_key ), resolved AS ( SELECT root_key AS node_key, - 0 AS depth, - ARRAY[]::text[] AS path, insert_nodes_at_paths.root AS node_id UNION ALL SELECT child.node_key, - child.depth, - child.path, - parent_obj.kids[object_store_utils.array_index_of (parent_obj.ktree, child.path[child.depth])] + parent_obj.kids[object_store_utils.array_index_of (parent_obj.ktree, child.name)] FROM resolved AS r - JOIN dirs AS child ON child.depth = r.depth + 1 - AND child.path[1:r.depth] = r.path + JOIN dirs AS child ON child.parent = r.node_key LEFT JOIN object_store_public.object AS parent_obj ON parent_obj.id = r.node_id AND parent_obj.scope_id = insert_nodes_at_paths.s_id ) diff --git a/packages/object-store/sql/object-store--0.39.0.bundle.tar.gz b/packages/object-store/sql/object-store--0.39.0.bundle.tar.gz index c7402b60..1734bec9 100644 Binary files a/packages/object-store/sql/object-store--0.39.0.bundle.tar.gz and b/packages/object-store/sql/object-store--0.39.0.bundle.tar.gz differ diff --git a/packages/object-store/sql/object-store--0.39.0.sql b/packages/object-store/sql/object-store--0.39.0.sql index 26dcf3ec..7dae5626 100644 --- a/packages/object-store/sql/object-store--0.39.0.sql +++ b/packages/object-store/sql/object-store--0.39.0.sql @@ -713,31 +713,38 @@ BEGIN max_depth; -- 3. resolve each dirty directory against the pre-existing tree, walking down - -- from the current root so untouched siblings survive the rebuild + -- from the current root so untouched siblings survive the rebuild. + -- + -- The descent joins a directory to its parent on the parent's key, which + -- every directory already carries: an equijoin the planner hashes once per + -- level. Recursing on depth instead and matching with a path-prefix filter + -- (`child.path[1:r.depth] = r.path`) is the same walk but not a join + -- condition, so every dirty directory is compared against every row of the + -- level above — 9.6M filtered comparisons per level on a 22k-directory + -- batch, each re-deriving the path from the key, which is 300s of a 307s + -- pass against 0.4s for this one. + -- + -- The root is the descent's seed, not a child: it is its own parent under + -- `to_jsonb(path[1:0])`, so leaving it in the recursive side joins it to + -- itself forever. WITH RECURSIVE dirs AS ( SELECT dr.node_key, - dr.depth, - ARRAY ( - SELECT - jsonb_array_elements_text(dr.node_key::jsonb))::text[] AS path - FROM unnest(d_key, d_depth) AS dr (node_key, depth) + dr.name, + dr.parent + FROM unnest(d_key, d_name, d_parent) AS dr (node_key, name, parent) + WHERE dr.node_key <> root_key ), resolved AS ( SELECT root_key AS node_key, - 0 AS depth, - ARRAY[]::text[] AS path, insert_nodes_at_paths.root AS node_id UNION ALL SELECT child.node_key, - child.depth, - child.path, - parent_obj.kids[object_store_utils.array_index_of (parent_obj.ktree, child.path[child.depth])] + parent_obj.kids[object_store_utils.array_index_of (parent_obj.ktree, child.name)] FROM resolved AS r - JOIN dirs AS child ON child.depth = r.depth + 1 - AND child.path[1:r.depth] = r.path + JOIN dirs AS child ON child.parent = r.node_key LEFT JOIN object_store_public.object AS parent_obj ON parent_obj.id = r.node_id AND parent_obj.scope_id = insert_nodes_at_paths.s_id ) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8349ea3d..398c94be 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -181,6 +181,9 @@ importers: '@pgpm/database-jobs': specifier: workspace:* version: link:../database-jobs + '@pgpm/errors': + specifier: workspace:* + version: link:../errors '@pgpm/jwt-claims': specifier: workspace:* version: link:../jwt-claims