Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ exports[`Export Flow E2E Export to second workspace should match snapshot for se
"migrate/apis.sql",
"migrate/app_module.sql",
"migrate/catalog_module.sql",
"migrate/catalog_private.apis.sql",
"migrate/database.sql",
"migrate/database_settings_module.sql",
"migrate/domain_module.sql",
Expand Down
52 changes: 52 additions & 0 deletions pgpm/export/src/catalog-projection.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import { Parser } from 'csv-to-pg';

/**
* Catalog plane projection: catalog_private.apis is trigger-derived from
* routing_public.apis by catalog_private.tg_apis_catalog_sync(). During
* migration replay the sync trigger is skipped (session_replication_role),
* and the catalog tables cannot be queried through the meta API (bare-name
* collisions with routing_public) — so the projection is materialized at
* export time, mirroring the trigger mapping 1:1. resolve_route() needs
* these rows to build resolved_config for api targets.
*
* Both the SQL and GraphQL export flows run this projection so their output
* stays byte-identical (cross-flow parity).
*/
export const projectCatalogApis = async (
apisRows: Record<string, unknown>[]
): Promise<string | undefined> => {
if (!apisRows.length) return undefined;

const projected = apisRows.map((r) => ({
id: r.id,
owner_scope: 'database',
owner_key: r.database_id,
is_visible: r.is_published ?? false,
database_id: r.database_id,
name: r.name,
dbname: r.dbname,
role_name: r.role_name,
anon_role: r.anon_role,
config: r.config ?? null
}));

const parser = new Parser({
schema: 'catalog_private',
table: 'apis',
fields: {
id: 'uuid',
owner_scope: 'text',
owner_key: 'uuid',
is_visible: 'boolean',
database_id: 'uuid',
name: 'text',
dbname: 'text',
role_name: 'text',
anon_role: 'text',
config: 'jsonb'
}
});

const parsed = await parser.parse(projected);
return parsed || undefined;
};
41 changes: 38 additions & 3 deletions pgpm/export/src/export-graphql-meta.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import { Parser } from 'csv-to-pg';
import { toSnakeCase } from 'inflekt';

import { projectCatalogApis } from './catalog-projection';
import { FieldType, getTimestampDefaultColumnsForTable, META_TABLE_CONFIG, META_TABLE_ORDER, TableConfig } from './export-utils';
import { GraphQLClient } from './graphql-client';
import {
Expand Down Expand Up @@ -144,6 +145,23 @@ export const exportGraphQLMeta = async ({
database_id
}: ExportGraphQLMetaParams): Promise<ExportGraphQLMetaResult> => {
const sql: Record<string, string> = {};
// Raw rows per key (post GraphQL→Postgres conversion), kept for derived
// projections (see catalog plane derivation below).
const rawRows: Record<string, Record<string, unknown>[]> = {};

// Binding tables (hostname_bindings, route_bindings) carry no database_id;
// tenant ownership flows through domain_id → routing_public.domains. Fetch
// the tenant's domain ids once so those keys can be filtered by
// domainId IN (...).
let domainIds: string[] = [];
if (META_TABLE_ORDER.some((k) => META_TABLE_CONFIG[k]?.filterViaDomainIds)) {
const domainRows = await client.fetchAllNodes<{ id: string }>(
getGraphQLQueryName('domains'),
'id',
{ databaseId: database_id }
);
domainIds = domainRows.map((r) => r.id);
}

const queryAndParse = async (key: string) => {
const tableConfig = META_TABLE_CONFIG[key];
Expand All @@ -152,8 +170,7 @@ export const exportGraphQLMeta = async ({
// Schema-qualified manifest keys (e.g. catalog_private.apis)
// mark tables whose name collides with a table in another plane. GraphQL
// type/query names are derived from the bare table name, so these cannot
// be addressed unambiguously through the meta API — only the SQL flow
// exports them.
// be addressed unambiguously through the meta API in a mixed build.
if (key.includes('.')) return;

// Build fields dynamically: either from hardcoded config or via introspection
Expand All @@ -167,7 +184,9 @@ export const exportGraphQLMeta = async ({
// The 'database' table is fetched by id, not by database_id
const condition = key === 'database'
? { id: database_id }
: { databaseId: database_id };
: tableConfig.filterViaDomainIds
? { domainId: domainIds }
: { databaseId: database_id };

try {
const rows = await client.fetchAllNodes(
Expand Down Expand Up @@ -225,6 +244,8 @@ export const exportGraphQLMeta = async ({

if (Object.keys(dynamicFields).length === 0) return;

rawRows[key] = pgRows;

// Omit columnDefaults columns from row data so the Parser never sees them.
// configFields already excludes them (via buildDynamicFieldsFromGraphQL),
// so dynamicFields won't contain them either — but the pgRow data still does.
Expand Down Expand Up @@ -279,5 +300,19 @@ export const exportGraphQLMeta = async ({
await Promise.all(keys.map(key => queryAndParse(key)));
}

// Catalog plane projection: catalog_private.apis is trigger-derived from
// routing_public.apis (catalog_private.tg_apis_catalog_sync). The sync
// trigger is skipped during migration replay (session_replication_role),
// and the catalog tables can't be queried through the meta API (bare-name
// collisions with routing_public) — so the projection is materialized here.
// Shared with the SQL flow (see catalog-projection.ts) for cross-flow parity.
const apisRows = rawRows['apis'];
if (sql['apis'] && apisRows?.length) {
const parsed = await projectCatalogApis(apisRows);
if (parsed) {
sql['catalog_private.apis'] = parsed;
}
}

return sql;
};
29 changes: 27 additions & 2 deletions pgpm/export/src/export-meta.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { Parser } from 'csv-to-pg';
import type { Pool } from 'pg';
import { getPgPool } from 'pg-cache';

import { projectCatalogApis } from './catalog-projection';
import { FieldType, getTableColumnsWithDefaults, isTimestampDefaultColumn,mapPgTypeToFieldType, META_TABLE_CONFIG, META_TABLE_ORDER, TableConfig } from './export-utils';

/**
Expand Down Expand Up @@ -84,6 +85,9 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams
database: dbname
});
const sql: Record<string, string> = {};
// Raw rows per key (post column-default stripping), kept for derived
// projections (see catalog plane projection below).
const rawRows: Record<string, Record<string, unknown>[]> = {};

// Cache for dynamically built parsers and their field configs
const parsers: Record<string, Parser> = {};
Expand Down Expand Up @@ -160,6 +164,8 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams
}
}

rawRows[key] = result.rows;

const parsed = await parser.parse(result.rows);
if (parsed) {
sql[key] = parsed;
Expand All @@ -179,8 +185,27 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams
// itself, which is keyed by id.
for (const key of META_TABLE_ORDER) {
const tableConfig = META_TABLE_CONFIG[key];
const filterColumn = key === 'database' ? 'id' : 'database_id';
await queryAndParse(key, `SELECT * FROM ${tableConfig.schema}.${tableConfig.table} WHERE ${filterColumn} = $1 ORDER BY id`);
// Binding tables (hostname_bindings, route_bindings) carry no database_id;
// tenant ownership flows through domain_id → routing_public.domains.
const filterSql = key === 'database'
? 'id = $1'
: tableConfig.filterViaDomainIds
? 'domain_id IN (SELECT id FROM routing_public.domains WHERE database_id = $1)'
: 'database_id = $1';
await queryAndParse(key, `SELECT * FROM ${tableConfig.schema}.${tableConfig.table} WHERE ${filterSql} ORDER BY id`);
}

// Catalog plane projection: catalog_private.apis is trigger-derived from
// routing_public.apis (catalog_private.tg_apis_catalog_sync). The sync
// trigger is skipped during migration replay (session_replication_role),
// so the projection is materialized here. Shared with the GraphQL flow
// (see catalog-projection.ts) for cross-flow parity.
const apisRows = rawRows['apis'];
if (sql['apis'] && apisRows?.length) {
const parsed = await projectCatalogApis(apisRows);
if (parsed) {
sql['catalog_private.apis'] = parsed;
}
}

return sql;
Expand Down
13 changes: 13 additions & 0 deletions pgpm/export/src/export-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,13 @@ export interface TableConfig {
conflictDoNothing?: boolean;
typeOverrides?: Record<string, FieldType>; // only for special types (image, upload, url) that can't be inferred
gqlTypeName?: string; // override for GraphQL type name when automatic derivation doesn't match PostGraphile's inflector
/**
* Table has no database_id column; rows belong to a tenant through their
* domain_id FK (e.g. hostname_bindings, route_bindings). The export filters
* them by `domainId in (<tenant's domain ids>)` — the tenant's domain ids
* are pre-fetched once per export run.
*/
filterViaDomainIds?: boolean;
/** Columns whose values are environment-specific and should be excluded from the
* exported INSERT so that the column's DDL DEFAULT applies at deploy time.
* Key = column name, Value = the SQL expression the column defaults to (for documentation).
Expand Down Expand Up @@ -205,6 +212,12 @@ export interface MetaExportTableEntry {
* values must come from DDL defaults at deploy time.
*/
export const META_TABLE_OVERRIDES: Record<string, Omit<TableConfig, 'schema' | 'table'>> = {
hostname_bindings: {
filterViaDomainIds: true
},
route_bindings: {
filterViaDomainIds: true
},
sites: {
typeOverrides: {
og_image: 'image',
Expand Down
1 change: 1 addition & 0 deletions pgpm/export/src/graphql-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ export class GraphQLClient {
if (condition && Object.keys(condition).length > 0) {
const filterParts = Object.entries(condition)
.map(([k, v]) => {
if (Array.isArray(v)) return `${k}: { in: [${v.map((item) => `"${item}"`).join(', ')}] }`;
if (typeof v === 'string') return `${k}: { equalTo: "${v}" }`;
if (typeof v === 'boolean') return `${k}: { equalTo: ${v} }`;
if (typeof v === 'number') return `${k}: { equalTo: ${v} }`;
Expand Down
21 changes: 16 additions & 5 deletions pgpm/export/src/meta-export-tables.json
Original file line number Diff line number Diff line change
Expand Up @@ -591,6 +591,16 @@
"schema": "routing_public",
"table": "routes"
},
{
"key": "hostname_bindings",
"schema": "routing_public",
"table": "hostname_bindings"
},
{
"key": "route_bindings",
"schema": "routing_public",
"table": "route_bindings"
},
{
"key": "site_app_links",
"schema": "routing_public",
Expand Down Expand Up @@ -671,11 +681,6 @@
"schema": "catalog_private",
"table": "domains"
},
{
"key": "functions",
"schema": "catalog_private",
"table": "functions"
},
{
"key": "namespaces",
"schema": "catalog_private",
Expand Down Expand Up @@ -957,6 +962,12 @@
"created_at",
"updated_at"
],
"routing_public.hostname_bindings": [
"updated_at"
],
"routing_public.route_bindings": [
"updated_at"
],
"routing_public.site_app_links": [
"created_at",
"updated_at"
Expand Down
Loading