From cc927f4f7796d68453488ee0351e0eab0dbc13ac Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Mon, 17 Aug 2026 11:04:15 -0400 Subject: [PATCH 1/5] ENG-1867-step1-crossAppToDb --- .../src/lib/dbToCrossAppConverters.ts | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 packages/database/src/lib/dbToCrossAppConverters.ts diff --git a/packages/database/src/lib/dbToCrossAppConverters.ts b/packages/database/src/lib/dbToCrossAppConverters.ts new file mode 100644 index 000000000..68e83dff5 --- /dev/null +++ b/packages/database/src/lib/dbToCrossAppConverters.ts @@ -0,0 +1,397 @@ +import { + CrossAppNodeSchema, + CrossAppRelationTypeSchema, + CrossAppRelationTripleSchema, + CrossAppRelation, +} from "../crossAppContracts"; +import { Tables, Json } from "../dbTypes"; +import type { DGSupabaseClient } from "./client"; +import { ridToSpaceUriAndLocalId, spaceUriAndLocalIdToRid, isRid } from "./rid"; + +type Concept = Tables<"Concept">; + +const getConceptMap = async ( + client: DGSupabaseClient, + conceptIds: number[], + spaceMap: Record, +): Promise> => { + const request = await client + .from("my_concepts") + .select("id, space_id, source_local_id") + .in("id", conceptIds) + .not("source_local_id", "is", null); + if (request.error) throw request.error; + return Object.fromEntries( + (request.data || []) + .map(({ id, source_local_id, space_id }) => { + const spaceUri: string | undefined = spaceMap[space_id ?? 0]; + return [ + id!, + spaceUri !== undefined + ? spaceUriAndLocalIdToRid(spaceUri, source_local_id) + : undefined, + ]; + }) + .filter(([, rid]) => rid !== undefined) as [number, string][], + ); +}; + +export const getAccountMap = async ( + client: DGSupabaseClient, + accountIds: number[], +): Promise> => { + const request = await client + .from("my_accounts") + .select("id,account_local_id") + .in("id", accountIds); + if (request.error) throw request.error; + return Object.fromEntries( + (request.data || []).map(({ id, account_local_id }) => [ + id!, + account_local_id!, + ]), + ); +}; + +export const getSpaceMap = async ( + client: DGSupabaseClient, + spaceIds?: number[], +): Promise> => { + let query = client.from("my_spaces").select("id, url"); + if (spaceIds !== undefined) query = query.in("id", [...spaceIds]); + + const { data, error } = await query; + if (error || !data) { + throw error; + } + return Object.fromEntries(data.map(({ id, url }) => [id!, url!])); +}; + +const asSimpleLocalId = ( + rid: string | undefined, + spaceUrl: string | undefined, + optional?: boolean, +): string | undefined => { + if (rid === undefined) return undefined; + if (!isRid(rid)) return rid; + const { spaceUri, sourceLocalId } = ridToSpaceUriAndLocalId(rid); + if (spaceUrl === spaceUri) return sourceLocalId; + if (optional !== true) throw new Error("Unexpected spaceUri"); + return rid; +}; + +export const dbNodeSchemaToCrossApp = ( + schema: Concept, + spaceMap: Record, + accountMap: Record, +): CrossAppNodeSchema => { + const { template, template_content, ...other } = + schema.literal_content as Record; + const authorId = accountMap[schema.author_id || 0]; + if (authorId === undefined) throw new Error("Missing author"); + const spaceUrl = spaceMap[schema.space_id]; + if (spaceUrl === undefined) throw new Error("Missing space"); + const rid = spaceUriAndLocalIdToRid( + spaceUrl, + schema.source_local_id!, + "schema", + ); + return { + rid, + localId: schema.source_local_id!, + createdAt: new Date(schema.created + "Z"), + modifiedAt: new Date(schema.last_modified + "Z"), + label: schema.name, + metadata: other, + template: template_content as string | undefined, + templateTitle: template as string | undefined, + authorId, + }; +}; + +export const dbNodeSchemasToCrossApp = async ({ + client, + schemas, + spaceMap, + accountMap, +}: { + client: DGSupabaseClient; + schemas: Concept[]; + spaceMap?: Record; + accountMap?: Record; +}): Promise => { + if (spaceMap === undefined) spaceMap = await getSpaceMap(client); + if (accountMap === undefined) { + const authorIds = new Set( + schemas.map((r) => r.author_id).filter((id) => typeof id === "number"), + ); + accountMap = await getAccountMap(client, [...authorIds]); + } + return schemas.map((r) => dbNodeSchemaToCrossApp(r, spaceMap, accountMap)); +}; + +export const dbRelationTypeSchemaToCrossApp = ( + schema: Concept, + spaceMap: Record, + accountMap: Record, +): CrossAppRelationTypeSchema => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { roles, label, complement, ...other } = + schema.literal_content as Record; + const authorId = accountMap[schema.author_id || 0]; + if (authorId === undefined) throw new Error("Missing author"); + const spaceUrl = spaceMap[schema.space_id]; + if (spaceUrl === undefined) throw new Error("Missing space"); + const rid = spaceUriAndLocalIdToRid( + spaceUrl, + schema.source_local_id!, + "schema", + ); + return { + rid, + localId: schema.source_local_id!, + createdAt: new Date(schema.created + "Z"), + modifiedAt: new Date(schema.last_modified + "Z"), + metadata: other, + label: label as string, + complement: complement as string, + authorId, + }; +}; + +export const dbRelationTypeSchemasToCrossApp = async ({ + client, + schemas, + spaceMap, + accountMap, +}: { + client: DGSupabaseClient; + schemas: Concept[]; + spaceMap?: Record; + accountMap?: Record; +}): Promise => { + if (spaceMap === undefined) spaceMap = await getSpaceMap(client); + if (accountMap === undefined) { + const authorIds = new Set( + schemas.map((r) => r.author_id).filter((id) => typeof id === "number"), + ); + accountMap = await getAccountMap(client, [...authorIds]); + } + + return schemas.map((r) => + dbRelationTypeSchemaToCrossApp(r, spaceMap, accountMap), + ); +}; + +export const dbRelationTripleSchemaToCrossApp = ({ + schema, + spaceMap, + accountMap, + conceptMap, +}: { + schema: Concept; + spaceMap: Record; + accountMap: Record; + conceptMap: Record; +}): CrossAppRelationTripleSchema => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { roles, label, complement, ...other } = + schema.literal_content as Record; + const authorId = accountMap[schema.author_id || 0]; + if (authorId === undefined) throw new Error("Missing author"); + const references = (schema.reference_content ?? {}) as Record; + const spaceUrl = spaceMap[schema.space_id]; + if (spaceUrl === undefined) throw new Error("Missing space"); + const rid = spaceUriAndLocalIdToRid( + spaceUrl, + schema.source_local_id!, + "schema", + ); + const relation = asSimpleLocalId( + conceptMap[references["relation_type"] ?? 0], + spaceUrl, + ); + const sourceType = asSimpleLocalId( + conceptMap[references["source"] ?? 0], + spaceUrl, + ); + const destinationType = asSimpleLocalId( + conceptMap[references["destination"] ?? 0], + spaceUrl, + ); + if (sourceType === undefined) throw new Error("Missing source type"); + if (destinationType === undefined) + throw new Error("Missing destination type"); + const base = { + rid, + localId: schema.source_local_id!, + createdAt: new Date(schema.created + "Z"), + modifiedAt: new Date(schema.last_modified + "Z"), + metadata: other, + authorId, + sourceType, + destinationType, + }; + if (relation) { + return { + ...base, + relation, + }; + } else { + if (typeof label !== "string" || typeof complement !== "string") + throw new Error("Missing either relation_type or relation_type data"); + return { + ...base, + label, + complement, + }; + } +}; + +export const dbRelationTripleSchemasToCrossApp = async ({ + client, + schemas, + spaceMap, + accountMap, + conceptMap, +}: { + client: DGSupabaseClient; + schemas: Concept[]; + spaceMap?: Record; + accountMap?: Record; + conceptMap?: Record; +}): Promise => { + if (spaceMap === undefined) spaceMap = await getSpaceMap(client); + if (accountMap === undefined) { + const authorIds = new Set( + schemas.map((r) => r.author_id).filter((id) => typeof id === "number"), + ); + accountMap = await getAccountMap(client, [...authorIds]); + } + if (conceptMap === undefined) { + const schemaIds = schemas + .map((r) => { + const refs = (r.reference_content ?? {}) as Record< + string, + number | number[] + >; + return [ + refs["source"] ?? [], + refs["destination"] ?? [], + refs["relation_type"] ?? [], + ]; + }) + .flat(2); + conceptMap = await getConceptMap(client, [...new Set(schemaIds)], spaceMap); + } + + return schemas.map((schema) => + dbRelationTripleSchemaToCrossApp({ + schema, + spaceMap, + accountMap, + conceptMap, + }), + ); +}; + +export const dbRelationToCrossApp = ({ + relation, + spaceMap, + accountMap, + conceptMap, +}: { + relation: Concept; + spaceMap: Record; + accountMap: Record; + conceptMap: Record; +}): CrossAppRelation => { + const authorId = accountMap[relation.author_id || 0]; + if (authorId === undefined) throw new Error("Missing author"); + const references = (relation.reference_content ?? {}) as Record< + string, + number + >; + const spaceUrl = spaceMap[relation.space_id]; + if (spaceUrl === undefined) throw new Error("Missing space"); + const rid = spaceUriAndLocalIdToRid( + spaceUrl, + relation.source_local_id!, + "relation", + ); + const relationType = asSimpleLocalId( + conceptMap[relation.schema_id || 0], + spaceUrl, + ); + if (relationType === undefined) throw new Error("Missing relationType"); + const source = asSimpleLocalId( + conceptMap[references["source"] || 0], + spaceUrl, + true, + ); + if (source === undefined) throw new Error("Missing source"); + const destination = asSimpleLocalId( + conceptMap[references["destination"] || 0], + spaceUrl, + true, + ); + if (destination === undefined) throw new Error("Missing destination"); + + return { + rid, + localId: relation.source_local_id!, + authorId, + createdAt: new Date(relation.created + "Z"), + modifiedAt: new Date(relation.last_modified + "Z"), + source, + destination, + relationType, + }; +}; + +export const dbRelationsToCrossApp = async ({ + client, + relations, + accountMap, + conceptMap, + spaceMap, +}: { + client: DGSupabaseClient; + relations: Concept[]; + accountMap?: Record; + conceptMap?: Record; + spaceMap?: Record; +}): Promise => { + if (accountMap === undefined) { + const authorIds = new Set( + relations.map((r) => r.author_id).filter((id) => typeof id === "number"), + ); + accountMap = await getAccountMap(client, [...authorIds]); + } + if (spaceMap === undefined) { + const spaceIds = relations.map((r) => r.space_id); + spaceMap = await getSpaceMap(client, [...new Set(spaceIds)]); + } + if (conceptMap === undefined) { + const nodeIds = relations + .map((r) => { + const refs = (r.reference_content ?? {}) as Record< + string, + number | number[] + >; + return [refs["source"] ?? [], refs["destination"] ?? []]; + }) + .flat(2); + const schemaIds = relations + .map((r) => r.schema_id) + .filter((id) => id !== null); + conceptMap = await getConceptMap( + client, + [...new Set([...schemaIds, ...nodeIds])], + spaceMap, + ); + } + return relations.map((relation) => + dbRelationToCrossApp({ relation, spaceMap, accountMap, conceptMap }), + ); +}; From b750b3cd9ad42eabdc07540bfdbc0c532a975700 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Mon, 17 Aug 2026 11:04:45 -0400 Subject: [PATCH 2/5] eng-1867-step2-discover-shared-relations --- .../roam/src/utils/discoverSharedRelations.ts | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 apps/roam/src/utils/discoverSharedRelations.ts diff --git a/apps/roam/src/utils/discoverSharedRelations.ts b/apps/roam/src/utils/discoverSharedRelations.ts new file mode 100644 index 000000000..ab0331d58 --- /dev/null +++ b/apps/roam/src/utils/discoverSharedRelations.ts @@ -0,0 +1,254 @@ +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import type { + CrossAppRelation, + CrossAppRelationTypeSchema, + CrossAppRelationTripleSchema, + CrossAppNodeSchema, +} from "@repo/database/crossAppContracts"; +import { + getAccountMap, + getSpaceMap, + dbRelationTripleSchemasToCrossApp, + dbRelationsToCrossApp, + dbRelationTypeSchemasToCrossApp, + dbNodeSchemasToCrossApp, +} from "@repo/database/lib/dbToCrossAppConverters"; +import { Tables } from "@repo/database/dbTypes"; +import { spaceUriAndLocalIdToRid } from "@repo/database/lib/rid"; +import { getImportedSourceRids } from "./importedSourceIdentity"; + +type Concept = Tables<"Concept">; + +export type DiscoverSharedRelationsResult = { + relations: CrossAppRelation[]; + relTripleSchemas: CrossAppRelationTripleSchema[]; + relTypeSchemas: CrossAppRelationTypeSchema[]; + nodeSchemas: CrossAppNodeSchema[]; + idToRid: Record; +}; + +export const discoverSharedRelations = async ( + client: DGSupabaseClient, + spaceId: number, + futureImportRids?: string[], +): Promise => { + const response: DiscoverSharedRelationsResult = { + relations: [], + relTripleSchemas: [], + relTypeSchemas: [], + nodeSchemas: [], + idToRid: {}, + }; + // TODO: paginate + const { data: dbAllImportableRelations, error: relError } = await client + .from("my_concepts") + .select( + "*, concepts_of_relation!inner(id, space_id, source_local_id, schema_id)", + ) + .neq("space_id", spaceId) + .eq("is_schema", false) + .gt("arity", 0); + + if (relError || !dbAllImportableRelations) { + throw relError; + } + if (dbAllImportableRelations.length === 0) return response; + const relatedNodeInfo = dbAllImportableRelations + .map((r) => r.concepts_of_relation) + .flat(); + const spaceIds = new Set(relatedNodeInfo.map(({ space_id }) => space_id!)); + const spaceMap = await getSpaceMap(client, [...spaceIds]); + const toRid = (spaceId: number, localId: string) => + spaceId in spaceMap + ? spaceUriAndLocalIdToRid(spaceMap[spaceId], localId, "note") + : undefined; + const idToRid: Record = Object.fromEntries( + relatedNodeInfo + .map( + ({ id, space_id, source_local_id }): [number, string] | undefined => { + if (id === null || space_id === null || source_local_id === null) + return; + const rid = toRid(space_id, source_local_id); + if (rid === undefined) return; + return [id, rid]; + }, + ) + .filter((x) => x !== undefined), + ); + + // We want those relations whose source/destinations are either already imported, + // or somehow connected by Rid to local nodes. + const refToLocalIds = new Set( + relatedNodeInfo + .filter(({ space_id }) => space_id === spaceId) + .map(({ id }) => id), + ); + const importedNodeRids = await getImportedSourceRids(); + if (futureImportRids !== undefined) { + futureImportRids.forEach((id) => importedNodeRids.add(id)); + } + const dbRelations = dbAllImportableRelations.filter((r) => { + const references = (r.reference_content || {}) as Record; + const sourceId = references["source"]; + const destinationId = references["destination"]; + if (!sourceId || !destinationId) return false; + return ( + (refToLocalIds.has(sourceId) || + importedNodeRids.has(idToRid[sourceId] ?? "")) && + (refToLocalIds.has(destinationId) || + importedNodeRids.has(idToRid[destinationId] ?? "")) + ); + }); + const relationSchemaIds = new Set( + dbRelations.map((r) => r.schema_id).filter((r) => r !== null), + ); + if (relationSchemaIds.size === 0) return response; + const { data: dbRelSchemas, error: relSchError } = await client + .from("my_concepts") + .select() + .in("id", [...relationSchemaIds]); + if (relSchError || !dbRelSchemas) { + throw relSchError; + } + const dbRelTripleSchemasDirect = dbRelSchemas.filter( + (r) => r.refs !== null && r.refs.length > 0, + ) as Concept[]; + const dbRelTypeSchemasDirect = dbRelSchemas.filter( + (r) => r.refs === null || r.refs.length === 0, + ) as Concept[]; + let dbRelTripleSchemas = dbRelTripleSchemasDirect; + let dbRelTypeSchemas = dbRelTypeSchemasDirect; + + const missingRelationTypeSchemaIds = new Set( + dbRelTypeSchemasDirect + .map( + (r) => + (typeof r.reference_content === "object" + ? (r.reference_content as Record) + : {})["relation_type"], + ) + .filter((id) => id !== undefined), + ); + + if (missingRelationTypeSchemaIds.size > 0) { + const { data, error: tysError } = await client + .from("my_concepts") + .select() + .in("id", [...missingRelationTypeSchemaIds]); + if (tysError || !data) { + throw tysError; + } + dbRelTypeSchemas = [...dbRelTypeSchemasDirect, ...(data as Concept[])]; + } + if (dbRelTypeSchemasDirect.length) { + // Fetch all corresponding triples and filter + const relTypeIds = dbRelTypeSchemasDirect.map((r) => r.id); + const { data, error: trsError } = await client + .from("my_concepts") + .select() + .eq("is_schema", true) + .eq("arity", 2) + .overlaps("refs", relTypeIds); + if (trsError || !data) { + throw trsError; + } + const triplesBySchemaId: Record = Object.fromEntries( + relTypeIds.map((id) => [id, []]), + ); + data.forEach((c) => { + triplesBySchemaId[ + ((c.reference_content ?? {}) as Record)["relation_type"] + ].push(c as Concept); + }); + const tripleIds = new Set(); + for (const relation of dbRelations) { + const potentialTriples = triplesBySchemaId[relation.schema_id || 0]; + if (potentialTriples === undefined) continue; + const refs = (relation.reference_content || {}) as Record; + const sourceContent = relation.concepts_of_relation.filter( + (cr) => cr.id === refs["source"], + ); + const destinationContent = relation.concepts_of_relation.filter( + (cr) => cr.id === refs["destination"], + ); + if (sourceContent.length !== 1 || destinationContent.length !== 1) + continue; + const matches = potentialTriples.filter( + (triple) => + ((triple.reference_content ?? {}) as Record)[ + "source" + ] === sourceContent[0].schema_id && + ((triple.reference_content ?? {}) as Record)[ + "destination" + ] === destinationContent[0].schema_id, + ); + if (matches.length === 1) { + const relationTripleSchemaId = matches[0].id; + tripleIds.add(relationTripleSchemaId); + // prentend that the obsidian relation referred to the triple + // for when we convert + relation.schema_id = relationTripleSchemaId; + } + } + dbRelTripleSchemas = [ + ...dbRelTripleSchemasDirect, + ...(data as Concept[]).filter((tr) => tripleIds.has(tr.id || 0)), + ]; + } + const nodeTypeSchemaIds = new Set( + dbRelTripleSchemas + .map((r) => { + const refs = (r.reference_content || {}) as Record; + return [refs.source, refs.destination]; + }) + .flat() + .filter((id) => id !== undefined), + ); + const { data: dbNodeTypeSchemas, error: nsError } = await client + .from("my_concepts") + .select() + .in("id", [...nodeTypeSchemaIds]); + if (nsError || !nodeTypeSchemaIds) { + throw nsError; + } + const authorIds = [ + ...dbRelations, + ...dbRelTripleSchemas, + ...dbRelTypeSchemas, + ...dbNodeTypeSchemas, + ] + .map((r) => r.author_id) + .filter((id) => id !== null); + const accountMap = await getAccountMap(client, [...new Set(authorIds)]); + const relTypeSchemas = await dbRelationTypeSchemasToCrossApp({ + client, + schemas: dbRelTypeSchemas, + spaceMap, + accountMap, + }); + const relTripleSchemas = await dbRelationTripleSchemasToCrossApp({ + client, + schemas: dbRelTripleSchemas, + spaceMap, + accountMap, + }); + const relations = await dbRelationsToCrossApp({ + client, + relations: dbRelations as Concept[], + accountMap, + spaceMap, + }); + const nodeSchemas = await dbNodeSchemasToCrossApp({ + client, + schemas: dbNodeTypeSchemas as Concept[], + spaceMap, + accountMap, + }); + return { + relations, + relTripleSchemas, + relTypeSchemas, + nodeSchemas, + idToRid, + }; +}; From 9de2c09cabd63a6b3d10e801cb34c24d15dfc043 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 9 Aug 2026 12:01:18 -0400 Subject: [PATCH 3/5] Second step: import after discover --- .../settings/DiscourseNodeConfigPanel.tsx | 42 +--- .../src/utils/createDiscourseNodeSchema.ts | 65 ++++++ apps/roam/src/utils/createReifiedBlock.ts | 2 +- apps/roam/src/utils/createRelationSchema.ts | 43 ++++ apps/roam/src/utils/importSharedRelations.ts | 210 ++++++++++++++++++ 5 files changed, 321 insertions(+), 41 deletions(-) create mode 100644 apps/roam/src/utils/createDiscourseNodeSchema.ts create mode 100644 apps/roam/src/utils/createRelationSchema.ts create mode 100644 apps/roam/src/utils/importSharedRelations.ts diff --git a/apps/roam/src/components/settings/DiscourseNodeConfigPanel.tsx b/apps/roam/src/components/settings/DiscourseNodeConfigPanel.tsx index 01a2c289d..96e2653b7 100644 --- a/apps/roam/src/components/settings/DiscourseNodeConfigPanel.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeConfigPanel.tsx @@ -10,7 +10,6 @@ import { import React, { useState } from "react"; import getDiscourseNodes from "~/utils/getDiscourseNodes"; import refreshConfigTree from "~/utils/refreshConfigTree"; -import createPage from "roamjs-components/writes/createPage"; import type { CustomField } from "roamjs-components/components/ConfigPanels/types"; import posthog from "posthog-js"; import getDiscourseRelations, { @@ -18,11 +17,10 @@ import getDiscourseRelations, { } from "~/utils/getDiscourseRelations"; import { deleteBlock } from "roamjs-components/writes"; import { formatHexColor } from "./DiscourseNodeCanvasSettings"; -import setBlockProps from "~/utils/setBlockProps"; -import { DiscourseNodeSchema } from "./utils/zodSchema"; import { getGlobalSettings, setGlobalSetting } from "./utils/accessors"; import { GLOBAL_KEYS } from "./utils/settingKeys"; import { invalidateDiscourseNodeTypeCaches } from "~/utils/discourseNodeTypeCache"; +import { createDiscourseNodeSchema } from "~/utils/createDiscourseNodeSchema"; type DiscourseNodeConfigPanelProps = React.ComponentProps< CustomField["options"]["component"] @@ -82,44 +80,8 @@ const DiscourseNodeConfigPanel: React.FC = ({ className="select-none" disabled={!label} onClick={() => { - const candidateShortcut = label.slice(0, 1).toUpperCase(); - const existingShortcuts = new Set( - getDiscourseNodes() - .map((n) => n.shortcut.toUpperCase()) - .filter(Boolean), - ); - const shortcut = existingShortcuts.has(candidateShortcut) - ? "" - : candidateShortcut; - const format = `[[${label.slice(0, 3).toUpperCase()}]] - {content}`; posthog.capture("Discourse Node: Type Created", { label: label }); - void createPage({ - title: `discourse-graph/nodes/${label}`, - tree: [ - { - text: "Shortcut", - children: [{ text: shortcut }], - }, - { - text: "Tag", - children: [{ text: "" }], - }, - { - text: "Format", - children: [{ text: format }], - }, - ], - }).then((valueUid) => { - setBlockProps( - valueUid, - DiscourseNodeSchema.parse({ - text: label, - type: valueUid, - shortcut, - format, - }), - ); - invalidateDiscourseNodeTypeCaches(); + void createDiscourseNodeSchema(label).then((valueUid) => { setNodes([ ...nodes, { diff --git a/apps/roam/src/utils/createDiscourseNodeSchema.ts b/apps/roam/src/utils/createDiscourseNodeSchema.ts new file mode 100644 index 000000000..01e6cbe84 --- /dev/null +++ b/apps/roam/src/utils/createDiscourseNodeSchema.ts @@ -0,0 +1,65 @@ +import createPage from "roamjs-components/writes/createPage"; +import setBlockProps from "~/utils/setBlockProps"; +import { DiscourseNodeSchema } from "~/components/settings/utils/zodSchema"; +import { invalidateDiscourseNodeTypeCaches } from "~/utils/discourseNodeTypeCache"; +import getDiscourseNodes from "./getDiscourseNodes"; + +export const createDiscourseNodeSchema = async ( + label: string, + options?: { + shortcut?: string; + format?: string; + template?: string; + }, +): Promise => { + let { shortcut, format } = options ?? {}; + const { template } = options ?? {}; + if (shortcut === undefined) { + const candidateShortcut = label.slice(0, 1).toUpperCase(); + const existingShortcuts = new Set( + getDiscourseNodes() + .map((n) => n.shortcut.toUpperCase()) + .filter(Boolean), + ); + shortcut = existingShortcuts.has(candidateShortcut) + ? "" + : candidateShortcut; + } + format = format ?? `[[${label.slice(0, 3).toUpperCase()}]] - {content}`; + const tree = [ + { + text: "Shortcut", + children: [{ text: shortcut }], + }, + { + text: "Tag", + children: [{ text: "" }], + }, + { + text: "Format", + children: [{ text: format }], + }, + ]; + if (template != undefined) { + // TODO: Make into a tree + tree.push({ + text: "Template", + children: [{ text: template ?? "" }], + }); + } + const valueUid = await createPage({ + title: `discourse-graph/nodes/${label}`, + tree, + }); + setBlockProps( + valueUid, + DiscourseNodeSchema.parse({ + text: label, + type: valueUid, + shortcut, + format, + }), + ); + invalidateDiscourseNodeTypeCaches(); + return valueUid; +}; diff --git a/apps/roam/src/utils/createReifiedBlock.ts b/apps/roam/src/utils/createReifiedBlock.ts index ad78975f7..d39bb420f 100644 --- a/apps/roam/src/utils/createReifiedBlock.ts +++ b/apps/roam/src/utils/createReifiedBlock.ts @@ -140,7 +140,7 @@ export const createReifiedRelation = async ({ sourceUid: string; relationBlockUid: string; destinationUid: string; -}): Promise => { +}): Promise => { return await createReifiedBlock({ destinationBlockUid: await getOrCreateRelationPageUid(), schemaUid: relationBlockUid, diff --git a/apps/roam/src/utils/createRelationSchema.ts b/apps/roam/src/utils/createRelationSchema.ts new file mode 100644 index 000000000..d175bd9b5 --- /dev/null +++ b/apps/roam/src/utils/createRelationSchema.ts @@ -0,0 +1,43 @@ +import discourseConfigRef from "~/utils/discourseConfigRef"; +import createBlock from "roamjs-components/writes/createBlock"; + +export const createRelationSchema = async ({ + label, + complement, + source, + destination, +}: { + label: string; + complement: string; + source: string; + destination: string; +}) => { + const grammarNode = discourseConfigRef.tree.find( + (node) => node.text === "grammar", + ); + const relationsNode = grammarNode?.children.find( + (node) => node.text === "relations", + ); + if (!relationsNode) throw new Error("Cannot find the relation grammar"); + return await createBlock({ + parentUid: relationsNode.uid, + order: "last", + node: { + text: label, + children: [ + { + text: "source", + children: [{ text: source }], + }, + { + text: "destination", + children: [{ text: destination }], + }, + { + text: "complement", + children: [{ text: complement }], + }, + ], + }, + }); +}; diff --git a/apps/roam/src/utils/importSharedRelations.ts b/apps/roam/src/utils/importSharedRelations.ts new file mode 100644 index 000000000..5096b2134 --- /dev/null +++ b/apps/roam/src/utils/importSharedRelations.ts @@ -0,0 +1,210 @@ +import type { + CrossAppRelation, + CrossAppRelationTypeSchema, + CrossAppRelationTripleSchema, + CrossAppNodeSchema, +} from "@repo/database/crossAppContracts"; +import { + spaceUriAndLocalIdToRid, + isRid, + ridToSpaceUriAndLocalId, +} from "@repo/database/lib/rid"; +import { + findImportedNodeUidBySourceRid, + getImportedSourceRids, + writeImportedSourceIdentity, +} from "./importedSourceIdentity"; +import getDiscourseRelations from "./getDiscourseRelations"; +import getDiscourseNodes from "./getDiscourseNodes"; +import { createDiscourseNodeSchema } from "./createDiscourseNodeSchema"; +import { createRelationSchema } from "./createRelationSchema"; +import { + createReifiedRelation, + getReifiedRelations, +} from "./createReifiedBlock"; +import { discoverSharedRelations } from "./discoverSharedRelations"; +import { DGSupabaseClient } from "@repo/database/lib/client"; + +const matchImportedNodeSchemas = async ( + nodeSchemas: CrossAppNodeSchema[], +): Promise> => { + const result: Record = {}; + const nodeSchemasByRid = Object.fromEntries( + nodeSchemas.map((s) => [s.rid!, s]), + ); + const existing = await getImportedSourceRids(); + const localNodeSchemas = getDiscourseNodes(); + const localNodeSchemasByLabel = Object.fromEntries( + localNodeSchemas.map((s) => [s.text.toLowerCase(), s]), + ); + const localNodeSchemasByLocalId = Object.fromEntries( + localNodeSchemas.map((s) => [s.type, s]), + ); + + for (const [rid, schema] of Object.entries(nodeSchemasByRid)) { + let blockUid: string | undefined | null; + if (existing.has(rid)) { + blockUid = await findImportedNodeUidBySourceRid(rid); + } + if (blockUid) { + result[rid] = blockUid; + continue; + } else if (schema.localId in localNodeSchemasByLocalId) { + blockUid = localNodeSchemasByLocalId[schema.localId].type; + } else if (schema.label.toLowerCase() in localNodeSchemasByLabel) { + blockUid = localNodeSchemasByLabel[schema.label.toLowerCase()].type; + } else { + // create a new node schema + blockUid = await createDiscourseNodeSchema(schema.label, { + template: schema.template, + // TODO: colour, other metadata? + }); + } + result[rid] = blockUid; + await writeImportedSourceIdentity({ + pageUid: blockUid, + sourceNodeRid: rid, + sourceModifiedAt: (schema.modifiedAt ?? new Date()).toISOString(), + }); + } + return result; +}; + +const matchImportedRelationSchemas = async ( + nodeSchemaRidToLocalId: Record, + relationTypeSchemas: CrossAppRelationTypeSchema[], + relationTripleSchemas: CrossAppRelationTripleSchema[], +): Promise> => { + const result: Record = {}; + const relationSchemas = getDiscourseRelations(); + const existing = await getImportedSourceRids(); + const relationTypeSchemasByRid = Object.fromEntries( + relationTypeSchemas.map((s) => [s.rid!, s]), + ); + const localRelationTripleSchemasByLocalId = Object.fromEntries( + relationSchemas.map((s) => [s.id, s]), + ); + + for (const tripleSchema of relationTripleSchemas) { + const rid = tripleSchema.rid!; + let blockUid: string | undefined | null; + if (existing.has(rid)) { + blockUid = await findImportedNodeUidBySourceRid(rid); + } + if (blockUid) { + result[rid] = blockUid; + continue; + } + if (tripleSchema.localId in localRelationTripleSchemasByLocalId) { + blockUid = localRelationTripleSchemasByLocalId[tripleSchema.localId].id; + } else { + const sourceTypeRid = tripleSchema.rid!; + const destinationTypeRid = tripleSchema.rid!; + const source = nodeSchemaRidToLocalId[sourceTypeRid] ?? "missing"; + const destination = + nodeSchemaRidToLocalId[destinationTypeRid] ?? "missing"; + if (source === "missing" || destination === "missing") + throw new Error("Missing source or destination"); + const relationType = + relationTypeSchemasByRid[tripleSchema.relation ?? ""]; + + const label = tripleSchema.label ?? relationType?.label; + const complement = tripleSchema.complement ?? relationType?.complement; + const match = relationSchemas.filter( + (r) => + r.label.toLowerCase() === label.toLowerCase() && + r.source === source && + r.destination === destination, + ); + if (match.length > 1) { + throw new Error("multiple matches"); + } + if (match.length === 1) { + blockUid = match[0].id; + } else { + blockUid = await createRelationSchema({ + label, + complement, + source, + destination, + }); + } + } + result[rid] = blockUid; + await writeImportedSourceIdentity({ + pageUid: blockUid, + sourceNodeRid: rid, + sourceModifiedAt: (tripleSchema.modifiedAt ?? new Date()).toISOString(), + }); + } + return result; +}; + +const importRelations = async ( + schemaRidToLocalId: Record, + relations: CrossAppRelation[], +): Promise => { + const existing = await getImportedSourceRids(); + const allRelations = await getReifiedRelations(); + for (const relation of relations) { + const sourceNodeRid = relation.rid; + if (sourceNodeRid === undefined) continue; + const { spaceUri } = ridToSpaceUriAndLocalId(sourceNodeRid); + if (existing.has(sourceNodeRid)) continue; + const relationBlockUid = schemaRidToLocalId[sourceNodeRid]; + if (relationBlockUid === undefined) + throw new Error(`Missing relation type: ${relation.relationType}`); + const relSource = isRid(relation.source) + ? relation.source + : spaceUriAndLocalIdToRid(spaceUri, relation.source, "note"); + const relDestination = isRid(relation.destination) + ? relation.destination + : spaceUriAndLocalIdToRid(spaceUri, relation.destination, "note"); + const sourceUid = schemaRidToLocalId[relSource]; + if (sourceUid === undefined) + throw new Error(`Missing relation source: ${relation.source}`); + const destinationUid = schemaRidToLocalId[relDestination]; + if (destinationUid === undefined) + throw new Error(`Missing relation destination: ${relation.destination}`); + const existingRel = allRelations.filter( + (r) => + r.hasSchema == relationBlockUid && + r.sourceUid == sourceUid && + r.destinationUid == r.destinationUid, + ); + if (existingRel.length > 1) throw new Error("Multiple matching relations"); + const uid = + existingRel.length === 1 + ? existingRel[0].relationId + : await createReifiedRelation({ + sourceUid, + destinationUid, + relationBlockUid, + }); + await writeImportedSourceIdentity({ + pageUid: uid, + sourceNodeRid, + sourceModifiedAt: (relation.modifiedAt ?? new Date()).toISOString(), + }); + } +}; + +export const importSharedRelations = async ( + client: DGSupabaseClient, + spaceId: number, +) => { + const { relations, relTripleSchemas, relTypeSchemas, nodeSchemas, idToRid } = + await discoverSharedRelations(client, spaceId); + let ridToId = Object.fromEntries( + Object.entries(idToRid).map(([id, rid]) => [rid, id]), + ); + const nodeSchemasMap = await matchImportedNodeSchemas(nodeSchemas); + ridToId = { ...ridToId, ...nodeSchemasMap }; + const relationSchemaMap = await matchImportedRelationSchemas( + ridToId, + relTypeSchemas, + relTripleSchemas, + ); + ridToId = { ...ridToId, ...relationSchemaMap }; + await importRelations(ridToId, relations); +}; From 65d06681b1d3c3c86f6c86b4511898ddbf11967d Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 9 Aug 2026 15:11:50 -0400 Subject: [PATCH 4/5] actually import the relations --- apps/roam/src/components/DiscoverSharedNodesDialog.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx index fb87c6859..d5044e039 100644 --- a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx +++ b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx @@ -21,6 +21,7 @@ import { isFailedSharedNodeImport, type SharedNodeImportItem, } from "~/utils/importSharedNodes"; +import { importSharedRelations } from "~/utils/importSharedRelations"; import internalError from "~/utils/internalError"; import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; @@ -146,6 +147,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { const [error, setError] = useState(""); const [searchTerm, setSearchTerm] = useState(""); const [selectedRids, setSelectedRids] = useState>(new Set()); + const [spaceId, setSpaceId] = useState(0); const [importProgress, setImportProgress] = useState<{ current: number; total: number; @@ -163,6 +165,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { try { const context = await getSupabaseContext(); if (!context) throw new Error("Could not connect to shared persistence."); + setSpaceId(context.spaceId); const client = await getLoggedInClient(); if (!client) throw new Error("Could not connect to shared persistence."); const { sharedNodes, importedSourceRids } = await discoverSharedNodes({ @@ -242,7 +245,6 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { sharedNodes: selectedNodes, onProgress: (current, total) => setImportProgress({ current, total }), }); - setImportResults(results); const newlyImportedRids = results .filter((item) => item.status !== "failed") .map((item) => item.sharedNode.rid); @@ -251,6 +253,8 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { newlyImportedRids.forEach((rid) => next.add(rid)); return next; }); + await importSharedRelations(client, spaceId); + setImportResults(results); const failedImports = results.filter(isFailedSharedNodeImport); setSelectedRids( new Set(failedImports.map((item) => item.sharedNode.rid)), From 93eb604883911ce61e0d340b12dbda8fbed8201a Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sat, 15 Aug 2026 15:46:02 -0400 Subject: [PATCH 5/5] tentative flag --- apps/roam/src/utils/createReifiedBlock.ts | 12 ++++++++---- apps/roam/src/utils/importSharedRelations.ts | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/roam/src/utils/createReifiedBlock.ts b/apps/roam/src/utils/createReifiedBlock.ts index d39bb420f..fd3ae4735 100644 --- a/apps/roam/src/utils/createReifiedBlock.ts +++ b/apps/roam/src/utils/createReifiedBlock.ts @@ -136,17 +136,21 @@ export const createReifiedRelation = async ({ sourceUid, relationBlockUid, destinationUid, + tentative, }: { sourceUid: string; relationBlockUid: string; destinationUid: string; + tentative?: boolean; }): Promise => { + const parameterUids: Record = { + sourceUid, + destinationUid, + }; + if (tentative !== undefined) parameterUids.tentative = tentative.toString(); return await createReifiedBlock({ destinationBlockUid: await getOrCreateRelationPageUid(), schemaUid: relationBlockUid, - parameterUids: { - sourceUid, - destinationUid, - }, + parameterUids, }); }; diff --git a/apps/roam/src/utils/importSharedRelations.ts b/apps/roam/src/utils/importSharedRelations.ts index 5096b2134..218e51049 100644 --- a/apps/roam/src/utils/importSharedRelations.ts +++ b/apps/roam/src/utils/importSharedRelations.ts @@ -180,6 +180,7 @@ const importRelations = async ( sourceUid, destinationUid, relationBlockUid, + tentative: true, }); await writeImportedSourceIdentity({ pageUid: uid,