diff --git a/packages/base/card-api.gts b/packages/base/card-api.gts index 6a4cf30ee7..3c6cf4d687 100644 --- a/packages/base/card-api.gts +++ b/packages/base/card-api.gts @@ -4864,7 +4864,7 @@ export function virtualNetworkFor( // `new URL()` throws on those, so we return the raw reference unchanged // instead of bubbling the error to callers (e.g. relationship deserialize // uses the returned string as a "did this resolve?" signal). -function resolveRef( +export function resolveRef( virtualNetwork: VirtualNetwork | undefined, reference: string, relativeTo: RealmResourceIdentifier | URL | undefined, diff --git a/packages/base/searchable.ts b/packages/base/searchable.ts new file mode 100644 index 0000000000..e1baadc7ec --- /dev/null +++ b/packages/base/searchable.ts @@ -0,0 +1,394 @@ +import { rawArrayValues } from './watched-array'; +import { isSavedInstance } from './-private'; +import { + assertIsSerializerName, + fieldSerializer, + getSerializer, + isCardError, + primitive, + relativeTo, +} from '@cardstack/runtime-common'; +import { + getFields, + isLinkError, + isLinkNotFound, + isNonPresentLink, + isNotLoadedValue, + peekAtField, +} from './field-support'; +import { + createFromSerialized, + getStore, + resolveRef, + type BaseDef, + type BaseDefConstructor, + type CardDef, + type CardStore, + type Field, +} from './card-api'; + +// ============================================================================ +// Searchable-driven search-doc generation — NON-AUTHORITATIVE. +// +// Parallel to `searchDoc` (in card-api), which stays in charge of production +// indexing. This path derives link depth from the explicit `field.searchable` +// annotation instead of from what the render happened to load into the store, +// and loads the named link targets itself (targeted loading) rather than +// relying on render-driven store residency. +// +// Routes are dotted paths rooted at the CURRENT card's link fields. Depth is +// governed ENTIRELY by the `searchable` annotations on the card being indexed +// (extended by their dotted paths): a card pulled in as a link target does NOT +// re-consult its own `searchable` — only the explicit route continues into it. +// `true` => the immediate ("self") link; `'a.b'` => the n+1 route a→b; an array +// combines routes. Cycle clipping, `{ id }` for unfollowed / broken / not-found +// links, and `linksToMany` id normalization are preserved from +// `BaseDef[queryableValue]`. The declared field type is enumerated for both +// links and contained values (not the runtime subtype), which drops the +// non-queryable polymorphic-subtype bloat. +export async function searchDocFromFields( + instance: CardDef, +): Promise> { + let routes = seedSearchableRoutes( + instance.constructor as unknown as typeof BaseDef, + ); + return (await searchableQueryableValue( + instance.constructor as unknown as typeof BaseDef, + instance, + routes, + [], + getStore(instance), + )) as Record; +} + +// Build the route set rooted at the indexed card's own link fields. This is +// the ONLY place `field.searchable` is read — deeper recursion follows the +// inherited routes, never a pulled-in target's own annotations. +function seedSearchableRoutes(cardClass: typeof BaseDef): string[] { + let routes: string[] = []; + for (let [fieldName, field] of Object.entries( + getFields(cardClass, { includeComputeds: true }), + )) { + let searchable = field?.searchable; + if (searchable == null) { + continue; + } + if (searchable === true) { + routes.push(fieldName); // self link, no deeper + continue; + } + // Tolerate a malformed annotation (a non-string array entry, a non-array + // non-string value) rather than emitting a junk route or throwing: a route + // can only ever be a dotted field path. An empty array contributes nothing. + let paths = + typeof searchable === 'string' + ? [searchable] + : Array.isArray(searchable) + ? searchable + : []; + for (let path of paths) { + if (typeof path !== 'string') { + continue; + } + routes.push(path === '' ? fieldName : `${fieldName}.${path}`); + } + } + return routes; +} + +// For `routes` rooted at the current card, find those whose head segment is +// `fieldName`. `matched` = the field is named by at least one route (so a link +// is expanded); `tails` = the non-empty remainders, which become the target's +// routes. An empty tail (head-only route, e.g. from `searchable: true`) marks +// the link as expanded-but-no-deeper and contributes no tail. +function matchSearchableRoutes( + routes: string[], + fieldName: string, +): { matched: boolean; tails: string[] } { + let matched = false; + let tails: string[] = []; + for (let route of routes) { + let dot = route.indexOf('.'); + let head = dot === -1 ? route : route.slice(0, dot); + if (head !== fieldName) { + continue; + } + matched = true; + if (dot !== -1) { + tails.push(route.slice(dot + 1)); + } + } + return { matched, tails }; +} + +// Targeted load of a link target by reference: reuse a fully-deserialized +// resident instance when present, else load + deserialize the document (the +// same load path the lazy link getter uses, sans dependency tracking — this +// generator is non-authoritative). Returns undefined if the target errors. +async function loadSearchableTarget( + store: CardStore, + reference: string, +): Promise { + let resident = store.getCard(reference); + if (resident && (resident as any)[isSavedInstance] === true) { + return resident; + } + // A missing / broken target (or one that can't be loaded) degrades to + // `{ id }` upstream. The + // store may surface that either as a returned `CardError` or a thrown + // rejection (e.g. a 404 / invalid-URL on the load path), so guard both. + try { + let cardDoc = await store.loadCardDocument(reference); + if (isCardError(cardDoc)) { + return undefined; + } + return (await createFromSerialized( + cardDoc.data, + cardDoc, + cardDoc.data.id!, + { store }, + )) as CardDef; + } catch { + return undefined; + } +} + +// Core recursion. `fieldCard` is the DECLARED type to enumerate; `value` is the +// runtime instance (a subtype's extra fields are dropped); `routes` are the +// dotted paths rooted at `value`'s fields; `stack` is the cycle guard. +async function searchableQueryableValue( + fieldCard: typeof BaseDef, + value: any, + routes: string[], + stack: BaseDef[], + // Threaded from the indexed instance rather than re-derived per value: a + // contained FieldDef value may not be store-associated, but its nested links + // must still load against the owner's store. + store: CardStore, +): Promise { + if (primitive in fieldCard) { + if (fieldSerializer in fieldCard) { + assertIsSerializerName((fieldCard as any)[fieldSerializer]); + return getSerializer((fieldCard as any)[fieldSerializer]).queryableValue( + value, + stack, + ); + } + return value; + } + if (value == null) { + return null; + } + let valueId = (value as { id?: string }).id; + // Cycle guard — identical to `BaseDef[queryableValue]`: object-identity and + // id-based, so a re-entered card (even as a fresh object) clips to `{ id }`. + if ( + stack.includes(value) || + (valueId != null && + stack.some((s) => (s as { id?: string }).id === valueId)) + ) { + return { id: valueId }; + } + let makeAbsoluteURL = (reference: string) => + value[relativeTo] + ? resolveRef(store.virtualNetwork, reference, value[relativeTo]) + : reference; + let nextStack = [value, ...stack]; + let entries: [string, any][] = []; + for (let [fieldName, field] of Object.entries( + getFields(fieldCard, { includeComputeds: true }), + )) { + // Query-backed relationships can't be invalidated, so they're never in the + // doc — same skip as the store-driven path. + if (field?.queryDefinition) { + continue; + } + let { matched, tails } = matchSearchableRoutes(routes, fieldName); + let rawValue = peekAtField(value, fieldName); + switch (field!.fieldType) { + case 'contains': { + entries.push([ + fieldName, + await searchableQueryableValue( + field!.card, + rawValue, + tails, + nextStack, + store, + ), + ]); + break; + } + case 'containsMany': { + // A whole-field sentinel (e.g. a computed containsMany that consumes + // an unresolved link) is not iterable; treat as null, the same as the + // linksToMany branch below. + if (rawValue == null || isNonPresentLink(rawValue)) { + entries.push([fieldName, null]); + break; + } + let items: any[] = []; + for (let item of rawArrayValues(rawValue)) { + if (item == null) { + continue; + } + let v = await searchableQueryableValue( + field!.card, + item, + tails, + nextStack, + store, + ); + if (v != null) { + items.push(v); + } + } + entries.push([fieldName, items.length === 0 ? null : items]); + break; + } + case 'linksTo': { + entries.push([ + fieldName, + await searchableLink( + field!, + rawValue, + matched, + tails, + nextStack, + store, + makeAbsoluteURL, + ), + ]); + break; + } + case 'linksToMany': { + entries.push([ + fieldName, + await searchableLinksToMany( + field!, + rawValue, + matched, + tails, + nextStack, + store, + makeAbsoluteURL, + ), + ]); + break; + } + } + } + return Object.fromEntries(entries); +} + +// A `linksTo` value: `{ id }` when the link isn't made searchable (or is +// broken / cannot be loaded); the expanded declared-type target when a route names +// it. Mirrors `LinksTo.queryableValue` + the `{ id }` sentinel handling. +async function searchableLink( + field: Field, + rawValue: any, + matched: boolean, + tails: string[], + stack: BaseDef[], + store: CardStore, + makeAbsoluteURL: (reference: string) => string, +): Promise { + if (rawValue == null) { + return null; + } + // A broken / not-found link can't be expanded — keep its reference as `{ id }`. + if (isLinkError(rawValue) || isLinkNotFound(rawValue)) { + return { id: makeAbsoluteURL(rawValue.reference) }; + } + if (!matched) { + return { + id: makeAbsoluteURL( + isNotLoadedValue(rawValue) + ? rawValue.reference + : (rawValue as CardDef).id, + ), + }; + } + let target = rawValue as CardDef; + if (isNotLoadedValue(rawValue)) { + // Resolve a relative reference (e.g. `./hassan`) against the owner's + // `relativeTo` before the store lookup/load — the same as the lazy link + // getter. The store can't `toURL` a relative string, which would otherwise + // degrade an expandable searchable link to `{ id }`. + let resolvedRef = makeAbsoluteURL(rawValue.reference); + let loaded = await loadSearchableTarget(store, resolvedRef); + if (loaded == null) { + return { id: resolvedRef }; + } + target = loaded; + } + return await searchableQueryableValue( + field.card, + target, + tails, + stack, + store, + ); +} + +// A `linksToMany` value: per-slot `{ id }` / expansion, with the same +// absolute-URL id normalization the store-driven path applies. +async function searchableLinksToMany( + field: Field, + rawValue: any, + matched: boolean, + tails: string[], + stack: BaseDef[], + store: CardStore, + makeAbsoluteURL: (reference: string) => string, +): Promise { + // A whole-field sentinel (errored/unresolved plural) is not iterable; treat + // as empty, same as `LinksToMany.queryableValue`. + if (rawValue == null || isNonPresentLink(rawValue)) { + return null; + } + let out: any[] = []; + for (let item of rawArrayValues(rawValue)) { + if (item == null) { + continue; + } + if (isLinkError(item) || isLinkNotFound(item)) { + out.push({ id: makeAbsoluteURL(item.reference) }); + continue; + } + if (!matched) { + out.push({ + id: makeAbsoluteURL( + isNotLoadedValue(item) ? item.reference : (item as CardDef).id, + ), + }); + continue; + } + let target = item as CardDef; + if (isNotLoadedValue(item)) { + // Resolve a relative reference before the load — see `searchableLink`. + let resolvedRef = makeAbsoluteURL(item.reference); + let loaded = await loadSearchableTarget(store, resolvedRef); + if (loaded == null) { + out.push({ id: resolvedRef }); + continue; + } + target = loaded; + } + let expanded = await searchableQueryableValue( + field.card, + target, + tails, + stack, + store, + ); + if (expanded != null) { + out.push( + expanded.id != null + ? { ...expanded, id: makeAbsoluteURL(expanded.id) } + : expanded, + ); + } + } + return out.length === 0 ? null : out; +} diff --git a/packages/host/tests/helpers/base-realm.ts b/packages/host/tests/helpers/base-realm.ts index 3c473f9638..ad5f37960c 100644 --- a/packages/host/tests/helpers/base-realm.ts +++ b/packages/host/tests/helpers/base-realm.ts @@ -29,6 +29,7 @@ import type * as NumberFieldModule from 'https://cardstack.com/base/number'; import type * as PhoneNumberFieldModule from 'https://cardstack.com/base/phone-number'; import type * as RealmFieldModule from 'https://cardstack.com/base/realm'; import type * as RichMarkdownModule from 'https://cardstack.com/base/rich-markdown'; +import type * as SearchableModule from 'https://cardstack.com/base/searchable'; import type * as SkillModule from 'https://cardstack.com/base/skill'; import type * as StringFieldModule from 'https://cardstack.com/base/string'; import type * as SystemCardModule from 'https://cardstack.com/base/system-card'; @@ -159,6 +160,8 @@ let createFromSerialized: (typeof CardAPIModule)['createFromSerialized']; let updateFromSerialized: (typeof CardAPIModule)['updateFromSerialized']; let rawSerializeCard: (typeof CardAPIModule)['serializeCard']; let rawSerializeFileDef: (typeof CardAPIModule)['serializeFileDef']; +let searchDoc: (typeof CardAPIModule)['searchDoc']; +let searchDocFromFields: (typeof SearchableModule)['searchDocFromFields']; // Test-side wrappers around the raw card-api serialize functions that // auto-supply `virtualNetwork` from the active loader. Tests that need a @@ -401,6 +404,7 @@ async function initialize() { getBrokenLinks, getDataBucket, getQueryableValue, + searchDoc, subscribeToChanges, unsubscribeFromChanges, flushLogs, @@ -414,6 +418,12 @@ async function initialize() { Theme, } = cardAPI); + // The searchable-driven generator lives in its own base module, not on + // card-api (so it stays out of every card's dependency closure). + searchDocFromFields = ( + await loader.import(`${baseRealm.url}searchable`) + ).searchDocFromFields; + enumField = (await loader.import(`${baseRealm.url}enum`)) .default; const enumModule = await loader.import( @@ -484,6 +494,8 @@ export { getBrokenLinks, getDataBucket, getQueryableValue, + searchDoc, + searchDocFromFields, subscribeToChanges, unsubscribeFromChanges, flushLogs, diff --git a/packages/host/tests/integration/searchable-search-doc-test.gts b/packages/host/tests/integration/searchable-search-doc-test.gts new file mode 100644 index 0000000000..1ed3feb2af --- /dev/null +++ b/packages/host/tests/integration/searchable-search-doc-test.gts @@ -0,0 +1,1196 @@ +import { getService } from '@universal-ember/test-support'; +import { module, test } from 'qunit'; + +import { baseRealm } from '@cardstack/runtime-common'; +import type { Realm, IndexedInstance } from '@cardstack/runtime-common'; +import type { Loader } from '@cardstack/runtime-common/loader'; + +import type StoreService from '@cardstack/host/services/store'; + +import type { CardDef as CardDefType } from 'https://cardstack.com/base/card-api'; + +import { + testRealmURL, + setupCardLogs, + setupLocalIndexing, + setupIntegrationTestRealm, +} from '../helpers'; +import { + setupBaseRealm, + field, + contains, + containsMany, + linksTo, + linksToMany, + CardDef, + FieldDef, + Component, + StringField, + searchDocFromFields, +} from '../helpers/base-realm'; +import { setupMockMatrix } from '../helpers/mock-matrix'; +import { setupRenderingTest } from '../helpers/setup'; + +let loader: Loader; +let realm: Realm; + +// Exercises the searchable-driven generator `searchDocFromFields`. +// +// Central property: search-doc depth is sourced ONLY from the `searchable` +// annotations on the card being indexed — a card pulled in as a link target +// does NOT re-consult its own `searchable`; only the route declared on the +// indexed card continues into it. +module('Integration | searchable search doc', function (hooks) { + setupRenderingTest(hooks); + setupBaseRealm(hooks); + + hooks.beforeEach(function () { + loader = getService('loader-service').loader; + }); + + setupLocalIndexing(hooks); + let mockMatrixUtils = setupMockMatrix(hooks); + setupCardLogs( + hooks, + async () => await loader.import(`${baseRealm.url}card-api`), + ); + + hooks.beforeEach(async function () { + // ---- leaf link targets ------------------------------------------------- + class Agent extends CardDef { + static displayName = 'Agent'; + @field name = contains(StringField); + } + class Headquarters extends CardDef { + static displayName = 'Headquarters'; + @field name = contains(StringField); + } + // A link target two hops deep with TWO further links, each made searchable + // on ITSELF — those annotations are dormant whenever Company is pulled in, + // and only fire when a route from the indexed card names them. + class Company extends CardDef { + static displayName = 'Company'; + @field name = contains(StringField); + @field ceo = linksTo(Agent, { searchable: true }); + @field hq = linksTo(Headquarters, { searchable: true }); + } + // The shared one-hop target. Its OWN `agent` link is searchable (dormant + // when Author is pulled in); `company` is unannotated. + class Author extends CardDef { + static displayName = 'Author'; + @field name = contains(StringField); + @field agent = linksTo(Agent, { searchable: true }); + @field company = linksTo(Company); + } + + // ---- linksTo route shapes (all link to Author/au1) --------------------- + class ArticleSelf extends CardDef { + static displayName = 'ArticleSelf'; + @field title = contains(StringField); + @field author = linksTo(Author, { searchable: true }); // self link only + } + class ArticleDeep extends CardDef { + static displayName = 'ArticleDeep'; + @field title = contains(StringField); + @field author = linksTo(Author, { searchable: 'agent' }); // 1-hop route + } + class ArticleShallow extends CardDef { + static displayName = 'ArticleShallow'; + @field title = contains(StringField); + @field author = linksTo(Author); // unannotated → {id} + } + class ArticleHop2 extends CardDef { + static displayName = 'ArticleHop2'; + @field title = contains(StringField); + @field author = linksTo(Author, { searchable: 'company' }); // 2-hop + } + class ArticleHop3 extends CardDef { + static displayName = 'ArticleHop3'; + @field title = contains(StringField); + // 3-segment route author.company.hq — the "a.b.c" dotted case. + @field author = linksTo(Author, { searchable: 'company.hq' }); + } + class ArticleShared extends CardDef { + static displayName = 'ArticleShared'; + @field title = contains(StringField); + // Shared ancestor: both routes pass through `company`, then diverge. + @field author = linksTo(Author, { + searchable: ['company.ceo', 'company.hq'], + }); + } + class ArticleMulti extends CardDef { + static displayName = 'ArticleMulti'; + @field title = contains(StringField); + // Array with divergent heads: one self link + one deep route. + @field author = linksTo(Author, { searchable: ['agent', 'company.hq'] }); + } + class ArticleEmptyPath extends CardDef { + static displayName = 'ArticleEmptyPath'; + @field title = contains(StringField); + @field author = linksTo(Author, { searchable: '' }); // '' == self link + } + // Malformed / impossible searchable values must degrade gracefully, never + // crash and never emit a junk expansion. + class ArticleEmptyArray extends CardDef { + static displayName = 'ArticleEmptyArray'; + @field title = contains(StringField); + @field author = linksTo(Author, { searchable: [] }); // → no route + } + class ArticleNullSearchable extends CardDef { + static displayName = 'ArticleNullSearchable'; + @field title = contains(StringField); + @field author = linksTo(Author, { searchable: null as any }); // unannotated + } + class ArticleArrayWithNull extends CardDef { + static displayName = 'ArticleArrayWithNull'; + @field title = contains(StringField); + // The non-string entry is ignored; the valid route still expands. + @field author = linksTo(Author, { searchable: ['agent', null] as any }); + } + class ArticleImpossiblePath extends CardDef { + static displayName = 'ArticleImpossiblePath'; + @field title = contains(StringField); + // `agent` is a leaf with no `deeper` field — the unreachable tail is a + // no-op; the reachable prefix (agent) still expands. + @field author = linksTo(Author, { searchable: 'agent.deeper' }); + } + class ArticleFalse extends CardDef { + static displayName = 'ArticleFalse'; + @field title = contains(StringField); + // `false` is not a valid searchable value — exercised as bad input. + @field author = linksTo(Author, { searchable: false as any }); + } + + // ---- cycles ------------------------------------------------------------ + class Person extends CardDef { + static displayName = 'Person'; + @field name = contains(StringField); + @field friend = linksTo(() => Person, { searchable: true }); + } + // A ring: the route `next.next.next` walks r1→r2→r3 and clips on re-entry. + class Ring extends CardDef { + static displayName = 'Ring'; + @field name = contains(StringField); + @field next = linksTo(() => Ring, { searchable: 'next.next' }); + } + // The same, plural: `nexts.nexts.nexts` walks the ring and clips. + class RingM extends CardDef { + static displayName = 'RingM'; + @field name = contains(StringField); + @field nexts = linksToMany(() => RingM, { searchable: 'nexts.nexts' }); + } + + // ---- contained values that hold links (the four combinations) ---------- + // A FieldDef carrying a contained scalar AND both link arities, so a route + // can pass through it — whether the FieldDef is `contains` or + // `containsMany` — into either a `linksTo` or a `linksToMany`. The contained + // scalar (`label`) is always included; an unrouted link inside stays { id }. + class Crew extends FieldDef { + static displayName = 'Crew'; + @field label = contains(StringField); + @field lead = linksTo(Agent); + @field roster = linksToMany(Agent); + } + class ArticleContainsLead extends CardDef { + static displayName = 'ArticleContainsLead'; + @field title = contains(StringField); + @field crew = contains(Crew, { searchable: 'lead' }); // contains → linksTo + } + class ArticleContainsRoster extends CardDef { + static displayName = 'ArticleContainsRoster'; + @field title = contains(StringField); + @field crew = contains(Crew, { searchable: 'roster' }); // contains → linksToMany + } + class ArticleManyLead extends CardDef { + static displayName = 'ArticleManyLead'; + @field title = contains(StringField); + @field crews = containsMany(Crew, { searchable: 'lead' }); // containsMany → linksTo + } + class ArticleManyRoster extends CardDef { + static displayName = 'ArticleManyRoster'; + @field title = contains(StringField); + @field crews = containsMany(Crew, { searchable: 'roster' }); // containsMany → linksToMany + } + class ArticleLabels extends CardDef { + static displayName = 'ArticleLabels'; + @field title = contains(StringField); + @field labels = containsMany(StringField); + } + + // ---- linksToMany ------------------------------------------------------- + // Plural self-link to Author: members expand, but each member.agent stays + // {id} (the pulled-in Author's own `searchable` is dormant). + class Team extends CardDef { + static displayName = 'Team'; + @field name = contains(StringField); + @field members = linksToMany(Author, { searchable: true }); + } + class TeamShallow extends CardDef { + static displayName = 'TeamShallow'; + @field name = contains(StringField); + @field members = linksToMany(Agent); // unannotated → [{id}] + } + class TeamDeep extends CardDef { + static displayName = 'TeamDeep'; + @field name = contains(StringField); + // Deep route into each plural element. + @field members = linksToMany(Author, { searchable: 'agent' }); + } + + // ---- declared-type enumeration / parity -------------------------------- + class SimpleAuthor extends CardDef { + static displayName = 'SimpleAuthor'; + @field name = contains(StringField); + } + class FancyAuthor extends SimpleAuthor { + static displayName = 'FancyAuthor'; + @field penName = contains(StringField); + } + class ParityArticle extends CardDef { + static displayName = 'ParityArticle'; + @field title = contains(StringField); + @field authors = linksToMany(SimpleAuthor, { searchable: true }); + static isolated = class extends Component { + + }; + } + class ArticleSubtype extends CardDef { + static displayName = 'ArticleSubtype'; + @field title = contains(StringField); + @field author = linksTo(SimpleAuthor, { searchable: true }); + } + class TeamSubtype extends CardDef { + static displayName = 'TeamSubtype'; + @field title = contains(StringField); + @field members = linksToMany(SimpleAuthor, { searchable: true }); + } + // A polymorphic contained value: the instance holds a FancyProfile, but the + // declared field type is Profile, so the subtype's `tagline` is dropped. + class Profile extends FieldDef { + static displayName = 'Profile'; + @field bio = contains(StringField); + } + class FancyProfile extends Profile { + static displayName = 'FancyProfile'; + @field tagline = contains(StringField); + } + class ArticleProfile extends CardDef { + static displayName = 'ArticleProfile'; + @field title = contains(StringField); + @field profile = contains(Profile); // unannotated — contains is always included + } + + // ---- query-backed field (must never appear in the doc) ----------------- + class ArticleQuery extends CardDef { + static displayName = 'ArticleQuery'; + @field title = contains(StringField); + @field related = linksToMany(() => Agent, { + searchable: true, + query: { filter: { eq: { name: 'Agent Smith' } } }, + }); + } + + let agentRef = (id: string) => ({ links: { self: id } }); + + ({ realm } = await setupIntegrationTestRealm({ + mockMatrixUtils, + contents: { + 'agent.gts': { Agent }, + 'headquarters.gts': { Headquarters }, + 'company.gts': { Company }, + 'author.gts': { Author }, + 'article.gts': { + ArticleSelf, + ArticleDeep, + ArticleShallow, + ArticleHop2, + ArticleHop3, + ArticleShared, + ArticleMulti, + ArticleEmptyPath, + ArticleEmptyArray, + ArticleNullSearchable, + ArticleArrayWithNull, + ArticleImpossiblePath, + ArticleFalse, + }, + 'person.gts': { Person }, + 'ring.gts': { Ring }, + 'ring-m.gts': { RingM }, + 'crew.gts': { + Crew, + ArticleContainsLead, + ArticleContainsRoster, + ArticleManyLead, + ArticleManyRoster, + ArticleLabels, + }, + 'team.gts': { Team, TeamShallow, TeamDeep, TeamSubtype }, + 'parity.gts': { + SimpleAuthor, + FancyAuthor, + ParityArticle, + ArticleSubtype, + }, + 'profile.gts': { Profile, FancyProfile, ArticleProfile }, + 'article-query.gts': { ArticleQuery }, + + // --- leaves + chain --- + 'Agent/a1.json': card('Agent Smith', 'agent', 'Agent'), + 'Agent/a2.json': card('Agent Jones', 'agent', 'Agent'), + 'Headquarters/h1.json': card('HQ One', 'headquarters', 'Headquarters'), + 'Company/co1.json': { + data: { + type: 'card', + id: `${testRealmURL}Company/co1`, + attributes: { name: 'Acme' }, + relationships: { + ceo: agentRef(`${testRealmURL}Agent/a1`), + hq: agentRef(`${testRealmURL}Headquarters/h1`), + }, + meta: adoptsFrom('company', 'Company'), + }, + }, + 'Author/au1.json': { + data: { + type: 'card', + id: `${testRealmURL}Author/au1`, + attributes: { name: 'Jo' }, + relationships: { + agent: agentRef(`${testRealmURL}Agent/a1`), + company: agentRef(`${testRealmURL}Company/co1`), + }, + meta: adoptsFrom('author', 'Author'), + }, + }, + 'Author/au2.json': { + data: { + type: 'card', + id: `${testRealmURL}Author/au2`, + attributes: { name: 'Mit' }, + relationships: { agent: agentRef(`${testRealmURL}Agent/a2`) }, + meta: adoptsFrom('author', 'Author'), + }, + }, + + // --- linksTo route shapes (all → Author/au1) --- + 'ArticleSelf/s1.json': article('Self', 'ArticleSelf', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + // Relative `links.self` — must resolve before the targeted load. + 'ArticleSelf/rel.json': article('Relative', 'ArticleSelf', { + author: agentRef('../Author/au1'), + }), + // Points at a card that does not exist (broken / 404 target). + 'ArticleSelf/broken.json': article('Broken', 'ArticleSelf', { + author: agentRef(`${testRealmURL}Author/ghost`), + }), + // author = null (no link at all). + 'ArticleSelf/nulllink.json': article('Null', 'ArticleSelf', { + author: { links: { self: null } }, + }), + 'ArticleDeep/d1.json': article('Deep', 'ArticleDeep', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + 'ArticleShallow/sh1.json': article('Shallow', 'ArticleShallow', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + 'ArticleHop2/h2.json': article('Hop2', 'ArticleHop2', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + 'ArticleHop3/h3.json': article('Hop3', 'ArticleHop3', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + 'ArticleShared/shr1.json': article('Shared', 'ArticleShared', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + 'ArticleMulti/m1.json': article('Multi', 'ArticleMulti', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + 'ArticleEmptyPath/ep1.json': article('Empty', 'ArticleEmptyPath', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + 'ArticleEmptyArray/ea1.json': article('EArr', 'ArticleEmptyArray', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + 'ArticleNullSearchable/ns1.json': article( + 'Nul', + 'ArticleNullSearchable', + { author: agentRef(`${testRealmURL}Author/au1`) }, + ), + 'ArticleArrayWithNull/awn1.json': article( + 'AwN', + 'ArticleArrayWithNull', + { author: agentRef(`${testRealmURL}Author/au1`) }, + ), + 'ArticleImpossiblePath/ip1.json': article( + 'Imp', + 'ArticleImpossiblePath', + { author: agentRef(`${testRealmURL}Author/au1`) }, + ), + 'ArticleFalse/f1.json': article('Fls', 'ArticleFalse', { + author: agentRef(`${testRealmURL}Author/au1`), + }), + + // --- cycles --- + 'Person/p1.json': { + data: { + type: 'card', + id: `${testRealmURL}Person/p1`, + attributes: { name: 'Solo' }, + relationships: { + friend: agentRef(`${testRealmURL}Person/p1`), + }, + meta: adoptsFrom('person', 'Person'), + }, + }, + 'Ring/r1.json': ring('R1', `${testRealmURL}Ring/r2`), + 'Ring/r2.json': ring('R2', `${testRealmURL}Ring/r3`), + 'Ring/r3.json': ring('R3', `${testRealmURL}Ring/r1`), + 'RingM/rm1.json': ringM('R1m', `${testRealmURL}RingM/rm2`), + 'RingM/rm2.json': ringM('R2m', `${testRealmURL}RingM/rm3`), + 'RingM/rm3.json': ringM('R3m', `${testRealmURL}RingM/rm1`), + + // --- contained values holding links (4 combinations) --- + // A single contained Crew: label + lead(linksTo) + roster(linksToMany). + 'ArticleContainsLead/cl1.json': { + data: { + type: 'card', + attributes: { title: 'CL', crew: { label: 'Alpha' } }, + relationships: { + 'crew.lead': agentRef(`${testRealmURL}Agent/a1`), + 'crew.roster.0': agentRef(`${testRealmURL}Agent/a1`), + 'crew.roster.1': agentRef(`${testRealmURL}Agent/a2`), + }, + meta: adoptsFrom('crew', 'ArticleContainsLead'), + }, + }, + 'ArticleContainsRoster/cr1.json': { + data: { + type: 'card', + attributes: { title: 'CR', crew: { label: 'Alpha' } }, + relationships: { + 'crew.lead': agentRef(`${testRealmURL}Agent/a1`), + 'crew.roster.0': agentRef(`${testRealmURL}Agent/a1`), + 'crew.roster.1': agentRef(`${testRealmURL}Agent/a2`), + }, + meta: adoptsFrom('crew', 'ArticleContainsRoster'), + }, + }, + // Two contained Crews, each with its own label + lead + roster. + 'ArticleManyLead/ml1.json': { + data: { + type: 'card', + attributes: { + title: 'ML', + crews: [{ label: 'C0' }, { label: 'C1' }], + }, + relationships: { + 'crews.0.lead': agentRef(`${testRealmURL}Agent/a1`), + 'crews.0.roster.0': agentRef(`${testRealmURL}Agent/a1`), + 'crews.1.lead': agentRef(`${testRealmURL}Agent/a2`), + 'crews.1.roster.0': agentRef(`${testRealmURL}Agent/a2`), + }, + meta: adoptsFrom('crew', 'ArticleManyLead'), + }, + }, + 'ArticleManyRoster/mr1.json': { + data: { + type: 'card', + attributes: { + title: 'MR', + crews: [{ label: 'C0' }, { label: 'C1' }], + }, + relationships: { + 'crews.0.lead': agentRef(`${testRealmURL}Agent/a1`), + 'crews.0.roster.0': agentRef(`${testRealmURL}Agent/a1`), + 'crews.1.lead': agentRef(`${testRealmURL}Agent/a2`), + 'crews.1.roster.0': agentRef(`${testRealmURL}Agent/a2`), + }, + meta: adoptsFrom('crew', 'ArticleManyRoster'), + }, + }, + // --- containsMany of primitives --- + 'ArticleLabels/l1.json': { + data: { + type: 'card', + id: `${testRealmURL}ArticleLabels/l1`, + attributes: { title: 'Labels', labels: ['red', 'blue'] }, + meta: adoptsFrom('crew', 'ArticleLabels'), + }, + }, + 'ArticleLabels/empty1.json': { + data: { + type: 'card', + id: `${testRealmURL}ArticleLabels/empty1`, + attributes: { title: 'Empty', labels: [] }, + meta: adoptsFrom('crew', 'ArticleLabels'), + }, + }, + + // --- linksToMany --- + 'Team/valid.json': team('Valid', 'Team', [ + `${testRealmURL}Author/au1`, + `${testRealmURL}Author/au2`, + ]), + 'Team/missone.json': team('MissOne', 'Team', [ + `${testRealmURL}Author/au1`, + `${testRealmURL}Author/ghost1`, + ]), + 'Team/missall.json': team('MissAll', 'Team', [ + `${testRealmURL}Author/ghost1`, + `${testRealmURL}Author/ghost2`, + ]), + 'Team/empty.json': team('EmptyTeam', 'Team', []), + 'TeamShallow/ts1.json': team('Shallow', 'TeamShallow', [ + `${testRealmURL}Agent/a1`, + ]), + 'TeamDeep/td1.json': team('Deep', 'TeamDeep', [ + `${testRealmURL}Author/au1`, + ]), + + // --- declared type / parity --- + 'SimpleAuthor/sa1.json': card('Plain', 'parity', 'SimpleAuthor'), + 'FancyAuthor/fa1.json': { + data: { + type: 'card', + id: `${testRealmURL}FancyAuthor/fa1`, + attributes: { name: 'Fancy', penName: 'Quill' }, + meta: adoptsFrom('parity', 'FancyAuthor'), + }, + }, + 'ParityArticle/pa1.json': { + data: { + type: 'card', + id: `${testRealmURL}ParityArticle/pa1`, + attributes: { title: 'Parity' }, + relationships: { + 'authors.0': agentRef(`${testRealmURL}SimpleAuthor/sa1`), + }, + meta: adoptsFrom('parity', 'ParityArticle'), + }, + }, + 'ArticleSubtype/sub1.json': { + data: { + type: 'card', + attributes: { title: 'Subtype' }, + relationships: { + author: agentRef(`${testRealmURL}FancyAuthor/fa1`), + }, + meta: adoptsFrom('parity', 'ArticleSubtype'), + }, + }, + 'TeamSubtype/tsub1.json': team('SubtypeTeam', 'TeamSubtype', [ + `${testRealmURL}FancyAuthor/fa1`, + ]), + 'ArticleProfile/prof1.json': { + data: { + type: 'card', + id: `${testRealmURL}ArticleProfile/prof1`, + attributes: { + title: 'Profile', + profile: { bio: 'a bio', tagline: 'a tagline' }, + }, + meta: { + adoptsFrom: { + module: `${testRealmURL}profile`, + name: 'ArticleProfile', + }, + fields: { + profile: { + adoptsFrom: { + module: `${testRealmURL}profile`, + name: 'FancyProfile', + }, + }, + }, + }, + }, + }, + + // --- query-backed field --- + 'ArticleQuery/q1.json': { + data: { + type: 'card', + id: `${testRealmURL}ArticleQuery/q1`, + attributes: { title: 'Query' }, + meta: adoptsFrom('article-query', 'ArticleQuery'), + }, + }, + }, + })); + }); + + // --- fixture builders ----------------------------------------------------- + function adoptsFrom(mod: string, name: string) { + return { adoptsFrom: { module: `${testRealmURL}${mod}`, name } }; + } + function card(name: string, mod: string, klass: string) { + return { + data: { + type: 'card', + attributes: { name }, + meta: adoptsFrom(mod, klass), + }, + }; + } + function article(title: string, klass: string, relationships: any) { + return { + data: { + type: 'card', + attributes: { title }, + relationships, + meta: adoptsFrom('article', klass), + }, + }; + } + function ring(name: string, next: string) { + return { + data: { + type: 'card', + attributes: { name }, + relationships: { next: { links: { self: next } } }, + meta: adoptsFrom('ring', 'Ring'), + }, + }; + } + function ringM(name: string, next: string) { + return { + data: { + type: 'card', + attributes: { name }, + relationships: { 'nexts.0': { links: { self: next } } }, + meta: adoptsFrom('ring-m', 'RingM'), + }, + }; + } + function team(name: string, klass: string, members: string[]) { + let relationships: any = {}; + members.forEach((m, i) => { + relationships[`members.${i}`] = { links: { self: m } }; + }); + return { + data: { + type: 'card', + attributes: { name }, + relationships, + meta: adoptsFrom('team', klass), + }, + }; + } + + async function loadAndGenerate(id: string) { + let store = getService('store') as StoreService; + let instance = (await store.get(id)) as CardDefType; + return await searchDocFromFields(instance); + } + + // The store-driven search doc the indexer produced, minus `_cardType` (which + // the prerender meta route appends, not the generator) — the differential + // parity baseline. + async function storeDrivenSearchDoc(id: string) { + let entry = await realm.realmIndexQueryEngine.instance(new URL(id)); + if (!entry || entry.type === 'instance-error') { + return undefined; + } + let { _cardType, ...rest } = (entry as IndexedInstance).searchDoc ?? {}; + return rest; + } + + let agentUrl = `${testRealmURL}Agent/a1`; + let agent2Url = `${testRealmURL}Agent/a2`; + let authorUrl = `${testRealmURL}Author/au1`; + let hqUrl = `${testRealmURL}Headquarters/h1`; + + // =========================================================================== + // Route seeding: the searchable forms and where depth comes from + // =========================================================================== + + test('routes come ONLY from the indexed card: a pulled-in target does not consult its own searchable', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleSelf/s1`); + assert.strictEqual(doc.author?.name, 'Jo', 'author is expanded'); + assert.deepEqual( + doc.author?.agent, + { id: agentUrl }, + "the pulled-in Author's own searchable agent link stays { id }", + ); + }); + + test('the same card indexed directly DOES honor its own searchable', async function (assert) { + let doc = await loadAndGenerate(authorUrl); + assert.strictEqual( + doc.agent?.name, + 'Agent Smith', + 'Author.agent expands when Author is the card being indexed', + ); + }); + + test('a single dotted route on the indexed card expands the deeper link', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleDeep/d1`); + assert.strictEqual( + doc.author?.agent?.name, + 'Agent Smith', + 'the route `author.agent` drives the depth', + ); + }); + + test('an unannotated link stays { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleShallow/sh1`); + assert.deepEqual(doc.author, { id: authorUrl }, 'captured as { id } only'); + }); + + test("an empty-string path ('') behaves as a self link", async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleEmptyPath/ep1`); + assert.strictEqual(doc.author?.name, 'Jo', 'author is expanded'); + assert.deepEqual( + doc.author?.agent, + { id: agentUrl }, + 'no deeper than the self link', + ); + }); + + // --- malformed / impossible searchable values (must degrade, never crash) -- + + test('an empty searchable array leaves the link as { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleEmptyArray/ea1`); + assert.deepEqual(doc.author, { id: authorUrl }, 'no route → { id }'); + }); + + test('a null searchable annotation is treated as unannotated', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleNullSearchable/ns1`); + assert.deepEqual(doc.author, { id: authorUrl }, 'null → { id }'); + }); + + test('a `false` searchable value (bad input) leaves the link as { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleFalse/f1`); + assert.deepEqual(doc.author, { id: authorUrl }, 'false → { id }'); + }); + + test('a searchable array with a non-string entry ignores it and still expands the valid route', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleArrayWithNull/awn1`); + assert.strictEqual(doc.title, 'AwN', 'did not crash'); + assert.strictEqual( + doc.author?.agent?.name, + 'Agent Smith', + 'the valid `agent` route still expands', + ); + }); + + test('a searchable path naming a non-existent field expands the reachable prefix only', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleImpossiblePath/ip1`); + assert.strictEqual( + doc.author?.agent?.name, + 'Agent Smith', + 'the reachable prefix (agent) expands', + ); + assert.notOk( + 'deeper' in (doc.author?.agent ?? {}), + 'the unreachable tail is a no-op (no junk key)', + ); + }); + + // =========================================================================== + // Multi-segment depth and the "dormant when pulled in" rule at depth + // =========================================================================== + + test('a two-hop route expands the intermediate but leaves its further links { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleHop2/h2`); + assert.strictEqual( + doc.author?.company?.name, + 'Acme', + 'company is expanded', + ); + assert.deepEqual( + doc.author?.company?.ceo, + { id: agentUrl }, + "the pulled-in Company's own searchable ceo stays { id } at depth 2", + ); + assert.deepEqual( + doc.author?.company?.hq, + { id: hqUrl }, + "the pulled-in Company's own searchable hq stays { id } at depth 2", + ); + }); + + test('a three-segment route (a.b.c) expands all the way down', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleHop3/h3`); + assert.strictEqual( + doc.author?.company?.hq?.name, + 'HQ One', + 'the route `author.company.hq` expands the third hop', + ); + assert.deepEqual( + doc.author?.company?.ceo, + { id: agentUrl }, + 'a sibling of the routed link (ceo) stays { id }', + ); + }); + + test('an array of routes sharing an ancestor collapses through that ancestor once', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleShared/shr1`); + assert.strictEqual( + doc.author?.company?.name, + 'Acme', + 'the shared ancestor `company` is expanded', + ); + assert.strictEqual( + doc.author?.company?.ceo?.name, + 'Agent Smith', + 'the first divergent route (company.ceo) expands', + ); + assert.strictEqual( + doc.author?.company?.hq?.name, + 'HQ One', + 'the second divergent route (company.hq) expands under the same ancestor', + ); + }); + + test('an array of routes with divergent heads expands each independently', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleMulti/m1`); + assert.strictEqual( + doc.author?.agent?.name, + 'Agent Smith', + 'the self-link route (agent) expands', + ); + assert.strictEqual( + doc.author?.company?.hq?.name, + 'HQ One', + 'the deep route (company.hq) expands', + ); + assert.deepEqual( + doc.author?.company?.ceo, + { id: agentUrl }, + 'an unrouted sibling (company.ceo) stays { id }', + ); + }); + + // =========================================================================== + // linksTo value states + // =========================================================================== + + test('a relative link reference is resolved before the targeted load', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleSelf/rel`); + assert.strictEqual( + doc.author?.name, + 'Jo', + 'a relative links.self resolves', + ); + }); + + test('a searchable link to a missing target degrades to { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleSelf/broken`); + assert.deepEqual( + doc.author, + { id: `${testRealmURL}Author/ghost` }, + 'an unloadable link keeps its reference as { id }', + ); + }); + + test('a null link is captured as null', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleSelf/nulllink`); + assert.strictEqual(doc.author, null, 'a missing link slot is null'); + }); + + // =========================================================================== + // contains / containsMany — routing through a FieldDef into its links. + // All four combinations (contains|containsMany × linksTo|linksToMany), with + // the contained scalar ALWAYS present and any unrouted link inside kept as + // { id }. + // =========================================================================== + + test('contains → linksTo: routed link expands; contained scalar + unrouted plural still present', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleContainsLead/cl1`); + assert.strictEqual( + doc.crew?.label, + 'Alpha', + 'the contained scalar is always included', + ); + assert.strictEqual( + doc.crew?.lead?.name, + 'Agent Smith', + 'the routed linksTo expands (target contained field present)', + ); + assert.deepEqual( + doc.crew?.roster, + [{ id: agentUrl }, { id: agent2Url }], + 'the unrouted linksToMany inside is still present as [{ id }]', + ); + }); + + test('contains → linksToMany: routed plural expands; unrouted linksTo stays { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleContainsRoster/cr1`); + assert.strictEqual( + doc.crew?.label, + 'Alpha', + 'the contained scalar is always included', + ); + assert.deepEqual( + (doc.crew?.roster ?? []).map((m: any) => m.name), + ['Agent Smith', 'Agent Jones'], + 'the routed linksToMany expands every element', + ); + assert.deepEqual( + doc.crew?.lead, + { id: agentUrl }, + 'the unrouted linksTo inside stays { id }', + ); + }); + + test('containsMany → linksTo: each element routes its link; each element scalar present', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleManyLead/ml1`); + assert.deepEqual( + (doc.crews ?? []).map((c: any) => c.label), + ['C0', 'C1'], + 'every element scalar is always included', + ); + assert.strictEqual( + doc.crews?.[0]?.lead?.name, + 'Agent Smith', + "the first element's link expands", + ); + assert.strictEqual( + doc.crews?.[1]?.lead?.name, + 'Agent Jones', + "the second element's link expands", + ); + assert.deepEqual( + doc.crews?.[0]?.roster, + [{ id: agentUrl }], + 'the unrouted plural in the element is [{ id }]', + ); + }); + + test('containsMany → linksToMany: each element routes its plural', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleManyRoster/mr1`); + assert.strictEqual( + doc.crews?.[0]?.roster?.[0]?.name, + 'Agent Smith', + "the first element's plural expands", + ); + assert.strictEqual( + doc.crews?.[1]?.roster?.[0]?.name, + 'Agent Jones', + "the second element's plural expands", + ); + assert.deepEqual( + doc.crews?.[0]?.lead, + { id: agentUrl }, + 'the unrouted linksTo in the element stays { id }', + ); + assert.strictEqual( + doc.crews?.[0]?.label, + 'C0', + 'the element scalar is present', + ); + }); + + test('a contained value is enumerated by its DECLARED type (subtype field dropped)', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleProfile/prof1`); + assert.strictEqual( + doc.profile?.bio, + 'a bio', + 'the declared contained field is present (unannotated contains is always included)', + ); + assert.notOk( + 'tagline' in (doc.profile ?? {}), + 'the polymorphic subtype-only field is dropped', + ); + }); + + test('containsMany of primitives is captured as an array', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleLabels/l1`); + assert.deepEqual( + doc.labels, + ['red', 'blue'], + 'all primitive items captured', + ); + }); + + test('an empty containsMany is null', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleLabels/empty1`); + assert.strictEqual(doc.labels, null, 'empty containsMany is null'); + }); + + test('contained scalar fields are always present regardless of searchable', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleShallow/sh1`); + assert.strictEqual( + doc.title, + 'Shallow', + 'the unrouted contained scalar is present', + ); + assert.deepEqual( + doc.author, + { id: authorUrl }, + 'while the unannotated link stays { id }', + ); + }); + + // =========================================================================== + // linksToMany + // =========================================================================== + + test('a searchable linksToMany expands every element', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}Team/valid`); + assert.deepEqual( + (doc.members ?? []).map((m: any) => m.name), + ['Jo', 'Mit'], + 'both members are expanded in slot order', + ); + }); + + test('linksToMany targets do NOT consult their own searchable', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}Team/valid`); + assert.deepEqual( + doc.members?.[0]?.agent, + { id: agentUrl }, + "the pulled-in member's own searchable agent link stays { id }", + ); + }); + + test('an unannotated linksToMany is an array of { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}TeamShallow/ts1`); + assert.deepEqual( + doc.members, + [{ id: agentUrl }], + 'each slot is { id } only', + ); + }); + + test('a deep route into a linksToMany expands each element along it', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}TeamDeep/td1`); + assert.strictEqual( + doc.members?.[0]?.agent?.name, + 'Agent Smith', + 'the route `members.agent` drives the depth into each element', + ); + }); + + test('a linksToMany with one missing slot expands the rest and keeps the missing one as { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}Team/missone`); + assert.strictEqual( + doc.members?.[0]?.name, + 'Jo', + 'the present member expands', + ); + assert.deepEqual( + doc.members?.[1], + { id: `${testRealmURL}Author/ghost1` }, + 'the missing member is { id }', + ); + }); + + test('a linksToMany with all slots missing is an array of { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}Team/missall`); + assert.deepEqual( + doc.members, + [ + { id: `${testRealmURL}Author/ghost1` }, + { id: `${testRealmURL}Author/ghost2` }, + ], + 'every missing slot keeps its reference as { id }', + ); + }); + + test('an empty linksToMany is null', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}Team/empty`); + assert.strictEqual(doc.members, null, 'empty linksToMany is null'); + }); + + // =========================================================================== + // cycles + // =========================================================================== + + test('a self-referential link clips the cycle to { id }', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}Person/p1`); + assert.deepEqual( + doc.friend, + { id: `${testRealmURL}Person/p1` }, + 'a self link clips to { id }', + ); + }); + + test('a three-card ring (linksTo) walks the ring then clips on re-entry', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}Ring/r1`); + assert.strictEqual(doc.next?.name, 'R2', 'first hop'); + assert.strictEqual(doc.next?.next?.name, 'R3', 'second hop'); + assert.deepEqual( + doc.next?.next?.next, + { id: `${testRealmURL}Ring/r1` }, + 'the fourth hop re-enters r1 and clips to { id }', + ); + }); + + test('a three-card ring (linksToMany) walks the ring then clips on re-entry', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}RingM/rm1`); + assert.strictEqual(doc.nexts?.[0]?.name, 'R2m', 'first hop'); + assert.strictEqual(doc.nexts?.[0]?.nexts?.[0]?.name, 'R3m', 'second hop'); + assert.deepEqual( + doc.nexts?.[0]?.nexts?.[0]?.nexts, + [{ id: `${testRealmURL}RingM/rm1` }], + 'the fourth hop re-enters rm1 and clips to { id }', + ); + }); + + // =========================================================================== + // skips and declared-type enumeration + // =========================================================================== + + test('a query-backed field never appears in the doc', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleQuery/q1`); + assert.strictEqual(doc.title, 'Query', 'plain fields are present'); + assert.notOk('related' in doc, 'the query-backed field is skipped'); + }); + + test('a linksTo target is enumerated by its DECLARED type (subtype bloat dropped)', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}ArticleSubtype/sub1`); + assert.strictEqual( + doc.author?.name, + 'Fancy', + 'the declared field is present', + ); + assert.notOk( + 'penName' in (doc.author ?? {}), + 'the subtype-only field is dropped', + ); + }); + + test('a linksToMany target is enumerated by its DECLARED type (subtype bloat dropped)', async function (assert) { + let doc = await loadAndGenerate(`${testRealmURL}TeamSubtype/tsub1`); + assert.strictEqual( + doc.members?.[0]?.name, + 'Fancy', + 'the declared field is present', + ); + assert.notOk( + 'penName' in (doc.members?.[0] ?? {}), + 'the subtype-only field is dropped', + ); + }); + + // =========================================================================== + // differential parity with the store-driven doc + // =========================================================================== + + // Byte-for-byte equality of the whole doc is NOT asserted: the + // searchable-driven spec keeps `{ id }`/`null` for every relationship while + // the store-driven path omits unused links via `usedLinksToFieldsOnly`, and + // the two enumerate link targets at different type granularity. This proves + // the expansion matches; whole-doc equality is the concern of the realm-scale + // parity diff, once realms carry `searchable` annotations. + test('searchable expansion pulls in the same target+data as the store-driven load', async function (assert) { + let generated = await loadAndGenerate(`${testRealmURL}ParityArticle/pa1`); + let storeDriven = await storeDrivenSearchDoc( + `${testRealmURL}ParityArticle/pa1`, + ); + assert.deepEqual( + (generated.authors ?? []).map((a: any) => a.id), + (storeDriven?.authors ?? []).map((a: any) => a.id), + 'follows the searchable link to the same target the store loaded', + ); + assert.deepEqual( + (generated.authors ?? []).map((a: any) => a.name), + (storeDriven?.authors ?? []).map((a: any) => a.name), + 'pulls the same contained data from the expanded target', + ); + }); +}); diff --git a/packages/host/tests/unit/searchable-option-test.ts b/packages/host/tests/unit/searchable-option-test.ts index 75a0ef8b16..0d43872eb7 100644 --- a/packages/host/tests/unit/searchable-option-test.ts +++ b/packages/host/tests/unit/searchable-option-test.ts @@ -122,8 +122,8 @@ module('Unit | searchable option', function (hooks) { }); } // A FieldDef that itself declares a link, so a `searchable` path can route - // *through* a contained value to reach a deeper link (the §4 citations - // case). FieldDefs may declare linksTo — see e.g. base/skill-reference. + // *through* a contained value to reach a deeper link. FieldDefs may declare + // linksTo — see e.g. base/skill-reference. class Citation extends FieldDef { @field label = contains(StringField); @field article = linksTo(() => Author); diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json index 4ccc2d57a3..b22af29d60 100644 --- a/packages/realm-server/package.json +++ b/packages/realm-server/package.json @@ -76,6 +76,7 @@ "qs": "catalog:", "qunit": "catalog:", "recast": "catalog:", + "safe-stable-stringify": "catalog:", "sane": "catalog:", "sinon": "catalog:", "start-server-and-test": "catalog:", diff --git a/packages/realm-server/scripts/searchable-parity-diff.ts b/packages/realm-server/scripts/searchable-parity-diff.ts new file mode 100644 index 0000000000..e18b72b234 --- /dev/null +++ b/packages/realm-server/scripts/searchable-parity-diff.ts @@ -0,0 +1,184 @@ +/** + * searchable-parity-diff — compare a realm's LIVE search docs (store-driven, + * from `boxel_index`) against the searchable-driven generator's output, per card. + * + * This is the realm-scale parity validator for the "generate search doc from + * field definitions only" project. It is meaningful only once a realm's cards + * carry `searchable` annotations that make the new generator reproduce the + * depth the store-driven path produces; without those annotations the two + * paths differ by design (the searchable-driven spec keeps `{ id }` for every + * relationship; the store-driven path omits unused links via + * `usedLinksToFieldsOnly`). The CI fixture test in + * `packages/host/tests/integration/searchable-search-doc-test.gts` covers the + * generator's behavior; this script is the realm-scale before/after check. + * + * It takes two JSON files, each a map of `{ : }`: + * --live the store-driven docs. Gather from staging/prod + * read-only (see the `aws-access` + `indexing-diagnostics` + * skills), e.g. over the SSM psql tunnel: + * SELECT url, search_doc FROM boxel_index + * WHERE realm_url = $1 AND type = 'instance'; + * then shape the rows into `{ url: search_doc }`. + * --generated the `searchDocFromFields` output for the same cards, + * produced in a host environment (the generator needs the + * loader/store; it can't run in plain node). Pull the + * realm source with `boxel realm pull`, index/load it, + * and dump `await searchDocFromFields(instance)` per card. + * + * Output: a per-card report of real divergences and a non-zero exit if any are + * found. `_cardType` (appended by the prerender meta route, not the generator) + * is ignored. With `--ignore-shallow-links`, a relationship that is `{ id }`-only + * (or null) on one side and absent on the other is treated as equivalent — the + * known, intended `{ id }`-vs-omitted difference — so the report surfaces only + * divergences that matter (changed expansions, missing data). + * + * Usage: + * node packages/realm-server/scripts/searchable-parity-diff.ts \ + * --live live.json --generated generated.json [--ignore-shallow-links] + */ +import { readFileSync } from 'fs'; +import { fileURLToPath } from 'url'; +import stringify from 'safe-stable-stringify'; + +type SearchDoc = Record; +type DocMap = Record; + +function parseArgs(argv: string[]) { + let args: { + live?: string; + generated?: string; + ignoreShallowLinks: boolean; + } = { ignoreShallowLinks: false }; + for (let i = 0; i < argv.length; i++) { + let a = argv[i]; + if (a === '--live') args.live = argv[++i]; + else if (a === '--generated') args.generated = argv[++i]; + else if (a === '--ignore-shallow-links') args.ignoreShallowLinks = true; + } + if (!args.live || !args.generated) { + throw new Error( + 'usage: searchable-parity-diff --live --generated [--ignore-shallow-links]', + ); + } + return args as { + live: string; + generated: string; + ignoreShallowLinks: boolean; + }; +} + +// A relationship slot is "shallow" when it carries no contained data beyond a +// bare reference: `null`, a bare `{ id }`, or a plural whose every element is +// shallow (an empty plural included). The store-driven path omits unused links +// while this generator keeps the `{ id }`, so under --ignore-shallow-links a +// shallow-vs-absent slot is treated as equivalent — see `diffDoc`. +export function isShallowLink(value: unknown): boolean { + if (value == null) return true; + if (Array.isArray(value)) return value.every(isShallowLink); + if (typeof value !== 'object') return false; + let keys = Object.keys(value as object); + return keys.length === 1 && keys[0] === 'id'; +} + +// The bare reference ids carried by a shallow slot, flattened across a plural. +// A `null` / absent / empty slot contributes none. Used to tell the intended +// omit-vs-keep-`{id}` difference (one side has no ids) apart from a CHANGED +// reference (`{id:A}` vs `{id:B}`), which is a real divergence worth reporting. +export function shallowIds(value: unknown): string[] { + if (value == null) return []; + if (Array.isArray(value)) return value.flatMap(shallowIds); + if (typeof value === 'object') { + let id = (value as { id?: unknown }).id; + return typeof id === 'string' ? [id] : []; + } + return []; +} + +export function diffDoc( + live: SearchDoc, + generated: SearchDoc, + ignoreShallowLinks: boolean, +): string[] { + let diffs: string[] = []; + let strip = (d: SearchDoc) => { + let { _cardType, ...rest } = d; + return rest; + }; + let l = strip(live ?? {}); + let g = strip(generated ?? {}); + let keys = new Set([...Object.keys(l), ...Object.keys(g)]); + for (let key of keys) { + let lPresent = key in l; + let gPresent = key in g; + let lv = (l as SearchDoc)[key]; + let gv = (g as SearchDoc)[key]; + if (ignoreShallowLinks && isShallowLink(lv) && isShallowLink(gv)) { + let lIds = shallowIds(lv); + let gIds = shallowIds(gv); + // Omit-vs-keep-`{id}` (one side carries no ids) is the intended, + // ignored difference. A changed reference (both sides present, ids + // differ) is a real divergence and falls through to be reported. + if (lIds.length === 0 || gIds.length === 0) { + continue; + } + if (stringify(lIds) === stringify(gIds)) { + continue; + } + } + let ls = lPresent ? (stringify(lv) ?? 'null') : 'absent'; + let gs = gPresent ? (stringify(gv) ?? 'null') : 'absent'; + if (ls !== gs) { + diffs.push(` ${key}: live=${ls} generated=${gs}`); + } + } + return diffs; +} + +function main() { + let { live, generated, ignoreShallowLinks } = parseArgs( + process.argv.slice(2), + ); + let liveDocs = JSON.parse(readFileSync(live, 'utf8')) as DocMap; + let generatedDocs = JSON.parse(readFileSync(generated, 'utf8')) as DocMap; + + let urls = new Set([...Object.keys(liveDocs), ...Object.keys(generatedDocs)]); + let divergent = 0; + let onlyLive = 0; + let onlyGenerated = 0; + for (let url of urls) { + if (!(url in generatedDocs)) { + onlyLive++; + console.log(`MISSING from generated: ${url}`); + continue; + } + if (!(url in liveDocs)) { + onlyGenerated++; + console.log(`MISSING from live: ${url}`); + continue; + } + let docDiffs = diffDoc( + liveDocs[url], + generatedDocs[url], + ignoreShallowLinks, + ); + if (docDiffs.length > 0) { + divergent++; + console.log(`DIVERGENT ${url}`); + for (let d of docDiffs) console.log(d); + } + } + + console.log( + `\n${urls.size} cards compared — ${divergent} divergent, ${onlyLive} live-only, ${onlyGenerated} generated-only` + + (ignoreShallowLinks ? ' (shallow-link differences ignored)' : ''), + ); + if (divergent > 0 || onlyLive > 0 || onlyGenerated > 0) { + process.exitCode = 1; + } +} + +// Run only when invoked directly as a script — importing the pure functions +// above (e.g. from a test) must not execute the file I/O in `main`. +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main(); +} diff --git a/packages/realm-server/tests/index.ts b/packages/realm-server/tests/index.ts index e80917a89c..01803c1560 100644 --- a/packages/realm-server/tests/index.ts +++ b/packages/realm-server/tests/index.ts @@ -224,6 +224,7 @@ const ALL_TEST_FILES: string[] = [ './codemod-context-search-test', './cpu-profiler-affinity-gate-test', './definition-lookup-test', + './searchable-parity-diff-test', './file-watcher-events-test', './full-index-on-startup-test', './full-reindex-test', diff --git a/packages/realm-server/tests/searchable-parity-diff-test.ts b/packages/realm-server/tests/searchable-parity-diff-test.ts new file mode 100644 index 0000000000..c60c4036ef --- /dev/null +++ b/packages/realm-server/tests/searchable-parity-diff-test.ts @@ -0,0 +1,129 @@ +import QUnit from 'qunit'; +const { module, test } = QUnit; + +import { + diffDoc, + isShallowLink, + shallowIds, +} from '../scripts/searchable-parity-diff.ts'; + +// Unit coverage for the realm-scale parity validator's pure comparison logic. +// The differ ignores `_cardType`, normalizes object key order, and (under +// --ignore-shallow-links) treats the store-driven omit-vs-keep-`{id}` difference +// as equivalent while still catching a CHANGED reference. +module('Unit | searchable-parity-diff', function () { + module('isShallowLink', function () { + test('null / a bare { id } / an array of bare { id } are shallow', function (assert) { + assert.true(isShallowLink(null), 'null'); + assert.true(isShallowLink({ id: 'x' }), 'bare { id }'); + assert.true(isShallowLink([{ id: 'x' }, { id: 'y' }]), 'array of { id }'); + assert.true(isShallowLink([]), 'empty array'); + }); + + test('an object with data beyond id is not shallow', function (assert) { + assert.false(isShallowLink({ id: 'x', name: 'n' }), 'has extra field'); + assert.false(isShallowLink([{ id: 'x', name: 'n' }]), 'array w/ data'); + assert.false(isShallowLink('scalar'), 'a scalar is not a link'); + }); + }); + + module('shallowIds', function () { + test('extracts ids, flattening plurals; null/empty contribute none', function (assert) { + assert.deepEqual(shallowIds({ id: 'a' }), ['a'], 'singular'); + assert.deepEqual( + shallowIds([{ id: 'a' }, { id: 'b' }]), + ['a', 'b'], + 'plural flattened', + ); + assert.deepEqual(shallowIds(null), [], 'null → none'); + assert.deepEqual(shallowIds([]), [], 'empty → none'); + assert.deepEqual(shallowIds('scalar'), [], 'scalar → none'); + }); + }); + + module('diffDoc', function () { + test('identical docs have no diffs and _cardType is ignored', function (assert) { + assert.deepEqual( + diffDoc( + { title: 'A', _cardType: 'X' }, + { title: 'A', _cardType: 'Y' }, + false, + ), + [], + 'same data, differing _cardType → equal', + ); + }); + + test('key order does not produce a diff', function (assert) { + assert.deepEqual( + diffDoc({ a: 1, b: 2 }, { b: 2, a: 1 }, false), + [], + 'reordered keys → equal', + ); + }); + + test('a changed scalar is reported', function (assert) { + let diffs = diffDoc({ title: 'A' }, { title: 'B' }, false); + assert.strictEqual(diffs.length, 1, 'one diff'); + assert.ok(diffs[0].includes('title'), 'names the field'); + }); + + test('a key present on only one side is reported as absent (not null)', function (assert) { + let diffs = diffDoc({ extra: 1 }, {}, false); + assert.strictEqual(diffs.length, 1, 'one diff'); + assert.ok( + diffs[0].includes('generated=absent'), + 'the missing side reads "absent", not "null"', + ); + }); + + test('present-null vs absent is a real diff without --ignore-shallow-links', function (assert) { + let diffs = diffDoc({ link: null }, {}, false); + assert.strictEqual(diffs.length, 1, 'null vs absent diverges'); + }); + + module('--ignore-shallow-links', function () { + test('a bare { id } vs absent is ignored (the intended omit-vs-keep difference)', function (assert) { + assert.deepEqual( + diffDoc({}, { link: { id: 'a' } }, true), + [], + 'generated keeps { id }, live omits → equal', + ); + assert.deepEqual( + diffDoc({ link: null }, { link: { id: 'a' } }, true), + [], + 'null vs { id } → equal', + ); + assert.deepEqual( + diffDoc({}, { links: [{ id: 'a' }] }, true), + [], + 'plural [{ id }] vs absent → equal', + ); + }); + + test('a CHANGED reference ({id:A} vs {id:B}) is still reported', function (assert) { + let diffs = diffDoc({ link: { id: 'A' } }, { link: { id: 'B' } }, true); + assert.strictEqual(diffs.length, 1, 'changed id is a real divergence'); + assert.ok(diffs[0].includes('link'), 'names the field'); + }); + + test('a changed reference inside a plural is still reported', function (assert) { + let diffs = diffDoc( + { links: [{ id: 'A' }] }, + { links: [{ id: 'B' }] }, + true, + ); + assert.strictEqual(diffs.length, 1, 'changed plural id diverges'); + }); + + test('an expanded vs shallow slot is reported (data difference, not shallow-vs-absent)', function (assert) { + let diffs = diffDoc( + { author: { id: 'a', name: 'Jo' } }, + { author: { id: 'a' } }, + true, + ); + assert.strictEqual(diffs.length, 1, 'expanded vs shallow diverges'); + }); + }); + }); +}); diff --git a/packages/runtime-common/definitions.ts b/packages/runtime-common/definitions.ts index 2ee1d212d6..fc252f1cc4 100644 --- a/packages/runtime-common/definitions.ts +++ b/packages/runtime-common/definitions.ts @@ -194,19 +194,30 @@ export async function validateSearchablePaths( for (let [fieldName, defId] of Object.entries(definition.fields)) { let fieldDef = definition.fieldDefs[defId]; let searchable = fieldDef?.searchable; + // Omitted = no annotation; `true` = the always-valid self link. Neither + // carries a path to resolve. A malformed value (not a string or array — + // e.g. a stray `false`) yields no paths via the fallback below, so it can't + // throw. Mirrors `seedSearchableRoutes`. if (searchable == null || searchable === true) { continue; } - let paths = typeof searchable === 'string' ? [searchable] : searchable; - if (paths.length === 0) { - continue; - } + let paths = + typeof searchable === 'string' + ? [searchable] + : Array.isArray(searchable) + ? searchable + : []; // A path is rooted at this field's target type. If we can't even identify // that target, none of the paths can be followed. let targetDef = isResolvedCodeRef(fieldDef.fieldOrCard) ? await lookupDefinition(fieldDef.fieldOrCard) : undefined; for (let path of paths) { + // A non-string entry is malformed and names nothing; an empty string is + // the self link (no deeper path to resolve). Both are valid, not issues. + if (typeof path !== 'string' || path === '') { + continue; + } let resolved = targetDef ? await getFieldDef(targetDef, path, lookupDefinition) : undefined; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0bd7f5a0af..d1406acc92 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2798,6 +2798,9 @@ importers: recast: specifier: 'catalog:' version: 0.23.11 + safe-stable-stringify: + specifier: 'catalog:' + version: 2.5.0 sane: specifier: 'catalog:' version: 5.0.1