diff --git a/src/__tests__/converters/pt9/analysisMerger.test.ts b/src/__tests__/converters/pt9/analysisMerger.test.ts
new file mode 100644
index 00000000..8ea99b55
--- /dev/null
+++ b/src/__tests__/converters/pt9/analysisMerger.test.ts
@@ -0,0 +1,438 @@
+///
+
+import type { LexemeKeyData } from 'parsers/pt9/lexemeKey';
+import { mergeLanguageAnalyses } from '../../../converters/pt9/analysisMerger';
+import type {
+ LangPhraseRecord,
+ LangTokenRecord,
+} from '../../../converters/pt9/languageAnalysisBuilder';
+import {
+ Pt9LexiconResolver,
+ unresolvedPt9LexiconResolver,
+} from '../../../converters/pt9/lexiconResolver';
+import { emptyPt9ImportReport } from '../../../converters/pt9/report';
+
+const STAMP = '2026-08-01T00:00:00.000Z';
+
+/** A resolver that answers from literal maps, keyed by composed key id and sense id. */
+function fakeResolver(
+ entries: Record,
+ senses: Record,
+): Pt9LexiconResolver {
+ return {
+ resolveEntry: (key) => {
+ const id = entries[`${key.Type}:${key.Form}`];
+ return id === undefined ? undefined : { entryId: id };
+ },
+ resolveSense: (key, senseId) => {
+ const id = senses[`${key.Type}:${key.Form}#${senseId}`];
+ return id === undefined ? undefined : { senseId: id };
+ },
+ };
+}
+
+const HELLO_KEY: LexemeKeyData = { Type: 'Word', Form: 'hello' };
+
+/** A word-facet contribution with overridable fields. */
+function wordRecord(overrides: Partial = {}): LangTokenRecord {
+ return {
+ tag: 'en',
+ tokenRef: 'GEN 1:1:0',
+ tokenSurface: 'hello',
+ tokenWritingSystem: 'en',
+ status: 'approved',
+ ambiguous: false,
+ word: { key: HELLO_KEY, keyId: 'Word:hello', senseId: 'S1', glossText: 'greeting' },
+ ...overrides,
+ };
+}
+
+/** A parse facet for hello = hell + o. */
+function helloParse(senseIds: (string | undefined)[] = [undefined, undefined]) {
+ return {
+ lexemes: [
+ { key: { Type: 'Stem', Form: 'hell' }, keyId: 'Stem:hell', senseId: senseIds[0] },
+ { key: { Type: 'Suffix', Form: 'o' }, keyId: 'Suffix:o', senseId: senseIds[1] },
+ ],
+ signature: 'Stem:hell/Suffix:o',
+ };
+}
+
+function merge(
+ records: LangTokenRecord[],
+ phrases: LangPhraseRecord[] = [],
+ resolver = unresolvedPt9LexiconResolver,
+) {
+ const report = emptyPt9ImportReport();
+ const result = mergeLanguageAnalyses({ records, phrases, resolver, importedAt: STAMP, report });
+ return { result, report };
+}
+
+describe('mergeLanguageAnalyses - token records', () => {
+ it('emits one payload and link per contribution with stamps, producer, and snapshot', () => {
+ const { result } = merge([wordRecord()]);
+
+ expect(result.tokenAnalyses).toStrictEqual([
+ {
+ id: 'pt9:ta:GEN 1:1:0:0',
+ createdAt: STAMP,
+ updatedAt: STAMP,
+ surfaceText: 'hello',
+ producer: 'pt9-import',
+ gloss: { en: 'greeting' },
+ },
+ ]);
+ expect(result.tokenAnalysisLinks).toStrictEqual([
+ {
+ analysisId: 'pt9:ta:GEN 1:1:0:0',
+ createdAt: STAMP,
+ updatedAt: STAMP,
+ status: 'approved',
+ token: { tokenRef: 'GEN 1:1:0', surfaceText: 'hello' },
+ },
+ ]);
+ });
+
+ it('merges same-word contributions across languages into MultiString glosses', () => {
+ const { result, report } = merge([
+ wordRecord(),
+ wordRecord({
+ tag: 'fr',
+ word: { key: HELLO_KEY, keyId: 'Word:hello', senseId: 'S1', glossText: 'salut' },
+ }),
+ ]);
+
+ expect(result.tokenAnalyses).toHaveLength(1);
+ expect(result.tokenAnalyses[0].gloss).toStrictEqual({ en: 'greeting', fr: 'salut' });
+ expect(report.merge.mergedTokenRecords).toBe(1);
+ });
+
+ it('keeps the first gloss per tag when the same tag contributes twice', () => {
+ const { result } = merge([
+ wordRecord(),
+ wordRecord({
+ word: { key: HELLO_KEY, keyId: 'Word:hello', senseId: 'S1', glossText: 'other' },
+ }),
+ ]);
+
+ expect(result.tokenAnalyses[0].gloss).toStrictEqual({ en: 'greeting' });
+ });
+
+ it('approves a merged record only when every contribution is approved', () => {
+ const { result } = merge([wordRecord(), wordRecord({ tag: 'fr', status: 'suggested' })]);
+ expect(result.tokenAnalysisLinks[0].status).toBe('suggested');
+ });
+
+ it('rejects a merged record only when every contribution is rejected', () => {
+ const rejected = merge([
+ wordRecord({ status: 'rejected' }),
+ wordRecord({ tag: 'fr', status: 'rejected' }),
+ ]);
+ expect(rejected.result.tokenAnalysisLinks[0].status).toBe('rejected');
+
+ const mixed = merge([
+ wordRecord({ status: 'rejected' }),
+ wordRecord({ tag: 'fr', status: 'approved' }),
+ ]);
+ expect(mixed.result.tokenAnalysisLinks[0].status).toBe('suggested');
+ });
+
+ it('merges two languages carrying the identical parse into one record', () => {
+ const { result, report } = merge([
+ wordRecord({ parse: helloParse() }),
+ wordRecord({ tag: 'fr', parse: helloParse() }),
+ ]);
+
+ expect(result.tokenAnalyses).toHaveLength(1);
+ expect(report.merge.parseConflicts).toBe(0);
+ });
+
+ it('keeps the first parse columns when the same tag contributes a parse-only record too', () => {
+ const resolver = fakeResolver({}, { 'Stem:hell#P1': 'would-resolve' });
+ const { result } = merge(
+ [
+ wordRecord({ parse: helloParse() }),
+ wordRecord({ word: undefined, parse: helloParse(['P1', undefined]) }),
+ ],
+ [],
+ resolver,
+ );
+
+ expect(result.tokenAnalyses).toHaveLength(1);
+ // The word record's en columns (no senses) win, so the standalone's P1 never resolves.
+ expect(result.tokenAnalyses[0].morphemes?.[0].senseRef).toBeUndefined();
+ });
+
+ it('adopts a parse from the language that has one', () => {
+ const { result } = merge([wordRecord(), wordRecord({ tag: 'fr', parse: helloParse() })]);
+
+ expect(result.tokenAnalyses).toHaveLength(1);
+ expect(result.tokenAnalyses[0].morphemes?.map((m) => m.form)).toStrictEqual(['hell', 'o']);
+ expect(result.tokenAnalyses[0].morphemes?.map((m) => m.id)).toStrictEqual(['m0', 'm1']);
+ });
+
+ it('splits genuinely conflicting parses into competing records and demotes the later approved one', () => {
+ const otherParse = {
+ lexemes: [
+ { key: { Type: 'Stem', Form: 'he' }, keyId: 'Stem:he', senseId: undefined },
+ { key: { Type: 'Suffix', Form: 'llo' }, keyId: 'Suffix:llo', senseId: undefined },
+ ],
+ signature: 'Stem:he/Suffix:llo',
+ };
+ const { result, report } = merge([
+ wordRecord({ parse: helloParse() }),
+ wordRecord({ tag: 'fr', parse: otherParse }),
+ wordRecord({ tag: 'de', parse: otherParse }),
+ ]);
+
+ expect(result.tokenAnalyses).toHaveLength(2);
+ expect(result.tokenAnalyses[1].morphemes?.map((m) => m.form)).toStrictEqual(['he', 'llo']);
+ expect(report.merge.parseConflicts).toBe(1);
+ expect(result.tokenAnalysisLinks.map((l) => l.status)).toStrictEqual(['approved', 'candidate']);
+ expect(report.merge.approvedDemotedToCandidate).toBe(1);
+ });
+
+ it('demotes a second would-be-approved record on one token to candidate', () => {
+ const { result, report } = merge([
+ wordRecord(),
+ wordRecord({
+ word: {
+ key: { Type: 'Word', Form: 'other' },
+ keyId: 'Word:other',
+ senseId: 'S9',
+ glossText: 'x',
+ },
+ }),
+ ]);
+
+ expect(result.tokenAnalysisLinks.map((l) => l.status)).toStrictEqual(['approved', 'candidate']);
+ expect(report.merge.approvedDemotedToCandidate).toBe(1);
+ });
+
+ it('fuses a parse-only contribution onto the word record sharing its parse', () => {
+ const { result } = merge([
+ wordRecord({ parse: helloParse() }),
+ wordRecord({ tag: 'fr', word: undefined, parse: helloParse(['P1', undefined]) }),
+ ]);
+
+ expect(result.tokenAnalyses).toHaveLength(1);
+ expect(result.tokenAnalyses[0].morphemes).toHaveLength(2);
+ });
+
+ it('fuses a parse-only contribution onto the sole word record lacking a parse', () => {
+ const { result } = merge([
+ wordRecord(),
+ wordRecord({ tag: 'fr', word: undefined, parse: helloParse() }),
+ ]);
+
+ expect(result.tokenAnalyses).toHaveLength(1);
+ expect(result.tokenAnalyses[0].gloss).toStrictEqual({ en: 'greeting' });
+ expect(result.tokenAnalyses[0].morphemes?.map((m) => m.form)).toStrictEqual(['hell', 'o']);
+ });
+
+ it('keeps a parse-only record standalone when the fuse target is ambiguous', () => {
+ const { result } = merge([
+ wordRecord(),
+ wordRecord({
+ word: { key: { Type: 'Word', Form: 'other' }, keyId: 'Word:other', senseId: undefined },
+ }),
+ wordRecord({ tag: 'fr', word: undefined, parse: helloParse() }),
+ ]);
+
+ expect(result.tokenAnalyses).toHaveLength(3);
+ const standalone = result.tokenAnalyses[2];
+ expect(standalone.gloss).toBeUndefined();
+ expect(standalone.morphemes).toHaveLength(2);
+ });
+
+ it('merges two parse-only contributions with the same signature', () => {
+ const { result } = merge([
+ wordRecord({ word: undefined, parse: helloParse() }),
+ wordRecord({ tag: 'fr', word: undefined, parse: helloParse() }),
+ ]);
+
+ expect(result.tokenAnalyses).toHaveLength(1);
+ });
+
+ it('sets glossSenseRef only for a unanimous sense the resolver can resolve', () => {
+ const resolver = fakeResolver({}, { 'Word:hello#S1': 'sense-guid' });
+ const unanimousMerge = merge([wordRecord(), wordRecord({ tag: 'fr' })], [], resolver);
+ expect(unanimousMerge.result.tokenAnalyses[0].glossSenseRef).toStrictEqual({
+ senseId: 'sense-guid',
+ });
+ expect(unanimousMerge.report.senses.senseRefsResolved).toBe(1);
+
+ const contested = merge(
+ [
+ wordRecord(),
+ wordRecord({
+ tag: 'fr',
+ word: { key: HELLO_KEY, keyId: 'Word:hello', senseId: 'S2', glossText: 'salut' },
+ }),
+ ],
+ [],
+ resolver,
+ );
+ expect(contested.result.tokenAnalyses[0].glossSenseRef).toBeUndefined();
+ expect(contested.report.senses.senseRefsResolved).toBe(0);
+ expect(contested.report.senses.senseRefsUnresolved).toBe(0);
+ });
+
+ it('counts an attempted but unresolved sense ref', () => {
+ const { result, report } = merge([wordRecord()]);
+ expect(result.tokenAnalyses[0].glossSenseRef).toBeUndefined();
+ expect(report.senses.senseRefsUnresolved).toBe(1);
+ });
+
+ it('resolves morpheme entry and sense refs through the resolver and counts outcomes', () => {
+ const resolver = fakeResolver(
+ { 'Stem:hell': 'entry-guid' },
+ { 'Stem:hell#P1': 'morph-sense-guid' },
+ );
+ const { result, report } = merge(
+ [wordRecord({ word: undefined, parse: helloParse(['P1', undefined]) })],
+ [],
+ resolver,
+ );
+
+ const { morphemes } = result.tokenAnalyses[0];
+ expect(morphemes?.[0].entryRef).toStrictEqual({ entryId: 'entry-guid' });
+ expect(morphemes?.[0].senseRef).toStrictEqual({ senseId: 'morph-sense-guid' });
+ expect(morphemes?.[1].entryRef).toBeUndefined();
+ expect(morphemes?.[1].senseRef).toBeUndefined();
+ expect(report.senses.entryRefsResolved).toBe(1);
+ expect(report.senses.entryRefsUnresolved).toBe(1);
+ });
+
+ it('builds morpheme glosses per language and marks ambiguous anchors low confidence', () => {
+ const withGlosses = {
+ lexemes: [
+ {
+ key: { Type: 'Stem', Form: 'hell' },
+ keyId: 'Stem:hell',
+ senseId: undefined,
+ glossText: 'inferno',
+ },
+ { key: { Type: 'Suffix', Form: 'o' }, keyId: 'Suffix:o', senseId: undefined },
+ ],
+ signature: 'Stem:hell/Suffix:o',
+ };
+ const { result } = merge([
+ wordRecord({ word: undefined, parse: withGlosses, ambiguous: true }),
+ ]);
+
+ expect(result.tokenAnalyses[0].morphemes?.[0].gloss).toStrictEqual({ en: 'inferno' });
+ expect(result.tokenAnalyses[0].morphemes?.[1].gloss).toBeUndefined();
+ expect(result.tokenAnalysisLinks[0].confidence).toBe('low');
+ });
+
+ it('collects converted parse identities for bare-payload dedupe', () => {
+ const { result } = merge([wordRecord({ tokenSurface: 'Hello', parse: helloParse() })]);
+ expect(result.clusterParseIdentities).toStrictEqual(new Set(['hello|Stem:hell/Suffix:o']));
+ });
+});
+
+describe('mergeLanguageAnalyses - phrases', () => {
+ const phraseRecord = (overrides: Partial = {}): LangPhraseRecord => ({
+ tag: 'en',
+ phrase: {
+ key: { Type: 'Phrase', Form: 'in the' },
+ keyId: 'Phrase:in the',
+ senseId: 'S5',
+ glossText: 'in',
+ },
+ tokens: [
+ { ref: 'GEN 1:1:0', surface: 'in' },
+ { ref: 'GEN 1:1:3', surface: 'the' },
+ ],
+ status: 'approved',
+ ambiguous: false,
+ ...overrides,
+ });
+
+ it('emits a phrase payload and link with joined surface and snapshots', () => {
+ const { result } = merge([], [phraseRecord()]);
+
+ expect(result.phraseAnalyses).toStrictEqual([
+ {
+ id: 'pt9:pa:GEN 1:1:0:0',
+ createdAt: STAMP,
+ updatedAt: STAMP,
+ surfaceText: 'in the',
+ producer: 'pt9-import',
+ gloss: { en: 'in' },
+ },
+ ]);
+ expect(result.phraseAnalysisLinks).toStrictEqual([
+ {
+ analysisId: 'pt9:pa:GEN 1:1:0:0',
+ createdAt: STAMP,
+ updatedAt: STAMP,
+ status: 'approved',
+ tokens: [
+ { tokenRef: 'GEN 1:1:0', surfaceText: 'in' },
+ { tokenRef: 'GEN 1:1:3', surfaceText: 'the' },
+ ],
+ },
+ ]);
+ });
+
+ it('merges same-phrase contributions across languages and resolves a unanimous sense', () => {
+ const resolver = fakeResolver({}, { 'Phrase:in the#S5': 'phrase-sense' });
+ const { result } = merge(
+ [],
+ [
+ phraseRecord(),
+ phraseRecord({
+ tag: 'fr',
+ phrase: {
+ key: { Type: 'Phrase', Form: 'in the' },
+ keyId: 'Phrase:in the',
+ senseId: 'S5',
+ glossText: 'dans',
+ },
+ }),
+ ],
+ resolver,
+ );
+
+ expect(result.phraseAnalyses).toHaveLength(1);
+ expect(result.phraseAnalyses[0].gloss).toStrictEqual({ en: 'in', fr: 'dans' });
+ expect(result.phraseAnalyses[0].senseRef).toStrictEqual({ senseId: 'phrase-sense' });
+ });
+
+ it('demotes a second approved phrase overlapping an approved one and flags ambiguity', () => {
+ const overlapping = phraseRecord({
+ phrase: {
+ key: { Type: 'Phrase', Form: 'the book' },
+ keyId: 'Phrase:the book',
+ senseId: undefined,
+ },
+ tokens: [
+ { ref: 'GEN 1:1:3', surface: 'the' },
+ { ref: 'GEN 1:1:7', surface: 'book' },
+ ],
+ ambiguous: true,
+ });
+ const { result, report } = merge([], [phraseRecord(), overlapping]);
+
+ expect(result.phraseAnalysisLinks.map((l) => l.status)).toStrictEqual([
+ 'approved',
+ 'candidate',
+ ]);
+ expect(result.phraseAnalysisLinks[1].confidence).toBe('low');
+ expect(report.merge.approvedDemotedToCandidate).toBe(1);
+ expect(result.phraseAnalyses[1].gloss).toBeUndefined();
+ });
+
+ it('keeps rejected and mixed phrase statuses by the agreement rule', () => {
+ const rejected = merge([], [phraseRecord({ status: 'rejected' })]);
+ expect(rejected.result.phraseAnalysisLinks[0].status).toBe('rejected');
+
+ const mixed = merge(
+ [],
+ [phraseRecord({ status: 'rejected' }), phraseRecord({ tag: 'fr', status: 'approved' })],
+ );
+ expect(mixed.result.phraseAnalysisLinks[0].status).toBe('suggested');
+ });
+});
diff --git a/src/__tests__/converters/pt9/bareWordAnalyses.test.ts b/src/__tests__/converters/pt9/bareWordAnalyses.test.ts
new file mode 100644
index 00000000..40326d3d
--- /dev/null
+++ b/src/__tests__/converters/pt9/bareWordAnalyses.test.ts
@@ -0,0 +1,216 @@
+///
+
+import type { LexiconData } from 'parsers/pt9/lexiconXmlParser';
+import type { WordAnalysesData } from 'parsers/pt9/wordAnalysesXmlParser';
+import { buildBareWordAnalyses } from '../../../converters/pt9/bareWordAnalyses';
+import {
+ Pt9LexiconResolver,
+ unresolvedPt9LexiconResolver,
+} from '../../../converters/pt9/lexiconResolver';
+import { createPt9GlossSource } from '../../../converters/pt9/pt9GlossSource';
+import { emptyPt9ImportReport } from '../../../converters/pt9/report';
+
+const STAMP = '2026-08-01T00:00:00.000Z';
+
+const LEXICON: LexiconData = {
+ Entries: [
+ {
+ Key: { Type: 'Stem', Form: 'hello' },
+ Senses: [{ Id: 'SG', Glosses: [{ Language: 'en', Text: 'greet' }] }],
+ },
+ ],
+ Analyses: {
+ legacy: [
+ { Type: 'Stem', Form: 'lega' },
+ { Type: 'Suffix', Form: 'cy' },
+ ],
+ },
+};
+
+function build(args: {
+ wordAnalyses?: WordAnalysesData;
+ lexicon?: LexiconData;
+ languages?: { raw: string; tag: string }[];
+ resolver?: Pt9LexiconResolver;
+ clusterParseIdentities?: Set;
+}) {
+ const report = emptyPt9ImportReport();
+ const payloads = buildBareWordAnalyses({
+ wordAnalyses: args.wordAnalyses,
+ lexicon: args.lexicon,
+ languages: args.languages ?? [{ raw: 'en', tag: 'en' }],
+ glossSource: createPt9GlossSource(args.lexicon),
+ resolver: args.resolver ?? unresolvedPt9LexiconResolver,
+ clusterParseIdentities: args.clusterParseIdentities ?? new Set(),
+ writingSystem: 'grc',
+ importedAt: STAMP,
+ report,
+ });
+ return { payloads, report };
+}
+
+describe('buildBareWordAnalyses', () => {
+ it('builds an unlinked payload per wordform analysis with DefaultSingle morpheme glosses', () => {
+ const { payloads, report } = build({
+ wordAnalyses: {
+ Entries: [{ Word: 'helloing', Analyses: [{ LexemeIds: ['Stem:hello', 'Suffix:ing'] }] }],
+ },
+ lexicon: { ...LEXICON, Analyses: {} },
+ });
+
+ expect(payloads).toStrictEqual([
+ {
+ id: 'pt9:wa:helloing:0',
+ createdAt: STAMP,
+ updatedAt: STAMP,
+ surfaceText: 'helloing',
+ producer: 'pt9-import:word-analyses',
+ morphemes: [
+ { id: 'm0', form: 'hello', writingSystem: 'grc', gloss: { en: 'greet' } },
+ { id: 'm1', form: 'ing', writingSystem: 'grc' },
+ ],
+ },
+ ]);
+ expect(report.barePayloads.added).toBe(1);
+ expect(report.senses.entryRefsUnresolved).toBe(2);
+ });
+
+ it('includes the legacy lexicon analyses after the newer inventory', () => {
+ const { payloads } = build({
+ wordAnalyses: {
+ Entries: [{ Word: 'helloing', Analyses: [{ LexemeIds: ['Stem:hello', 'Suffix:ing'] }] }],
+ },
+ lexicon: LEXICON,
+ });
+
+ expect(payloads.map((p) => p.surfaceText)).toStrictEqual(['helloing', 'legacy']);
+ });
+
+ it('dedupes an identical analysis appearing in both inventories, homograph 1 normalized', () => {
+ const { payloads } = build({
+ wordAnalyses: {
+ Entries: [{ Word: 'legacy', Analyses: [{ LexemeIds: ['Stem:lega', 'Suffix:cy'] }] }],
+ },
+ lexicon: {
+ Entries: [],
+ Analyses: {
+ legacy: [
+ { Type: 'Stem', Form: 'lega', Homograph: 1 },
+ { Type: 'Suffix', Form: 'cy', Homograph: 1 },
+ ],
+ },
+ },
+ });
+
+ expect(payloads).toHaveLength(1);
+ });
+
+ it('numbers several analyses of one wordform sequentially', () => {
+ const { payloads } = build({
+ wordAnalyses: {
+ Entries: [
+ {
+ Word: 'abe',
+ Analyses: [{ LexemeIds: ['Stem:ab', 'Suffix:e'] }, { LexemeIds: ['Stem:abe'] }],
+ },
+ ],
+ },
+ });
+
+ expect(payloads.map((p) => p.id)).toStrictEqual(['pt9:wa:abe:0', 'pt9:wa:abe:1']);
+ });
+
+ it('skips an analysis identical to a converted cluster parse', () => {
+ const { payloads, report } = build({
+ wordAnalyses: {
+ Entries: [{ Word: 'Abe', Analyses: [{ LexemeIds: ['Stem:ab', 'Suffix:e'] }] }],
+ },
+ clusterParseIdentities: new Set(['abe|Stem:ab/Suffix:e']),
+ });
+
+ expect(payloads).toHaveLength(0);
+ expect(report.barePayloads.skippedExistingIdentical).toBe(1);
+ });
+
+ it('drops an analysis with an unparseable lexeme id', () => {
+ const { payloads, report } = build({
+ wordAnalyses: { Entries: [{ Word: 'x', Analyses: [{ LexemeIds: ['garbage'] }] }] },
+ });
+
+ expect(payloads).toHaveLength(0);
+ expect(report.barePayloads.droppedUnparseable).toBe(1);
+ });
+
+ it('drops an analysis with no lexemes at all', () => {
+ const { payloads, report } = build({
+ wordAnalyses: { Entries: [{ Word: 'x', Analyses: [{ LexemeIds: [] }] }] },
+ });
+
+ expect(payloads).toHaveLength(0);
+ expect(report.barePayloads.droppedEmpty).toBe(1);
+ });
+
+ it('resolves a sense ref when every language finding a default agrees on it', () => {
+ const resolver: Pt9LexiconResolver = {
+ resolveEntry: (key) => ({ entryId: `entry-${key.Form}` }),
+ resolveSense: (_key, senseId) => ({ senseId: `guid-${senseId}` }),
+ };
+ const { payloads, report } = build({
+ wordAnalyses: { Entries: [{ Word: 'helloing', Analyses: [{ LexemeIds: ['Stem:hello'] }] }] },
+ lexicon: LEXICON,
+ // French abstains (no default in fr); English finds SG, so the lone vote carries.
+ languages: [
+ { raw: 'en', tag: 'en' },
+ { raw: 'fr', tag: 'fr' },
+ ],
+ resolver,
+ });
+
+ expect(payloads[0].morphemes?.[0].senseRef).toStrictEqual({ senseId: 'guid-SG' });
+ expect(report.senses.senseRefsResolved).toBe(1);
+ });
+
+ it('keeps the first gloss when two raw languages share one tag', () => {
+ const { payloads } = build({
+ wordAnalyses: { Entries: [{ Word: 'helloing', Analyses: [{ LexemeIds: ['Stem:hello'] }] }] },
+ lexicon: {
+ Entries: [
+ {
+ Key: { Type: 'Stem', Form: 'hello' },
+ Senses: [
+ {
+ Id: 'SG',
+ Glosses: [
+ { Language: 'English', Text: 'first' },
+ { Language: 'ENGLISH2', Text: 'second' },
+ ],
+ },
+ ],
+ },
+ ],
+ Analyses: {},
+ },
+ languages: [
+ { raw: 'English', tag: 'en' },
+ { raw: 'ENGLISH2', tag: 'en' },
+ ],
+ });
+
+ expect(payloads[0].morphemes?.[0].gloss).toStrictEqual({ en: 'first' });
+ });
+
+ it('counts an attempted but unresolved sense ref', () => {
+ const { report } = build({
+ wordAnalyses: { Entries: [{ Word: 'helloing', Analyses: [{ LexemeIds: ['Stem:hello'] }] }] },
+ lexicon: LEXICON,
+ });
+
+ expect(report.senses.senseRefsUnresolved).toBe(1);
+ });
+
+ it('builds nothing from absent inventories', () => {
+ const { payloads, report } = build({});
+ expect(payloads).toStrictEqual([]);
+ expect(report.barePayloads.added).toBe(0);
+ });
+});
diff --git a/src/__tests__/converters/pt9/clusterAnchoring.test.ts b/src/__tests__/converters/pt9/clusterAnchoring.test.ts
new file mode 100644
index 00000000..26eeec27
--- /dev/null
+++ b/src/__tests__/converters/pt9/clusterAnchoring.test.ts
@@ -0,0 +1,266 @@
+///
+
+import type { ClusterData } from 'parsers/pt9/interlinearXmlParser';
+import { anchorVerseClusters, classifyCluster } from '../../../converters/pt9/clusterAnchoring';
+import { makeSegment, makeWordToken, makePunctToken } from '../../test-helpers';
+
+/** Builds a ClusterData literal the way the parser composes one. */
+function mkCluster(
+ index: number,
+ length: number,
+ lexemes: [id: string, senseId?: string][],
+ excluded = false,
+): ClusterData {
+ const lexemesId = lexemes.map(([id]) => id).join('/');
+ return {
+ TextRange: { Index: index, Length: length },
+ Lexemes: lexemes.map(([LexemeId, senseId]) => ({
+ LexemeId,
+ ...(senseId !== undefined && { SenseId: senseId }),
+ })),
+ LexemesId: lexemesId,
+ Id: lexemesId ? `${lexemesId}/${index}-${length}` : `${index}-${length}`,
+ Excluded: excluded,
+ };
+}
+
+/** A segment whose word tokens carry real offsets within the given text. */
+function segmentOf(text: string): ReturnType {
+ const tokens = Array.from(text.matchAll(/[^\s.,]+|[.,]/gu), (match) =>
+ /[.,]/.test(match[0])
+ ? makePunctToken(`GEN 1:1:${match.index}`, match[0], match.index)
+ : makeWordToken(`GEN 1:1:${match.index}`, match[0], match.index),
+ );
+ return makeSegment('GEN 1:1', text, tokens);
+}
+
+describe('classifyCluster', () => {
+ it('classifies a single Word lexeme as a word cluster', () => {
+ const classified = classifyCluster(mkCluster(0, 5, [['Word:hello', 'S1']]));
+ expect(classified.kind).toBe('word');
+ });
+
+ it('classifies any stem/suffix/prefix presence as a word parse, even mixed with Word', () => {
+ expect(classifyCluster(mkCluster(0, 5, [['Stem:hell'], ['Suffix:o']])).kind).toBe('wordParse');
+ expect(classifyCluster(mkCluster(0, 5, [['Word:hello'], ['Prefix:o']])).kind).toBe('wordParse');
+ });
+
+ it('classifies a single Phrase lexeme as a phrase cluster', () => {
+ expect(classifyCluster(mkCluster(0, 8, [['Phrase:in the']])).kind).toBe('phrase');
+ });
+
+ it.each([
+ ['a Lemma cluster', mkCluster(0, 4, [['Lemma:go']])],
+ ['an empty cluster', mkCluster(0, 4, [])],
+ ['a multi-Word cluster', mkCluster(0, 4, [['Word:a'], ['Word:b']])],
+ ])('drops %s as lemmaOrOther', (_label, cluster) => {
+ expect(classifyCluster(cluster)).toStrictEqual({
+ kind: 'drop',
+ cluster,
+ reason: 'lemmaOrOther',
+ });
+ });
+
+ it('drops a cluster containing an unparseable lexeme id', () => {
+ const cluster = mkCluster(0, 4, [['Stem:ok'], ['garbage']]);
+ expect(classifyCluster(cluster)).toStrictEqual({
+ kind: 'drop',
+ cluster,
+ reason: 'unparseableLexemeId',
+ });
+ });
+});
+
+describe('anchorVerseClusters', () => {
+ it('anchors a word cluster to the token folding to its form', () => {
+ const segment = segmentOf('Hello world');
+ const result = anchorVerseClusters(segment, [mkCluster(5, 5, [['Word:hello', 'S1']])]);
+
+ expect(result.groups).toHaveLength(1);
+ expect(result.groups[0].token.surfaceText).toBe('Hello');
+ expect(result.groups[0].word?.lexeme).toStrictEqual({
+ key: { Type: 'Word', Form: 'hello' },
+ senseId: 'S1',
+ });
+ expect(result.groups[0].parse).toBeUndefined();
+ expect(result.groups[0].ambiguous).toBe(false);
+ expect(result.ambiguousCount).toBe(0);
+ });
+
+ it('pairs a word and a parse cluster at the identical range onto one token', () => {
+ const segment = segmentOf('hello world');
+ const result = anchorVerseClusters(segment, [
+ mkCluster(0, 5, [['Word:hello', 'S1']]),
+ mkCluster(0, 5, [['Stem:hello'], ['Suffix:ing']]),
+ ]);
+
+ expect(result.groups).toHaveLength(1);
+ expect(result.groups[0].word).toBeDefined();
+ expect(result.groups[0].parse?.lexemes.map((l) => l.key.Form)).toStrictEqual(['hello', 'ing']);
+ });
+
+ it('anchors a parse-only cluster by its concatenated forms', () => {
+ const segment = segmentOf('exaucera demain');
+ const result = anchorVerseClusters(segment, [
+ mkCluster(0, 8, [['Stem:exauc'], ['Suffix:era']]),
+ ]);
+
+ expect(result.groups).toHaveLength(1);
+ expect(result.groups[0].token.surfaceText).toBe('exaucera');
+ expect(result.groups[0].word).toBeUndefined();
+ });
+
+ it('keeps only the first of two same-kind clusters at one range', () => {
+ const segment = segmentOf('hello');
+ const result = anchorVerseClusters(segment, [
+ mkCluster(0, 5, [['Word:hello', 'S1']]),
+ mkCluster(0, 5, [['Word:hello', 'S2']]),
+ mkCluster(0, 5, [['Stem:hell'], ['Suffix:o']]),
+ mkCluster(0, 5, [['Stem:he'], ['Suffix:llo']]),
+ ]);
+
+ expect(result.dropCounts.duplicateCluster).toBe(2);
+ expect(result.groups).toHaveLength(1);
+ expect(result.groups[0].word?.lexeme.senseId).toBe('S1');
+ expect(result.groups[0].parse?.lexemes.map((l) => l.key.Form)).toStrictEqual(['hell', 'o']);
+ });
+
+ it('drops both facets of an unmatched pair as form mismatches', () => {
+ const segment = segmentOf('nothing here');
+ const result = anchorVerseClusters(segment, [
+ mkCluster(0, 3, [['Word:zzz']]),
+ mkCluster(0, 3, [['Stem:zz'], ['Suffix:z']]),
+ ]);
+
+ expect(result.dropCounts.formMismatch).toBe(2);
+ expect(result.groups).toHaveLength(0);
+ });
+
+ it('assigns clusters to same-fold tokens monotonically by range order', () => {
+ const segment = segmentOf('a b a');
+ const result = anchorVerseClusters(segment, [
+ mkCluster(0, 1, [['Word:a', 'S1']]),
+ mkCluster(4, 1, [['Word:a', 'S2']]),
+ ]);
+
+ expect(result.groups.map((g) => g.token.charStart)).toStrictEqual([0, 4]);
+ // The first pick sees both same-fold tokens ahead of the cursor, so it counts as ambiguous
+ // even though the prior lands it correctly; the second has one candidate left.
+ expect(result.groups.map((g) => g.ambiguous)).toStrictEqual([true, false]);
+ expect(result.ambiguousCount).toBe(1);
+ });
+
+ it('orders same-index range groups by length', () => {
+ const segment = segmentOf('ab abcd');
+ const result = anchorVerseClusters(segment, [
+ mkCluster(0, 4, [['Word:abcd']]),
+ mkCluster(0, 2, [['Word:ab']]),
+ ]);
+
+ expect(result.groups.map((g) => g.token.surfaceText)).toStrictEqual(['ab', 'abcd']);
+ });
+
+ it('drops a cluster whose only match lies behind the cursor', () => {
+ const segment = segmentOf('a b');
+ const result = anchorVerseClusters(segment, [
+ mkCluster(0, 1, [['Word:b']]),
+ mkCluster(4, 1, [['Word:a']]),
+ ]);
+
+ expect(result.groups).toHaveLength(1);
+ expect(result.groups[0].token.surfaceText).toBe('b');
+ expect(result.dropCounts.formMismatch).toBe(1);
+ });
+
+ it('disambiguates repeated surface forms by the proportional-position prior', () => {
+ const segment = segmentOf('a b a');
+ const result = anchorVerseClusters(segment, [mkCluster(80, 1, [['Word:a', 'S1']])]);
+
+ expect(result.groups).toHaveLength(1);
+ expect(result.groups[0].token.charStart).toBe(4);
+ expect(result.groups[0].ambiguous).toBe(true);
+ expect(result.ambiguousCount).toBe(1);
+ });
+
+ it('counts an unparseable and a lemma cluster under their own drop reasons', () => {
+ const segment = segmentOf('went');
+ const result = anchorVerseClusters(segment, [
+ mkCluster(0, 4, [['Lemma:go']]),
+ mkCluster(0, 4, [['nonsense']]),
+ ]);
+
+ expect(result.dropCounts.lemmaOrOther).toBe(1);
+ expect(result.dropCounts.unparseableLexemeId).toBe(1);
+ });
+
+ it('anchors a phrase to the consecutive word tokens folding to its words', () => {
+ const segment = segmentOf('Look in the book');
+ const result = anchorVerseClusters(segment, [mkCluster(5, 6, [['Phrase:in the', 'S1']])]);
+
+ expect(result.phrases).toHaveLength(1);
+ expect(result.phrases[0].tokens.map((t) => t.surfaceText)).toStrictEqual(['in', 'the']);
+ expect(result.phrases[0].lexeme.senseId).toBe('S1');
+ expect(result.phrases[0].ambiguous).toBe(false);
+ });
+
+ it('matches phrase words across intervening punctuation tokens', () => {
+ const segment = segmentOf('in, the book');
+ const result = anchorVerseClusters(segment, [mkCluster(0, 6, [['Phrase:in the']])]);
+
+ expect(result.phrases).toHaveLength(1);
+ expect(result.phrases[0].tokens.map((t) => t.surfaceText)).toStrictEqual(['in', 'the']);
+ });
+
+ it('drops an unmatched phrase as a form mismatch', () => {
+ const segment = segmentOf('in the book');
+ const result = anchorVerseClusters(segment, [mkCluster(0, 6, [['Phrase:on the']])]);
+
+ expect(result.phrases).toHaveLength(0);
+ expect(result.dropCounts.formMismatch).toBe(1);
+ });
+
+ it('drops a phrase whose form folds to no words', () => {
+ const segment = segmentOf('in the book');
+ const result = anchorVerseClusters(segment, [mkCluster(0, 1, [['Phrase: ']])]);
+
+ expect(result.dropCounts.formMismatch).toBe(1);
+ });
+
+ it('disambiguates a repeated phrase window by the proportional-position prior', () => {
+ const segment = segmentOf('in the x in the');
+ const result = anchorVerseClusters(segment, [mkCluster(90, 6, [['Phrase:in the']])]);
+
+ expect(result.phrases).toHaveLength(1);
+ expect(result.phrases[0].tokens[0].charStart).toBe(9);
+ expect(result.phrases[0].ambiguous).toBe(true);
+ });
+
+ it('carries the Excluded flag on word, parse, and phrase anchors', () => {
+ const segment = segmentOf('hello in the');
+ const result = anchorVerseClusters(segment, [
+ mkCluster(0, 5, [['Word:hello']], true),
+ mkCluster(0, 5, [['Stem:hell'], ['Suffix:o']], true),
+ mkCluster(6, 6, [['Phrase:in the']], true),
+ ]);
+
+ expect(result.groups[0].word?.excluded).toBe(true);
+ expect(result.groups[0].parse?.excluded).toBe(true);
+ expect(result.phrases[0].excluded).toBe(true);
+ });
+
+ it('returns empty results for a verse with no clusters', () => {
+ const segment = segmentOf('hello');
+ expect(anchorVerseClusters(segment, [])).toStrictEqual({
+ groups: [],
+ phrases: [],
+ dropCounts: {
+ verseNotFound: 0,
+ formMismatch: 0,
+ lemmaOrOther: 0,
+ duplicateCluster: 0,
+ unparseableLexemeId: 0,
+ },
+ ambiguousCount: 0,
+ });
+ });
+});
diff --git a/src/__tests__/converters/pt9/convertPt9Project.test.ts b/src/__tests__/converters/pt9/convertPt9Project.test.ts
new file mode 100644
index 00000000..f14ea284
--- /dev/null
+++ b/src/__tests__/converters/pt9/convertPt9Project.test.ts
@@ -0,0 +1,280 @@
+///
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import type { InterlinearData } from 'parsers/pt9/interlinearXmlParser';
+import { InterlinearXmlParser } from 'parsers/pt9/interlinearXmlParser';
+import { InterlinearSetupXmlParser } from 'parsers/pt9/interlinearSetupXmlParser';
+import { LexiconXmlParser } from 'parsers/pt9/lexiconXmlParser';
+import { WordAnalysesXmlParser } from 'parsers/pt9/wordAnalysesXmlParser';
+import { convertPt9Project } from '../../../converters/pt9';
+import type { Pt9LexiconResolver } from '../../../converters/pt9';
+import { makeVerseBook } from '../../test-helpers';
+
+const STAMP = '2026-08-01T00:00:00.000Z';
+
+/** A one-verse interlinear file with a single word cluster. */
+function fileWith(
+ language: string,
+ bookId: string,
+ form: string,
+ senseId?: string,
+ hash?: string,
+): InterlinearData {
+ const id = `Word:${form}`;
+ return {
+ GlossLanguage: language,
+ BookId: bookId,
+ Verses: {
+ [`${bookId} 1:1`]: {
+ ...(hash !== undefined && { Hash: hash }),
+ Clusters: [
+ {
+ TextRange: { Index: 0, Length: form.length },
+ Lexemes: [{ LexemeId: id, ...(senseId !== undefined && { SenseId: senseId }) }],
+ LexemesId: id,
+ Id: `${id}/0-${form.length}`,
+ Excluded: false,
+ },
+ ],
+ Punctuations: [],
+ },
+ },
+ };
+}
+
+describe('convertPt9Project', () => {
+ it('throws on two interlinear files for the same language and book', () => {
+ expect(() =>
+ convertPt9Project({
+ data: { interlinear: [fileWith('en', 'GEN', 'a'), fileWith('en', 'GEN', 'b')] },
+ books: [],
+ importedAt: STAMP,
+ }),
+ ).toThrow('Duplicate interlinear data for language "en" book "GEN"');
+ });
+
+ it('merges languages across files and reports a same-tag collision', () => {
+ const books = [makeVerseBook([{ sid: 'GEN 1:1', text: 'hello' }])];
+ const result = convertPt9Project({
+ data: {
+ interlinear: [
+ fileWith('EN', 'GEN', 'hello', 'S1', 'AA'),
+ fileWith('en', 'GEN', 'hello', 'S1', 'BB'),
+ ],
+ },
+ books,
+ importedAt: STAMP,
+ });
+
+ expect(result.analysisLanguages).toStrictEqual(['en']);
+ expect(result.report.merge.sameTagCollisions).toStrictEqual([['EN', 'en']]);
+ expect(result.report.languages.map((l) => l.rawLanguage)).toStrictEqual(['EN', 'en']);
+ expect(result.analysis.tokenAnalyses).toHaveLength(1);
+ expect(result.analysis.tokenAnalysisLinks[0].status).toBe('approved');
+ });
+
+ it('keeps distinct tags in discovery order and merges records across them', () => {
+ const books = [makeVerseBook([{ sid: 'GEN 1:1', text: 'hello' }])];
+ const result = convertPt9Project({
+ data: {
+ interlinear: [
+ fileWith('fr', 'GEN', 'hello', 'S1', 'AA'),
+ fileWith('en', 'GEN', 'hello', 'S1'),
+ ],
+ },
+ books,
+ importedAt: STAMP,
+ });
+
+ expect(result.analysisLanguages).toStrictEqual(['fr', 'en']);
+ expect(result.report.merge.sameTagCollisions).toStrictEqual([]);
+ expect(result.analysis.tokenAnalyses).toHaveLength(1);
+ // fr's verse is hashed, en's is not: strict all-hashed merging yields suggested.
+ expect(result.analysis.tokenAnalysisLinks[0].status).toBe('suggested');
+ expect(result.report.merge.mergedTokenRecords).toBe(1);
+ });
+
+ it('reports a missing book without records', () => {
+ const result = convertPt9Project({
+ data: { interlinear: [fileWith('en', 'EXO', 'hello', 'S1')] },
+ books: [makeVerseBook([{ sid: 'GEN 1:1', text: 'hello' }])],
+ importedAt: STAMP,
+ });
+
+ expect(result.report.languages[0].books[0].bookFound).toBe(false);
+ expect(result.analysis.tokenAnalyses).toHaveLength(0);
+ });
+
+ it('names the project from ScrTextName when a file carries one', () => {
+ const named: InterlinearData = { ...fileWith('en', 'GEN', 'hello'), ScrTextName: 'MyProj' };
+ const result = convertPt9Project({
+ data: { interlinear: [named] },
+ books: [],
+ importedAt: STAMP,
+ });
+
+ expect(result.suggestedName).toBe('MyProj interlinear (en)');
+ });
+
+ it('describes setups per language, defaulting an absent type and skipping unmatched setups', () => {
+ const result = convertPt9Project({
+ data: {
+ interlinear: [fileWith('en', 'GEN', 'hello'), fileWith('fr', 'GEN', 'salut')],
+ setups: {
+ Setups: [
+ { LanguageId: 'en', MdlScrTextName: 'MDL', ExportScrTextName: 'BT1' },
+ { LanguageId: 'de', Type: 'Glossing' },
+ ],
+ },
+ },
+ books: [],
+ importedAt: STAMP,
+ });
+
+ expect(result.suggestedDescription).toBe(
+ 'Imported from Paratext 9 interlinear data. Setups: en: unknown type (model MDL) -> BT1.',
+ );
+ });
+
+ it('describes nothing beyond the base sentence without setups', () => {
+ const result = convertPt9Project({
+ data: { interlinear: [fileWith('en', 'GEN', 'hello')] },
+ books: [],
+ importedAt: STAMP,
+ });
+
+ expect(result.suggestedDescription).toBe('Imported from Paratext 9 interlinear data.');
+ });
+
+ it('resolves refs through a provided resolver', () => {
+ const resolver: Pt9LexiconResolver = {
+ resolveEntry: () => undefined,
+ resolveSense: (key, senseId) => ({ senseId: `${key.Form}#${senseId}` }),
+ };
+ const result = convertPt9Project({
+ data: { interlinear: [fileWith('en', 'GEN', 'hello', 'S1', 'AA')] },
+ books: [makeVerseBook([{ sid: 'GEN 1:1', text: 'hello' }])],
+ resolver,
+ importedAt: STAMP,
+ });
+
+ expect(result.analysis.tokenAnalyses[0].glossSenseRef).toStrictEqual({ senseId: 'hello#S1' });
+ expect(result.report.senses.senseRefsResolved).toBe(1);
+ });
+
+ it('falls back to the und writing system for bare payloads when no book has a word token', () => {
+ const result = convertPt9Project({
+ data: {
+ interlinear: [fileWith('en', 'GEN', 'hello')],
+ wordAnalyses: { Entries: [{ Word: 'x', Analyses: [{ LexemeIds: ['Stem:x'] }] }] },
+ },
+ books: [],
+ importedAt: STAMP,
+ });
+
+ expect(result.analysis.tokenAnalyses[0].morphemes?.[0].writingSystem).toBe('und');
+ });
+
+ describe('against the coherent test-data fixture set', () => {
+ const readFixture = (name: string): string =>
+ fs.readFileSync(path.join(__dirname, '..', '..', '..', '..', 'test-data', name), 'utf-8');
+
+ const data = {
+ interlinear: [new InterlinearXmlParser().parse(readFixture('Interlinear_en_MAT.xml'))],
+ lexicon: new LexiconXmlParser().parse(readFixture('Lexicon.xml')),
+ wordAnalyses: new WordAnalysesXmlParser().parse(readFixture('WordAnalyses.xml')),
+ setups: new InterlinearSetupXmlParser().parse(readFixture('InterlinearSetup.xml')),
+ };
+ const books = [
+ makeVerseBook([
+ { sid: 'MAT 1:1', text: 'hello aokaybe abe abc this is a footnote with a note تمان oj' },
+ { sid: 'MAT 1:2', text: 'oooo dearly' },
+ { sid: 'MAT 1:9', text: 'hello' },
+ ]),
+ ];
+ const result = convertPt9Project({ data, books, importedAt: STAMP });
+
+ it('identifies the language, name, and setup provenance', () => {
+ expect(result.analysisLanguages).toStrictEqual(['en']);
+ expect(result.suggestedName).toBe('Paratext 9 interlinear (en)');
+ expect(result.suggestedDescription).toBe(
+ 'Imported from Paratext 9 interlinear data. Setups: en: Glossing.',
+ );
+ });
+
+ it('converts the covered verses and drops the rest as verse-not-found', () => {
+ const [language] = result.report.languages;
+ expect(language.tag).toBe('en');
+ expect(language.tagIsFallback).toBe(false);
+ const [book] = language.books;
+ expect(book.bookId).toBe('MAT');
+ expect(book.bookFound).toBe(true);
+ expect(book.versesTotal).toBe(36);
+ expect(book.versesHashed).toBe(6);
+ expect(book.versesNotFound).toBe(33);
+ expect(book.clustersTotal).toBe(61);
+ expect(book.clustersConverted).toBe(24);
+ expect(book.clusterDrops).toStrictEqual({
+ verseNotFound: 37,
+ formMismatch: 0,
+ lemmaOrOther: 0,
+ duplicateCluster: 0,
+ unparseableLexemeId: 0,
+ });
+ expect(book.ambiguousAnchors).toBe(1);
+ expect(book.punctuationEntriesIgnored).toBe(1);
+ expect(book.phrasesConverted).toBe(0);
+ });
+
+ it('merges the hello word and parse pair with lexicon glosses, approved', () => {
+ const link = result.analysis.tokenAnalysisLinks.find((l) => l.token.tokenRef === 'MAT 1:1:0');
+ expect(link?.status).toBe('approved');
+ const analysis = result.analysis.tokenAnalyses.find((a) => a.id === link?.analysisId);
+ expect(analysis?.surfaceText).toBe('hello');
+ expect(analysis?.gloss).toStrictEqual({ en: 'greeting' });
+ expect(analysis?.morphemes).toStrictEqual([
+ { id: 'm0', form: 'hello', writingSystem: 'en', gloss: { en: 'greet' } },
+ { id: 'm1', form: 'ing', writingSystem: 'en', gloss: { en: 'PROG' } },
+ ]);
+ });
+
+ it('imports a dangling sense selection without gloss text', () => {
+ const link = result.analysis.tokenAnalysisLinks.find((l) => l.token.tokenRef === 'MAT 1:9:0');
+ expect(link?.status).toBe('approved');
+ const analysis = result.analysis.tokenAnalyses.find((a) => a.id === link?.analysisId);
+ expect(analysis?.gloss).toBeUndefined();
+ });
+
+ it('marks the proportionally disambiguated token low confidence', () => {
+ const ambiguousLinks = result.analysis.tokenAnalysisLinks.filter(
+ (l) => l.confidence === 'low',
+ );
+ expect(ambiguousLinks).toHaveLength(1);
+ expect(ambiguousLinks[0].token.surfaceText).toBe('a');
+ });
+
+ it('adds bare payloads for unconverted analyses and skips the cluster-identical one', () => {
+ const bare = result.analysis.tokenAnalyses.filter(
+ (a) => a.producer === 'pt9-import:word-analyses',
+ );
+ expect(bare.map((a) => a.surfaceText).sort()).toStrictEqual(['aaaa', 'abe', 'helloing']);
+ expect(result.report.barePayloads.added).toBe(3);
+ expect(result.report.barePayloads.skippedExistingIdentical).toBe(1);
+
+ const helloing = bare.find((a) => a.surfaceText === 'helloing');
+ expect(helloing?.morphemes?.map((m) => m.gloss)).toStrictEqual([
+ { en: 'greet' },
+ { en: 'PROG' },
+ ]);
+ });
+
+ it('emits fifteen linked token records and no phrases or segment analyses', () => {
+ expect(result.analysis.tokenAnalysisLinks).toHaveLength(15);
+ expect(result.analysis.phraseAnalyses).toStrictEqual([]);
+ expect(result.analysis.segmentAnalyses).toStrictEqual([]);
+ expect(result.analysis.tokenAnalyses).toHaveLength(18);
+ });
+ });
+});
diff --git a/src/__tests__/converters/pt9/glossLanguageTags.test.ts b/src/__tests__/converters/pt9/glossLanguageTags.test.ts
new file mode 100644
index 00000000..08e3b522
--- /dev/null
+++ b/src/__tests__/converters/pt9/glossLanguageTags.test.ts
@@ -0,0 +1,25 @@
+///
+
+import { resolveGlossLanguageTag } from '../../../converters/pt9/glossLanguageTags';
+
+describe('resolveGlossLanguageTag', () => {
+ it.each([
+ ['en', 'en'],
+ ['EN', 'en'],
+ ['grc', 'grc'],
+ ['kmr-latn', 'kmr-Latn'],
+ ['KMR-LATN', 'kmr-Latn'],
+ ['en-us', 'en-US'],
+ ['zh-hans-cn', 'zh-Hans-CN'],
+ ['en-x-priv8', 'en-x-priv8'],
+ ])('normalizes valid tag "%s" to "%s"', (raw, tag) => {
+ expect(resolveGlossLanguageTag(raw)).toStrictEqual({ tag, isFallback: false });
+ });
+
+ it.each(['English', 'UpperEnglish', 'e', '', 'en-', 'en--US', '123', 'en-toolongsubtag1'])(
+ 'passes invalid value "%s" through verbatim as a fallback',
+ (raw) => {
+ expect(resolveGlossLanguageTag(raw)).toStrictEqual({ tag: raw, isFallback: true });
+ },
+ );
+});
diff --git a/src/__tests__/converters/pt9/languageAnalysisBuilder.test.ts b/src/__tests__/converters/pt9/languageAnalysisBuilder.test.ts
new file mode 100644
index 00000000..2047cc75
--- /dev/null
+++ b/src/__tests__/converters/pt9/languageAnalysisBuilder.test.ts
@@ -0,0 +1,219 @@
+///
+
+import type { InterlinearData, VerseData } from 'parsers/pt9/interlinearXmlParser';
+import { buildLanguageBookAnalyses } from '../../../converters/pt9/languageAnalysisBuilder';
+import { createPt9GlossSource } from '../../../converters/pt9/pt9GlossSource';
+import { emptyPt9ImportReport } from '../../../converters/pt9/report';
+import { makeVerseBook } from '../../test-helpers';
+
+/** Builds a one-lexeme word cluster the way the parser shapes one. */
+function wordCluster(index: number, length: number, form: string, senseId?: string) {
+ const id = `Word:${form}`;
+ return {
+ TextRange: { Index: index, Length: length },
+ Lexemes: [{ LexemeId: id, ...(senseId !== undefined && { SenseId: senseId }) }],
+ LexemesId: id,
+ Id: `${id}/${index}-${length}`,
+ Excluded: false,
+ };
+}
+
+/** Builds a stem+suffix parse cluster. */
+function parseCluster(index: number, length: number, stem: string, suffix: string) {
+ const lexemesId = `Stem:${stem}/Suffix:${suffix}`;
+ return {
+ TextRange: { Index: index, Length: length },
+ Lexemes: [{ LexemeId: `Stem:${stem}` }, { LexemeId: `Suffix:${suffix}` }],
+ LexemesId: lexemesId,
+ Id: `${lexemesId}/${index}-${length}`,
+ Excluded: false,
+ };
+}
+
+/** Wraps verses into an InterlinearData literal. */
+function fileOf(verses: Record, language = 'en'): InterlinearData {
+ return { GlossLanguage: language, BookId: 'GEN', Verses: verses };
+}
+
+const LEXICON = {
+ Entries: [
+ {
+ Key: { Type: 'Word', Form: 'hello' },
+ Senses: [{ Id: 'S1', Glosses: [{ Language: 'en', Text: 'greeting' }] }],
+ },
+ {
+ Key: { Type: 'Suffix', Form: 'ing' },
+ Senses: [{ Id: 'S2', Glosses: [{ Language: 'en', Text: 'PROG' }] }],
+ },
+ ],
+ Analyses: {},
+};
+
+describe('buildLanguageBookAnalyses', () => {
+ it('builds an approved word+parse record for a hashed verse and resolves glosses', () => {
+ const { senses } = emptyPt9ImportReport();
+ const book = makeVerseBook([{ sid: 'GEN 1:1', text: 'hello world' }]);
+ const build = buildLanguageBookAnalyses({
+ file: fileOf({
+ 'GEN 1:1': {
+ Hash: 'ABCD1234',
+ Clusters: [wordCluster(0, 5, 'hello', 'S1'), parseCluster(0, 5, 'hell', 'o')],
+ Punctuations: [{ TextRange: { Index: 0, Length: 1 }, BeforeText: ',', AfterText: '' }],
+ },
+ }),
+ tag: 'en',
+ book,
+ glossSource: createPt9GlossSource(LEXICON),
+ senses,
+ });
+
+ expect(build.records).toHaveLength(1);
+ const record = build.records[0];
+ expect(record.status).toBe('approved');
+ expect(record.tag).toBe('en');
+ expect(record.tokenRef).toBe('GEN 1:1:0');
+ expect(record.tokenSurface).toBe('hello');
+ expect(record.word).toStrictEqual({
+ key: { Type: 'Word', Form: 'hello' },
+ keyId: 'Word:hello',
+ senseId: 'S1',
+ glossText: 'greeting',
+ });
+ expect(record.parse?.signature).toBe('Stem:hell/Suffix:o');
+ expect(record.parse?.lexemes.map((l) => l.keyId)).toStrictEqual(['Stem:hell', 'Suffix:o']);
+
+ expect(build.bookReport).toStrictEqual({
+ bookId: 'GEN',
+ bookFound: true,
+ versesTotal: 1,
+ versesHashed: 1,
+ versesNotFound: 0,
+ clustersTotal: 2,
+ clustersConverted: 2,
+ phrasesConverted: 0,
+ clusterDrops: {
+ verseNotFound: 0,
+ formMismatch: 0,
+ lemmaOrOther: 0,
+ duplicateCluster: 0,
+ unparseableLexemeId: 0,
+ },
+ ambiguousAnchors: 0,
+ punctuationEntriesIgnored: 1,
+ });
+ expect(senses.specificResolved).toBe(1);
+ expect(senses.unresolvedGlossText).toBe(2);
+ });
+
+ it('marks records from an unhashed verse as suggested', () => {
+ const { senses } = emptyPt9ImportReport();
+ const book = makeVerseBook([{ sid: 'GEN 1:1', text: 'hello' }]);
+ const build = buildLanguageBookAnalyses({
+ file: fileOf({
+ 'GEN 1:1': { Clusters: [wordCluster(0, 5, 'hello')], Punctuations: [] },
+ }),
+ tag: 'en',
+ book,
+ glossSource: createPt9GlossSource(LEXICON),
+ senses,
+ });
+
+ expect(build.records[0].status).toBe('suggested');
+ expect(build.records[0].word?.senseId).toBe('S1');
+ expect(build.records[0].word?.glossText).toBe('greeting');
+ expect(senses.defaultSingleResolved).toBe(1);
+ });
+
+ it('marks a record rejected only when every anchored facet is excluded', () => {
+ const { senses } = emptyPt9ImportReport();
+ const book = makeVerseBook([{ sid: 'GEN 1:1', text: 'hello world' }]);
+ const excludedWord = { ...wordCluster(0, 5, 'hello', 'S1'), Excluded: true };
+ const mixedParse = parseCluster(0, 5, 'hell', 'o');
+ const excludedOnly = { ...wordCluster(6, 5, 'world', 'S9'), Excluded: true };
+ const build = buildLanguageBookAnalyses({
+ file: fileOf({
+ 'GEN 1:1': {
+ Hash: 'ABCD1234',
+ Clusters: [excludedWord, mixedParse, excludedOnly],
+ Punctuations: [],
+ },
+ }),
+ tag: 'en',
+ book,
+ glossSource: createPt9GlossSource(LEXICON),
+ senses,
+ });
+
+ expect(build.records.map((r) => r.status)).toStrictEqual(['approved', 'rejected']);
+ });
+
+ it('counts verses whose key has no segment and drops their clusters', () => {
+ const { senses } = emptyPt9ImportReport();
+ const book = makeVerseBook([{ sid: 'GEN 1:1', text: 'hello' }]);
+ const build = buildLanguageBookAnalyses({
+ file: fileOf({
+ 'GEN 1:1': { Hash: 'AA', Clusters: [wordCluster(0, 5, 'hello', 'S1')], Punctuations: [] },
+ 'GEN 1:9': {
+ Clusters: [wordCluster(0, 5, 'ghost')],
+ Punctuations: [{ TextRange: { Index: 0, Length: 1 }, BeforeText: '', AfterText: '' }],
+ },
+ }),
+ tag: 'en',
+ book,
+ glossSource: createPt9GlossSource(LEXICON),
+ senses,
+ });
+
+ expect(build.records).toHaveLength(1);
+ expect(build.bookReport.versesTotal).toBe(2);
+ expect(build.bookReport.versesNotFound).toBe(1);
+ expect(build.bookReport.clusterDrops.verseNotFound).toBe(1);
+ expect(build.bookReport.punctuationEntriesIgnored).toBe(1);
+ });
+
+ it('treats a missing book as every verse missing', () => {
+ const { senses } = emptyPt9ImportReport();
+ const build = buildLanguageBookAnalyses({
+ file: fileOf({
+ 'GEN 1:1': { Hash: 'AA', Clusters: [wordCluster(0, 5, 'hello', 'S1')], Punctuations: [] },
+ }),
+ tag: 'en',
+ book: undefined,
+ glossSource: createPt9GlossSource(LEXICON),
+ senses,
+ });
+
+ expect(build.records).toHaveLength(0);
+ expect(build.bookReport.bookFound).toBe(false);
+ expect(build.bookReport.versesNotFound).toBe(1);
+ expect(build.bookReport.versesHashed).toBe(1);
+ expect(build.bookReport.clusterDrops.verseNotFound).toBe(1);
+ });
+
+ it('builds phrase records with the verse status and Excluded rejection', () => {
+ const { senses } = emptyPt9ImportReport();
+ const book = makeVerseBook([{ sid: 'GEN 1:1', text: 'in the beginning in the' }]);
+ const phrase = (index: number, excluded: boolean) => ({
+ TextRange: { Index: index, Length: 6 },
+ Lexemes: [{ LexemeId: 'Phrase:in the', SenseId: 'S7' }],
+ LexemesId: 'Phrase:in the',
+ Id: `Phrase:in the/${index}-6`,
+ Excluded: excluded,
+ });
+ const build = buildLanguageBookAnalyses({
+ file: fileOf({
+ 'GEN 1:1': { Hash: 'AA', Clusters: [phrase(0, false), phrase(17, true)], Punctuations: [] },
+ }),
+ tag: 'en',
+ book,
+ glossSource: createPt9GlossSource(LEXICON),
+ senses,
+ });
+
+ expect(build.phrases).toHaveLength(2);
+ expect(build.phrases[0].status).toBe('approved');
+ expect(build.phrases[0].tokens.map((t) => t.surface)).toStrictEqual(['in', 'the']);
+ expect(build.phrases[1].status).toBe('rejected');
+ expect(build.bookReport.phrasesConverted).toBe(2);
+ });
+});
diff --git a/src/__tests__/converters/pt9/pt9GlossSource.test.ts b/src/__tests__/converters/pt9/pt9GlossSource.test.ts
new file mode 100644
index 00000000..c20074b1
--- /dev/null
+++ b/src/__tests__/converters/pt9/pt9GlossSource.test.ts
@@ -0,0 +1,148 @@
+///
+
+import type { LexiconData } from 'parsers/pt9/lexiconXmlParser';
+import { createPt9GlossSource } from '../../../converters/pt9/pt9GlossSource';
+
+/** A lexicon whose entries exercise every resolution path. */
+const LEXICON: LexiconData = {
+ Language: 'en',
+ Entries: [
+ {
+ Key: { Type: 'Word', Form: 'hello', Homograph: 1 },
+ Senses: [
+ {
+ Id: 'S1',
+ Glosses: [
+ { Language: 'EN', Text: 'greeting' },
+ { Language: 'fr', Text: 'salut' },
+ ],
+ },
+ { Id: 'S2', Glosses: [{ Language: 'fr', Text: 'coucou' }] },
+ ],
+ },
+ {
+ Key: { Type: 'Word', Form: 'empty' },
+ Senses: [{ Id: 'S3', Glosses: [{ Language: 'en', Text: '' }] }],
+ },
+ {
+ Key: { Type: 'Word', Form: 'multi' },
+ Senses: [
+ { Id: 'S4', Glosses: [{ Language: 'en', Text: 'one' }] },
+ { Id: 'S5', Glosses: [{ Language: 'en', Text: 'two' }] },
+ ],
+ },
+ {
+ Key: { Type: 'Word', Form: 'idless' },
+ Senses: [{ Glosses: [{ Language: 'en', Text: 'bare' }] }],
+ },
+ {
+ Key: { Type: 'Word', Form: 'taggless' },
+ Senses: [{ Id: 'S6', Glosses: [{ Text: 'no language' }] }],
+ },
+ ],
+ Analyses: {},
+};
+
+describe('createPt9GlossSource', () => {
+ const source = createPt9GlossSource(LEXICON);
+
+ it('resolves an explicit sense to its gloss for the language, matching case-insensitively', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'hello' }, 'S1', 'en')).toStrictEqual({
+ kind: 'specific',
+ senseId: 'S1',
+ text: 'greeting',
+ });
+ });
+
+ it('treats an absent Homograph and homograph 1 as the same entry', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'hello', Homograph: 1 }, 'S1', 'fr')).toStrictEqual(
+ { kind: 'specific', senseId: 'S1', text: 'salut' },
+ );
+ });
+
+ it('resolves an explicit sense with no gloss in the language to no text', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'hello' }, 'S2', 'en')).toStrictEqual({
+ kind: 'specific',
+ senseId: 'S2',
+ });
+ });
+
+ it('resolves an explicit sense whose gloss is empty to no text', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'empty' }, 'S3', 'en')).toStrictEqual({
+ kind: 'specific',
+ senseId: 'S3',
+ });
+ });
+
+ it('resolves a dangling sense id to no text', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'hello' }, 'MISSING', 'en')).toStrictEqual({
+ kind: 'specific',
+ senseId: 'MISSING',
+ });
+ });
+
+ it('resolves an explicit sense on a missing entry to no text', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'absent' }, 'S9', 'en')).toStrictEqual({
+ kind: 'specific',
+ senseId: 'S9',
+ });
+ });
+
+ it('treats an empty sense id as no selection', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'hello' }, '', 'en')).toStrictEqual({
+ kind: 'defaultSingle',
+ senseId: 'S1',
+ text: 'greeting',
+ });
+ });
+
+ it('resolves no selection to the single glossed sense in the language', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'hello' }, undefined, 'en')).toStrictEqual({
+ kind: 'defaultSingle',
+ senseId: 'S1',
+ text: 'greeting',
+ });
+ });
+
+ it('resolves no selection to none when several senses are glossed in the language', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'multi' }, undefined, 'en')).toStrictEqual({
+ kind: 'none',
+ });
+ });
+
+ it('resolves no selection to none when no sense is glossed in the language', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'hello' }, undefined, 'de')).toStrictEqual({
+ kind: 'none',
+ });
+ });
+
+ it('resolves no selection to none for a missing entry', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'absent' }, undefined, 'en')).toStrictEqual({
+ kind: 'none',
+ });
+ });
+
+ it('returns a default from an id-less sense without a sense id', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'idless' }, undefined, 'en')).toStrictEqual({
+ kind: 'defaultSingle',
+ text: 'bare',
+ });
+ });
+
+ it('never matches a gloss that carries no Language attribute', () => {
+ expect(source.resolve({ Type: 'Word', Form: 'taggless' }, undefined, 'en')).toStrictEqual({
+ kind: 'none',
+ });
+ });
+
+ it('resolves everything to selection-only outcomes when there is no lexicon', () => {
+ const empty = createPt9GlossSource(undefined);
+ expect(empty.resolve({ Type: 'Word', Form: 'hello' }, 'S1', 'en')).toStrictEqual({
+ kind: 'specific',
+ senseId: 'S1',
+ });
+ expect(empty.resolve({ Type: 'Word', Form: 'hello' }, undefined, 'en')).toStrictEqual({
+ kind: 'none',
+ });
+ });
+});
diff --git a/src/converters/pt9/analysisMerger.ts b/src/converters/pt9/analysisMerger.ts
new file mode 100644
index 00000000..b9d593af
--- /dev/null
+++ b/src/converters/pt9/analysisMerger.ts
@@ -0,0 +1,402 @@
+import type {
+ AssignmentStatus,
+ MorphemeAnalysis,
+ MultiString,
+ PhraseAnalysis,
+ PhraseAnalysisLink,
+ TokenAnalysis,
+ TokenAnalysisLink,
+} from 'interlinearizer';
+import type { LexemeKeyData } from 'parsers/pt9/lexemeKey';
+import { normalizeSurfaceForm } from '../../utils/analysis-identity';
+import type {
+ LangPhraseRecord,
+ LangRecordStatus,
+ LangTokenRecord,
+} from './languageAnalysisBuilder';
+import type { Pt9LexiconResolver } from './lexiconResolver';
+import type { Pt9ImportReport } from './report';
+
+/** The merged analysis layer, plus the identity of every parse converted into it. */
+export interface MergedAnalyses {
+ tokenAnalyses: TokenAnalysis[];
+ tokenAnalysisLinks: TokenAnalysisLink[];
+ phraseAnalyses: PhraseAnalysis[];
+ phraseAnalysisLinks: PhraseAnalysisLink[];
+ /**
+ * `foldedSurface|keyId/keyId` for every parse converted here, so the same analysis arriving from
+ * another source can be recognized as a duplicate.
+ */
+ clusterParseIdentities: Set;
+}
+
+/** One language's per-morpheme sense and gloss columns for a parse. */
+interface ParseColumns {
+ senses: (string | undefined)[];
+ glosses: (string | undefined)[];
+}
+
+/** The parse facet of a merged token record, with per-language columns keyed by tag. */
+interface MergedParse {
+ signature: string;
+ keys: LexemeKeyData[];
+ columns: Map;
+}
+
+/** One token record accumulating contributions across languages. */
+interface MergedToken {
+ tokenRef: string;
+ surface: string;
+ writingSystem: string;
+ wordKey?: LexemeKeyData;
+ /** Tag -> word-level gloss text (first contribution per tag wins). */
+ glosses: Map;
+ /** Tag -> the word facet's effective sense id (first contribution per tag wins). */
+ wordSenses: Map;
+ parse?: MergedParse;
+ statuses: LangRecordStatus[];
+ ambiguous: boolean;
+ tags: Set;
+}
+
+/** One phrase record accumulating contributions across languages. */
+interface MergedPhrase {
+ key: LexemeKeyData;
+ tokens: { ref: string; surface: string }[];
+ glosses: Map;
+ senses: Map;
+ statuses: LangRecordStatus[];
+ ambiguous: boolean;
+}
+
+/** Merged review status: approved or rejected only when every contribution agrees. */
+function mergeStatus(statuses: LangRecordStatus[]): LangRecordStatus {
+ if (statuses.every((s) => s === 'approved')) return 'approved';
+ if (statuses.every((s) => s === 'rejected')) return 'rejected';
+ return 'suggested';
+}
+
+/** A MultiString built from per-tag texts, or `undefined` when no tag has one. */
+function toMultiString(entries: Iterable<[string, string | undefined]>): MultiString | undefined {
+ const result: MultiString = {};
+ [...entries].forEach(([tag, text]) => {
+ if (text !== undefined) result[tag] = text;
+ });
+ return Object.keys(result).length === 0 ? undefined : result;
+}
+
+/** The single value all contributions agree on, or `undefined` when absent or contested. */
+function unanimous(values: Iterable): string | undefined {
+ const list = [...values];
+ /* v8 ignore next -- no caller passes an empty collection; the guard keeps the contract total */
+ if (list.length === 0) return undefined;
+ const [agreed] = list;
+ if (agreed === undefined) return undefined;
+ return list.every((value) => value === agreed) ? agreed : undefined;
+}
+
+/**
+ * Merges per-language contributions into the final analysis layer.
+ *
+ * Contributions sharing a token and word lexeme become one record whose glosses are keyed by
+ * language tag; a parse from any language fills a record that lacks one, while a genuinely
+ * different parse on the same token becomes a separate competing record. A sense reference is kept
+ * only where every contributing language agrees on the sense. At most one record per token may be
+ * approved, so later would-be-approved records are demoted to candidate.
+ */
+export function mergeLanguageAnalyses(args: {
+ records: LangTokenRecord[];
+ phrases: LangPhraseRecord[];
+ resolver: Pt9LexiconResolver;
+ /** Stamp applied to every record and link's createdAt / updatedAt. */
+ importedAt: string;
+ /** Merge and sense counters are incremented in place. */
+ report: Pt9ImportReport;
+}): MergedAnalyses {
+ const { records, phrases, resolver, importedAt, report } = args;
+
+ const merged: MergedToken[] = [];
+ const mergedByKey = new Map();
+ const byToken = new Map();
+
+ const createMerged = (key: string, record: LangTokenRecord): MergedToken => {
+ const entry: MergedToken = {
+ tokenRef: record.tokenRef,
+ surface: record.tokenSurface,
+ writingSystem: record.tokenWritingSystem,
+ glosses: new Map(),
+ wordSenses: new Map(),
+ statuses: [],
+ ambiguous: false,
+ tags: new Set(),
+ };
+ merged.push(entry);
+ mergedByKey.set(key, entry);
+ let list = byToken.get(record.tokenRef);
+ if (list === undefined) {
+ list = [];
+ byToken.set(record.tokenRef, list);
+ }
+ list.push(entry);
+ return entry;
+ };
+
+ const contribute = (entry: MergedToken, record: LangTokenRecord): void => {
+ entry.statuses.push(record.status);
+ entry.ambiguous = entry.ambiguous || record.ambiguous;
+ entry.tags.add(record.tag);
+ if (record.word !== undefined) {
+ entry.wordKey = entry.wordKey ?? record.word.key;
+ if (!entry.wordSenses.has(record.tag)) entry.wordSenses.set(record.tag, record.word.senseId);
+ if (record.word.glossText !== undefined && !entry.glosses.has(record.tag))
+ entry.glosses.set(record.tag, record.word.glossText);
+ }
+ if (record.parse !== undefined) {
+ if (entry.parse === undefined) {
+ entry.parse = {
+ signature: record.parse.signature,
+ keys: record.parse.lexemes.map((l) => l.key),
+ columns: new Map(),
+ };
+ }
+ if (!entry.parse.columns.has(record.tag)) {
+ entry.parse.columns.set(record.tag, {
+ senses: record.parse.lexemes.map((l) => l.senseId),
+ glosses: record.parse.lexemes.map((l) => l.glossText),
+ });
+ }
+ }
+ };
+
+ records.forEach((record) => {
+ if (record.word !== undefined) {
+ const baseKey = `${record.tokenRef}|w|${record.word.keyId}`;
+ const existing = mergedByKey.get(baseKey);
+ if (existing === undefined) {
+ contribute(createMerged(baseKey, record), record);
+ return;
+ }
+ const recordParse = record.parse;
+ if (
+ recordParse === undefined ||
+ existing.parse === undefined ||
+ existing.parse.signature === recordParse.signature
+ ) {
+ contribute(existing, record);
+ return;
+ }
+ const conflictKey = `${baseKey}|p|${recordParse.signature}`;
+ const conflictEntry = mergedByKey.get(conflictKey);
+ if (conflictEntry !== undefined) {
+ contribute(conflictEntry, record);
+ return;
+ }
+ report.merge.parseConflicts += 1;
+ contribute(createMerged(conflictKey, record), record);
+ return;
+ }
+ // Parse-only contribution: keyed by its signature; fused onto word records afterward.
+ const parseKey = `${record.tokenRef}|p|${record.parse?.signature}`;
+ const existing = mergedByKey.get(parseKey);
+ contribute(existing ?? createMerged(parseKey, record), record);
+ });
+
+ // Fuse parse-only records onto the word record they complete: the one sharing their parse, or
+ // the sole word record still lacking a parse. An ambiguous target leaves them standalone.
+ const removed = new Set();
+ byToken.forEach((list) => {
+ list
+ .filter((entry) => entry.wordKey === undefined && entry.parse !== undefined)
+ .forEach((standalone) => {
+ const standaloneParse = standalone.parse;
+ /* v8 ignore next 2 -- the filter above guarantees a parse facet */
+ if (standaloneParse === undefined) return;
+ const sameParseTarget = list.find(
+ (entry) =>
+ entry !== standalone &&
+ !removed.has(entry) &&
+ entry.parse?.signature === standaloneParse.signature,
+ );
+ const wordsWithoutParse = list.filter(
+ (entry) =>
+ entry !== standalone &&
+ !removed.has(entry) &&
+ entry.wordKey !== undefined &&
+ entry.parse === undefined,
+ );
+ const target =
+ sameParseTarget ?? (wordsWithoutParse.length === 1 ? wordsWithoutParse[0] : undefined);
+ if (target === undefined) return;
+ const targetParse = target.parse;
+ if (targetParse === undefined) target.parse = standaloneParse;
+ else {
+ standaloneParse.columns.forEach((columns, tag) => {
+ if (!targetParse.columns.has(tag)) targetParse.columns.set(tag, columns);
+ });
+ }
+ target.statuses.push(...standalone.statuses);
+ target.ambiguous = target.ambiguous || standalone.ambiguous;
+ standalone.tags.forEach((tag) => target.tags.add(tag));
+ removed.add(standalone);
+ });
+ });
+
+ const resolveSenseRef = (key: LexemeKeyData, senseId: string | undefined) => {
+ if (senseId === undefined) return undefined;
+ const ref = resolver.resolveSense(key, senseId);
+ if (ref === undefined) report.senses.senseRefsUnresolved += 1;
+ else report.senses.senseRefsResolved += 1;
+ return ref;
+ };
+
+ const tokenAnalyses: TokenAnalysis[] = [];
+ const tokenAnalysisLinks: TokenAnalysisLink[] = [];
+ const clusterParseIdentities = new Set();
+ const approvedTokenSeen = new Set();
+ const idCounters = new Map();
+
+ merged
+ .filter((entry) => !removed.has(entry))
+ .forEach((entry) => {
+ if (entry.tags.size > 1) report.merge.mergedTokenRecords += 1;
+
+ let status: AssignmentStatus = mergeStatus(entry.statuses);
+ if (status === 'approved') {
+ if (approvedTokenSeen.has(entry.tokenRef)) {
+ status = 'candidate';
+ report.merge.approvedDemotedToCandidate += 1;
+ } else approvedTokenSeen.add(entry.tokenRef);
+ }
+
+ const { parse } = entry;
+ const morphemes: MorphemeAnalysis[] | undefined =
+ parse === undefined
+ ? undefined
+ : parse.keys.map((key, i) => {
+ const entryRef = resolver.resolveEntry(key);
+ if (entryRef === undefined) report.senses.entryRefsUnresolved += 1;
+ else report.senses.entryRefsResolved += 1;
+ const senseColumns = [...parse.columns.values()].map((column) => column.senses[i]);
+ const senseRef = resolveSenseRef(key, unanimous(senseColumns));
+ const gloss = toMultiString(
+ [...parse.columns.entries()].map(([tag, column]) => [tag, column.glosses[i]]),
+ );
+ return {
+ id: `m${i}`,
+ form: key.Form,
+ writingSystem: entry.writingSystem,
+ ...(gloss !== undefined && { gloss }),
+ ...(entryRef !== undefined && { entryRef }),
+ ...(senseRef !== undefined && { senseRef }),
+ };
+ });
+
+ const gloss = toMultiString(entry.glosses.entries());
+ const { wordKey } = entry;
+ const glossSenseRef =
+ wordKey === undefined
+ ? undefined
+ : resolveSenseRef(wordKey, unanimous(entry.wordSenses.values()));
+
+ const count = idCounters.get(entry.tokenRef) ?? 0;
+ idCounters.set(entry.tokenRef, count + 1);
+ const id = `pt9:ta:${entry.tokenRef}:${count}`;
+
+ if (parse !== undefined)
+ clusterParseIdentities.add(`${normalizeSurfaceForm(entry.surface)}|${parse.signature}`);
+
+ tokenAnalyses.push({
+ id,
+ createdAt: importedAt,
+ updatedAt: importedAt,
+ surfaceText: entry.surface,
+ producer: 'pt9-import',
+ ...(gloss !== undefined && { gloss }),
+ ...(glossSenseRef !== undefined && { glossSenseRef }),
+ ...(morphemes !== undefined && { morphemes }),
+ });
+ tokenAnalysisLinks.push({
+ analysisId: id,
+ createdAt: importedAt,
+ updatedAt: importedAt,
+ status,
+ ...(entry.ambiguous && { confidence: 'low' }),
+ token: { tokenRef: entry.tokenRef, surfaceText: entry.surface },
+ });
+ });
+
+ // Phrases merge on their token run plus phrase lexeme, then honor the one-approved-phrase-per-
+ // token invariant in insertion order.
+ const mergedPhrases: MergedPhrase[] = [];
+ const phrasesByKey = new Map();
+ phrases.forEach((record) => {
+ // Token refs contain spaces, so the joiner must be a character no ref can carry.
+ const key = `${record.tokens.map((t) => t.ref).join('\n')}|${record.phrase.keyId}`;
+ let entry = phrasesByKey.get(key);
+ if (entry === undefined) {
+ entry = {
+ key: record.phrase.key,
+ tokens: record.tokens,
+ glosses: new Map(),
+ senses: new Map(),
+ statuses: [],
+ ambiguous: false,
+ };
+ mergedPhrases.push(entry);
+ phrasesByKey.set(key, entry);
+ }
+ entry.statuses.push(record.status);
+ entry.ambiguous = entry.ambiguous || record.ambiguous;
+ if (!entry.senses.has(record.tag)) entry.senses.set(record.tag, record.phrase.senseId);
+ if (record.phrase.glossText !== undefined && !entry.glosses.has(record.tag))
+ entry.glosses.set(record.tag, record.phrase.glossText);
+ });
+
+ const phraseAnalyses: PhraseAnalysis[] = [];
+ const phraseAnalysisLinks: PhraseAnalysisLink[] = [];
+ const approvedPhraseTokens = new Set();
+ const phraseIdCounters = new Map();
+ mergedPhrases.forEach((entry) => {
+ let status: AssignmentStatus = mergeStatus(entry.statuses);
+ if (status === 'approved') {
+ if (entry.tokens.some((t) => approvedPhraseTokens.has(t.ref))) {
+ status = 'candidate';
+ report.merge.approvedDemotedToCandidate += 1;
+ } else entry.tokens.forEach((t) => approvedPhraseTokens.add(t.ref));
+ }
+
+ const gloss = toMultiString(entry.glosses.entries());
+ const senseRef = resolveSenseRef(entry.key, unanimous(entry.senses.values()));
+
+ const firstRef = entry.tokens[0].ref;
+ const count = phraseIdCounters.get(firstRef) ?? 0;
+ phraseIdCounters.set(firstRef, count + 1);
+ const id = `pt9:pa:${firstRef}:${count}`;
+
+ phraseAnalyses.push({
+ id,
+ createdAt: importedAt,
+ updatedAt: importedAt,
+ surfaceText: entry.tokens.map((t) => t.surface).join(' '),
+ producer: 'pt9-import',
+ ...(gloss !== undefined && { gloss }),
+ ...(senseRef !== undefined && { senseRef }),
+ });
+ phraseAnalysisLinks.push({
+ analysisId: id,
+ createdAt: importedAt,
+ updatedAt: importedAt,
+ status,
+ ...(entry.ambiguous && { confidence: 'low' }),
+ tokens: entry.tokens.map((t) => ({ tokenRef: t.ref, surfaceText: t.surface })),
+ });
+ });
+
+ return {
+ tokenAnalyses,
+ tokenAnalysisLinks,
+ phraseAnalyses,
+ phraseAnalysisLinks,
+ clusterParseIdentities,
+ };
+}
diff --git a/src/converters/pt9/bareWordAnalyses.ts b/src/converters/pt9/bareWordAnalyses.ts
new file mode 100644
index 00000000..a3a56825
--- /dev/null
+++ b/src/converters/pt9/bareWordAnalyses.ts
@@ -0,0 +1,146 @@
+import type { MorphemeAnalysis, TokenAnalysis } from 'interlinearizer';
+import type { LexiconData } from 'parsers/pt9/lexiconXmlParser';
+import { composeLexemeKeyId, LexemeKeyData, parseLexemeKeyId } from 'parsers/pt9/lexemeKey';
+import type { WordAnalysesData } from 'parsers/pt9/wordAnalysesXmlParser';
+import { normalizeSurfaceForm } from '../../utils/analysis-identity';
+import type { Pt9LexiconResolver } from './lexiconResolver';
+import type { Pt9GlossSource } from './pt9GlossSource';
+import type { Pt9ImportReport } from './report';
+
+/** One deduplicated wordform analysis from the inventories, keys parsed and signature composed. */
+interface InventoryAnalysis {
+ word: string;
+ keys: LexemeKeyData[];
+ signature: string;
+}
+
+/**
+ * Builds the unlinked token-analysis payloads from PT9's wordform-to-parse inventories:
+ * `WordAnalyses.xml` first, then the legacy lexicon `Analyses` section for the wordforms and parses
+ * the newer file lacks. These payloads describe a spelling rather than any one occurrence of it, so
+ * they carry no links. An analysis identical to one already converted from a cluster is skipped
+ * rather than duplicated.
+ */
+export function buildBareWordAnalyses(args: {
+ wordAnalyses: WordAnalysesData | undefined;
+ lexicon: LexiconData | undefined;
+ /** Gloss languages in discovery order, raw value paired with its resolved tag. */
+ languages: { raw: string; tag: string }[];
+ glossSource: Pt9GlossSource;
+ resolver: Pt9LexiconResolver;
+ /** Parse identities already converted from clusters, which are skipped here. */
+ clusterParseIdentities: ReadonlySet;
+ /** Writing system stamped on morpheme forms (bare payloads have no token to inherit from). */
+ writingSystem: string;
+ /** Stamp applied to every payload's createdAt / updatedAt. */
+ importedAt: string;
+ /** Bare-payload and lexicon-ref counters are incremented in place. */
+ report: Pt9ImportReport;
+}): TokenAnalysis[] {
+ const {
+ wordAnalyses,
+ lexicon,
+ languages,
+ glossSource,
+ resolver,
+ clusterParseIdentities,
+ writingSystem,
+ importedAt,
+ report,
+ } = args;
+
+ const inventory: InventoryAnalysis[] = [];
+ const seenByWord = new Map>();
+
+ const addAnalysis = (word: string, keys: LexemeKeyData[]): void => {
+ if (keys.length === 0) {
+ report.barePayloads.droppedEmpty += 1;
+ return;
+ }
+ const signature = keys.map(composeLexemeKeyId).join('/');
+ let seen = seenByWord.get(word);
+ if (seen === undefined) {
+ seen = new Set();
+ seenByWord.set(word, seen);
+ }
+ if (seen.has(signature)) return;
+ seen.add(signature);
+ inventory.push({ word, keys, signature });
+ };
+
+ (wordAnalyses?.Entries ?? []).forEach((entry) => {
+ entry.Analyses.forEach((analysis) => {
+ const keys = analysis.LexemeIds.flatMap((id) => {
+ const key = parseLexemeKeyId(id);
+ return key === undefined ? [] : [key];
+ });
+ if (keys.length !== analysis.LexemeIds.length) {
+ report.barePayloads.droppedUnparseable += 1;
+ return;
+ }
+ addAnalysis(entry.Word, keys);
+ });
+ });
+ Object.entries(lexicon?.Analyses ?? {}).forEach(([word, keys]) => {
+ addAnalysis(word, keys);
+ });
+
+ const payloads: TokenAnalysis[] = [];
+ const idCounters = new Map();
+
+ inventory.forEach(({ word, keys, signature }) => {
+ if (clusterParseIdentities.has(`${normalizeSurfaceForm(word)}|${signature}`)) {
+ report.barePayloads.skippedExistingIdentical += 1;
+ return;
+ }
+
+ const morphemes: MorphemeAnalysis[] = keys.map((key, i) => {
+ const entryRef = resolver.resolveEntry(key);
+ if (entryRef === undefined) report.senses.entryRefsUnresolved += 1;
+ else report.senses.entryRefsResolved += 1;
+
+ const glossEntries: [string, string][] = [];
+ const defaultSenseIds = new Set();
+ languages.forEach((language) => {
+ const outcome = glossSource.resolve(key, undefined, language.raw);
+ if (outcome.kind !== 'defaultSingle') return;
+ if (glossEntries.every(([tag]) => tag !== language.tag))
+ glossEntries.push([language.tag, outcome.text]);
+ if (outcome.senseId !== undefined) defaultSenseIds.add(outcome.senseId);
+ });
+
+ // Languages without a single default abstain; a ref is attempted only when every language
+ // that found one found the same sense.
+ let senseRef;
+ if (defaultSenseIds.size === 1) {
+ const [senseId] = defaultSenseIds;
+ senseRef = resolver.resolveSense(key, senseId);
+ if (senseRef === undefined) report.senses.senseRefsUnresolved += 1;
+ else report.senses.senseRefsResolved += 1;
+ }
+
+ return {
+ id: `m${i}`,
+ form: key.Form,
+ writingSystem,
+ ...(glossEntries.length > 0 && { gloss: Object.fromEntries(glossEntries) }),
+ ...(entryRef !== undefined && { entryRef }),
+ ...(senseRef !== undefined && { senseRef }),
+ };
+ });
+
+ const count = idCounters.get(word) ?? 0;
+ idCounters.set(word, count + 1);
+ report.barePayloads.added += 1;
+ payloads.push({
+ id: `pt9:wa:${word}:${count}`,
+ createdAt: importedAt,
+ updatedAt: importedAt,
+ surfaceText: word,
+ producer: 'pt9-import:word-analyses',
+ morphemes,
+ });
+ });
+
+ return payloads;
+}
diff --git a/src/converters/pt9/clusterAnchoring.ts b/src/converters/pt9/clusterAnchoring.ts
new file mode 100644
index 00000000..fe8aea72
--- /dev/null
+++ b/src/converters/pt9/clusterAnchoring.ts
@@ -0,0 +1,248 @@
+import type { Segment, Token } from 'interlinearizer';
+import type { ClusterData } from 'parsers/pt9/interlinearXmlParser';
+import { LexemeKeyData, parseLexemeKeyId } from 'parsers/pt9/lexemeKey';
+import { normalizeSurfaceForm } from '../../utils/analysis-identity';
+import { emptyClusterDrops, Pt9ClusterDropReason } from './report';
+
+/** One lexeme of a classified cluster: its parsed key and the cluster's sense selection for it. */
+export interface ClassifiedLexeme {
+ key: LexemeKeyData;
+ senseId?: string;
+}
+
+/**
+ * A cluster sorted into the kind that decides its conversion, mirroring how PT9 derives cluster
+ * type from lexeme types: any stem/suffix/prefix makes a word parse; a single Word or Phrase lexeme
+ * makes those kinds; everything else (Lemma clusters, empty clusters, unknown types) is dropped.
+ */
+export type ClassifiedCluster =
+ | { kind: 'word'; cluster: ClusterData; lexeme: ClassifiedLexeme }
+ | { kind: 'wordParse'; cluster: ClusterData; lexemes: ClassifiedLexeme[] }
+ | { kind: 'phrase'; cluster: ClusterData; lexeme: ClassifiedLexeme }
+ | { kind: 'drop'; cluster: ClusterData; reason: 'lemmaOrOther' | 'unparseableLexemeId' };
+
+/** Lexeme types whose presence makes a cluster a word parse in PT9's derivation. */
+const PARSE_TYPES = new Set(['Stem', 'Suffix', 'Prefix']);
+
+/** Classifies one cluster by its lexeme-id prefixes (the type is never persisted in the XML). */
+export function classifyCluster(cluster: ClusterData): ClassifiedCluster {
+ const lexemes = cluster.Lexemes.flatMap((lexeme): ClassifiedLexeme[] => {
+ const key = parseLexemeKeyId(lexeme.LexemeId);
+ if (key === undefined) return [];
+ return [{ key, ...(lexeme.SenseId !== undefined && { senseId: lexeme.SenseId }) }];
+ });
+ if (lexemes.length !== cluster.Lexemes.length)
+ return { kind: 'drop', cluster, reason: 'unparseableLexemeId' };
+
+ if (lexemes.some((l) => PARSE_TYPES.has(l.key.Type)))
+ return { kind: 'wordParse', cluster, lexemes };
+ if (lexemes.length === 1 && lexemes[0].key.Type === 'Word')
+ return { kind: 'word', cluster, lexeme: lexemes[0] };
+ if (lexemes.length === 1 && lexemes[0].key.Type === 'Phrase')
+ return { kind: 'phrase', cluster, lexeme: lexemes[0] };
+ return { kind: 'drop', cluster, reason: 'lemmaOrOther' };
+}
+
+/** A token with the word and/or parse cluster that anchored to it. */
+export interface AnchoredTokenGroup {
+ token: Token;
+ /** The word cluster that landed on this token, carrying its Excluded flag. */
+ word?: { lexeme: ClassifiedLexeme; excluded: boolean };
+ /** The parse cluster that landed on this token, carrying its Excluded flag. */
+ parse?: { lexemes: ClassifiedLexeme[]; excluded: boolean };
+ /** True when several tokens folded to the cluster's form, so the choice among them is a guess. */
+ ambiguous: boolean;
+}
+
+/** A phrase cluster anchored to a consecutive run of word tokens. */
+export interface AnchoredPhrase {
+ lexeme: ClassifiedLexeme;
+ excluded: boolean;
+ tokens: Token[];
+ ambiguous: boolean;
+}
+
+/** The anchoring outcome for one verse. */
+export interface VerseAnchorResult {
+ groups: AnchoredTokenGroup[];
+ phrases: AnchoredPhrase[];
+ dropCounts: Record;
+ /** Anchors (token groups and phrases) placed ambiguously. */
+ ambiguousCount: number;
+}
+
+/**
+ * Picks the candidate whose relative text position best matches the cluster's relative range
+ * position, which is what separates repeated surface forms. Offsets index different strings (plain
+ * baseline vs. PT9's marker-bearing USFM), so only the proportion is meaningful, never the absolute
+ * values themselves.
+ */
+function pickByProportionalPrior(
+ candidates: number[],
+ wordTokens: Token[],
+ clusterIndex: number,
+ baselineLength: number,
+ verseExtent: number,
+): number {
+ const target = clusterIndex / verseExtent;
+ let best = candidates[0];
+ let bestDistance = Number.POSITIVE_INFINITY;
+ candidates.forEach((candidate) => {
+ const distance = Math.abs(wordTokens[candidate].charStart / baselineLength - target);
+ if (distance < bestDistance) {
+ bestDistance = distance;
+ best = candidate;
+ }
+ });
+ return best;
+}
+
+/** A word and parse cluster paired by their identical text range, the way PT9 pairs them. */
+interface RangeGroup {
+ index: number;
+ length: number;
+ word?: { classified: Extract };
+ parse?: { classified: Extract };
+}
+
+/**
+ * Anchors one verse's clusters onto its segment's word tokens.
+ *
+ * Lexeme forms are the ground truth: they are matched case- and normalization-folded, in order,
+ * because PT9's stored offsets index a different string than the segment's baseline text and cannot
+ * be applied to it. The range index therefore serves only as ordering and, among equal-folding
+ * candidates, as a position prior. Word and parse clusters covering the identical range anchor
+ * together onto one token, while phrases anchor to consecutive runs of word tokens. Clusters that
+ * match nothing are counted by reason rather than silently lost.
+ */
+export function anchorVerseClusters(segment: Segment, clusters: ClusterData[]): VerseAnchorResult {
+ const dropCounts = emptyClusterDrops();
+ const wordClassified: Extract[] = [];
+ const parseClassified: Extract[] = [];
+ const phraseClassified: Extract[] = [];
+
+ clusters.forEach((cluster) => {
+ const classified = classifyCluster(cluster);
+ if (classified.kind === 'drop') dropCounts[classified.reason] += 1;
+ else if (classified.kind === 'word') wordClassified.push(classified);
+ else if (classified.kind === 'wordParse') parseClassified.push(classified);
+ else phraseClassified.push(classified);
+ });
+
+ // Pair word and parse clusters by exact range; a second cluster of the same kind at the same
+ // range is corrupt by PT9's own selection rules, so only the first converts.
+ const rangeGroups = new Map();
+ const groupFor = (cluster: ClusterData): RangeGroup => {
+ const rangeKey = `${cluster.TextRange.Index}-${cluster.TextRange.Length}`;
+ let group = rangeGroups.get(rangeKey);
+ if (group === undefined) {
+ group = { index: cluster.TextRange.Index, length: cluster.TextRange.Length };
+ rangeGroups.set(rangeKey, group);
+ }
+ return group;
+ };
+ wordClassified.forEach((classified) => {
+ const group = groupFor(classified.cluster);
+ if (group.word !== undefined) dropCounts.duplicateCluster += 1;
+ else group.word = { classified };
+ });
+ parseClassified.forEach((classified) => {
+ const group = groupFor(classified.cluster);
+ if (group.parse !== undefined) dropCounts.duplicateCluster += 1;
+ else group.parse = { classified };
+ });
+
+ const wordTokens = segment.tokens.filter((t) => t.type === 'word');
+ const baselineLength = Math.max(1, segment.baselineText.length);
+ const verseExtent = Math.max(1, ...clusters.map((c) => c.TextRange.Index + c.TextRange.Length));
+
+ const groups: AnchoredTokenGroup[] = [];
+ let ambiguousCount = 0;
+ let cursor = 0;
+ [...rangeGroups.values()]
+ .sort((a, b) => a.index - b.index || a.length - b.length)
+ .forEach((group) => {
+ const { word, parse } = group;
+ const facetCount = (word === undefined ? 0 : 1) + (parse === undefined ? 0 : 1);
+ let expected: string;
+ if (word !== undefined) expected = normalizeSurfaceForm(word.classified.lexeme.key.Form);
+ else if (parse !== undefined)
+ expected = normalizeSurfaceForm(parse.classified.lexemes.map((l) => l.key.Form).join(''));
+ /* v8 ignore next 2 -- a range group is only ever created with at least one facet */
+ else return;
+
+ const candidates: number[] = [];
+ for (let j = cursor; j < wordTokens.length; j += 1) {
+ if (normalizeSurfaceForm(wordTokens[j].surfaceText) === expected) candidates.push(j);
+ }
+ if (candidates.length === 0) {
+ dropCounts.formMismatch += facetCount;
+ return;
+ }
+ const ambiguous = candidates.length > 1;
+ const chosen = ambiguous
+ ? pickByProportionalPrior(candidates, wordTokens, group.index, baselineLength, verseExtent)
+ : candidates[0];
+ if (ambiguous) ambiguousCount += 1;
+ groups.push({
+ token: wordTokens[chosen],
+ ...(group.word !== undefined && {
+ word: {
+ lexeme: group.word.classified.lexeme,
+ excluded: group.word.classified.cluster.Excluded,
+ },
+ }),
+ ...(group.parse !== undefined && {
+ parse: {
+ lexemes: group.parse.classified.lexemes,
+ excluded: group.parse.classified.cluster.Excluded,
+ },
+ }),
+ ambiguous,
+ });
+ cursor = chosen + 1;
+ });
+
+ const phrases: AnchoredPhrase[] = [];
+ let phraseCursor = 0;
+ [...phraseClassified]
+ .sort((a, b) => a.cluster.TextRange.Index - b.cluster.TextRange.Index)
+ .forEach((classified) => {
+ const words = normalizeSurfaceForm(classified.lexeme.key.Form)
+ .split(' ')
+ .filter((w) => w !== '');
+ if (words.length === 0) {
+ dropCounts.formMismatch += 1;
+ return;
+ }
+ const starts: number[] = [];
+ for (let s = phraseCursor; s + words.length <= wordTokens.length; s += 1) {
+ if (words.every((w, i) => normalizeSurfaceForm(wordTokens[s + i].surfaceText) === w))
+ starts.push(s);
+ }
+ if (starts.length === 0) {
+ dropCounts.formMismatch += 1;
+ return;
+ }
+ const ambiguous = starts.length > 1;
+ const start = ambiguous
+ ? pickByProportionalPrior(
+ starts,
+ wordTokens,
+ classified.cluster.TextRange.Index,
+ baselineLength,
+ verseExtent,
+ )
+ : starts[0];
+ if (ambiguous) ambiguousCount += 1;
+ phrases.push({
+ lexeme: classified.lexeme,
+ excluded: classified.cluster.Excluded,
+ tokens: wordTokens.slice(start, start + words.length),
+ ambiguous,
+ });
+ phraseCursor = start + 1;
+ });
+
+ return { groups, phrases, dropCounts, ambiguousCount };
+}
diff --git a/src/converters/pt9/convertPt9Project.ts b/src/converters/pt9/convertPt9Project.ts
new file mode 100644
index 00000000..6fe84992
--- /dev/null
+++ b/src/converters/pt9/convertPt9Project.ts
@@ -0,0 +1,182 @@
+import type { Book, TextAnalysis } from 'interlinearizer';
+import type { InterlinearData } from 'parsers/pt9/interlinearXmlParser';
+import type { InterlinearSetupsData } from 'parsers/pt9/interlinearSetupXmlParser';
+import type { LexiconData } from 'parsers/pt9/lexiconXmlParser';
+import type { WordAnalysesData } from 'parsers/pt9/wordAnalysesXmlParser';
+import { mergeLanguageAnalyses } from './analysisMerger';
+import { buildBareWordAnalyses } from './bareWordAnalyses';
+import { resolveGlossLanguageTag } from './glossLanguageTags';
+import {
+ buildLanguageBookAnalyses,
+ LangPhraseRecord,
+ LangTokenRecord,
+} from './languageAnalysisBuilder';
+import { Pt9LexiconResolver, unresolvedPt9LexiconResolver } from './lexiconResolver';
+import { createPt9GlossSource } from './pt9GlossSource';
+import { emptyPt9ImportReport, Pt9ImportReport, Pt9LanguageReport } from './report';
+
+/** The parsed PT9 file set for one project. */
+export interface Pt9ProjectData {
+ /** One parsed interlinear file per gloss language and book, in discovery order. */
+ interlinear: InterlinearData[];
+ lexicon?: LexiconData;
+ wordAnalyses?: WordAnalysesData;
+ setups?: InterlinearSetupsData;
+}
+
+/** Everything one conversion needs; conversion itself is a pure function of these inputs. */
+export interface Pt9ConversionInput {
+ data: Pt9ProjectData;
+ /** The source project's text layer, one entry per book. */
+ books: Book[];
+ /** Lexicon-extension resolution seam; defaults to resolving nothing. */
+ resolver?: Pt9LexiconResolver;
+ /** ISO 8601 stamp applied to every produced record and link. */
+ importedAt: string;
+}
+
+/** The converted analysis layer plus everything the import service persists and reports. */
+export interface Pt9ConversionResult {
+ analysis: TextAnalysis;
+ /** Resolved gloss-language tags in discovery order, one per distinct tag. */
+ analysisLanguages: string[];
+ report: Pt9ImportReport;
+ suggestedName: string;
+ suggestedDescription: string;
+}
+
+/** One gloss language's files and identity during conversion. */
+interface LanguageGroup {
+ raw: string;
+ tag: string;
+ files: InterlinearData[];
+ report: Pt9LanguageReport;
+}
+
+/**
+ * Converts a PT9 project's parsed interlinear data into the extension's analysis layer, paired with
+ * a report of what converted, what was dropped, and why.
+ *
+ * @throws {Error} If two interlinear files carry the same gloss language and book: one
+ * interlinearization per language and book is PT9's own file layout, so a duplicate means the
+ * caller assembled the input wrong.
+ */
+export function convertPt9Project(input: Pt9ConversionInput): Pt9ConversionResult {
+ const { data, books, importedAt } = input;
+ const resolver = input.resolver ?? unresolvedPt9LexiconResolver;
+ const report = emptyPt9ImportReport();
+
+ const seenFiles = new Set();
+ data.interlinear.forEach((file) => {
+ const fileKey = `${file.GlossLanguage}\n${file.BookId}`;
+ if (seenFiles.has(fileKey))
+ throw new Error(
+ `Duplicate interlinear data for language "${file.GlossLanguage}" book "${file.BookId}"`,
+ );
+ seenFiles.add(fileKey);
+ });
+
+ const languageGroups: LanguageGroup[] = [];
+ const groupByRaw = new Map();
+ data.interlinear.forEach((file) => {
+ let group = groupByRaw.get(file.GlossLanguage);
+ if (group === undefined) {
+ const resolved = resolveGlossLanguageTag(file.GlossLanguage);
+ group = {
+ raw: file.GlossLanguage,
+ tag: resolved.tag,
+ files: [],
+ report: {
+ rawLanguage: file.GlossLanguage,
+ tag: resolved.tag,
+ tagIsFallback: resolved.isFallback,
+ books: [],
+ },
+ };
+ languageGroups.push(group);
+ groupByRaw.set(file.GlossLanguage, group);
+ report.languages.push(group.report);
+ }
+ group.files.push(file);
+ });
+
+ const rawsByTag = new Map();
+ languageGroups.forEach((group) => {
+ const raws = rawsByTag.get(group.tag);
+ if (raws === undefined) rawsByTag.set(group.tag, [group.raw]);
+ else raws.push(group.raw);
+ });
+ rawsByTag.forEach((raws) => {
+ if (raws.length > 1) report.merge.sameTagCollisions.push(raws);
+ });
+
+ const booksByRef = new Map(books.map((book) => [book.bookRef, book]));
+ const glossSource = createPt9GlossSource(data.lexicon);
+
+ const records: LangTokenRecord[] = [];
+ const phrases: LangPhraseRecord[] = [];
+ languageGroups.forEach((group) => {
+ group.files.forEach((file) => {
+ const build = buildLanguageBookAnalyses({
+ file,
+ tag: group.tag,
+ book: booksByRef.get(file.BookId),
+ glossSource,
+ senses: report.senses,
+ });
+ group.report.books.push(build.bookReport);
+ records.push(...build.records);
+ phrases.push(...build.phrases);
+ });
+ });
+
+ const merged = mergeLanguageAnalyses({ records, phrases, resolver, importedAt, report });
+
+ const writingSystem =
+ books
+ .flatMap((book) => book.segments)
+ .flatMap((segment) => segment.tokens)
+ .find((token) => token.type === 'word')?.writingSystem ?? 'und';
+
+ const barePayloads = buildBareWordAnalyses({
+ wordAnalyses: data.wordAnalyses,
+ lexicon: data.lexicon,
+ languages: languageGroups.map((group) => ({ raw: group.raw, tag: group.tag })),
+ glossSource,
+ resolver,
+ clusterParseIdentities: merged.clusterParseIdentities,
+ writingSystem,
+ importedAt,
+ report,
+ });
+
+ const analysisLanguages: string[] = [];
+ languageGroups.forEach((group) => {
+ if (!analysisLanguages.includes(group.tag)) analysisLanguages.push(group.tag);
+ });
+
+ const scrTextName = data.interlinear.find((file) => file.ScrTextName !== undefined)?.ScrTextName;
+ const suggestedName = `${scrTextName ?? 'Paratext 9'} interlinear (${analysisLanguages.join(', ')})`;
+
+ const setupNotes = languageGroups.flatMap((group) => {
+ const setup = data.setups?.Setups.find((s) => s.LanguageId === group.raw);
+ if (setup === undefined) return [];
+ const model = setup.MdlScrTextName !== undefined ? ` (model ${setup.MdlScrTextName})` : '';
+ const exportTo = setup.ExportScrTextName !== undefined ? ` -> ${setup.ExportScrTextName}` : '';
+ return [`${group.raw}: ${setup.Type ?? 'unknown type'}${model}${exportTo}`];
+ });
+ const suggestedDescription = `Imported from Paratext 9 interlinear data.${
+ setupNotes.length > 0 ? ` Setups: ${setupNotes.join('; ')}.` : ''
+ }`;
+
+ const analysis: TextAnalysis = {
+ segmentAnalyses: [],
+ segmentAnalysisLinks: [],
+ tokenAnalyses: [...merged.tokenAnalyses, ...barePayloads],
+ tokenAnalysisLinks: merged.tokenAnalysisLinks,
+ phraseAnalyses: merged.phraseAnalyses,
+ phraseAnalysisLinks: merged.phraseAnalysisLinks,
+ };
+
+ return { analysis, analysisLanguages, report, suggestedName, suggestedDescription };
+}
diff --git a/src/converters/pt9/glossLanguageTags.ts b/src/converters/pt9/glossLanguageTags.ts
new file mode 100644
index 00000000..74f046a5
--- /dev/null
+++ b/src/converters/pt9/glossLanguageTags.ts
@@ -0,0 +1,38 @@
+/**
+ * Syntactic shape of a registered-language BCP 47 tag: a 2-3 letter primary language subtag
+ * followed by optional 1-8 character alphanumeric subtags. Deliberately excludes the reserved 4-8
+ * letter primary subtags, so legacy language names like `English` fall through to the verbatim
+ * fallback instead of masquerading as tags.
+ */
+const BCP47_RE = /^[a-z]{2,3}(-[a-z0-9]{1,8})*$/i;
+
+/** A resolved gloss-language identifier. */
+export interface ResolvedGlossLanguage {
+ /** The tag to key this language's glosses by. */
+ tag: string;
+ /** True when `tag` is the raw value passed through verbatim rather than a valid BCP 47 tag. */
+ isFallback: boolean;
+}
+
+/**
+ * Normalizes one subtag to conventional BCP 47 casing by position-independent shape: 4-letter
+ * subtags (scripts) are titlecased, 2-letter subtags (regions) are uppercased, everything else is
+ * lowercased.
+ */
+function normalizeSubtag(subtag: string, index: number): string {
+ const lower = subtag.toLowerCase();
+ if (index === 0) return lower;
+ if (/^[a-z]{4}$/.test(lower)) return lower[0].toUpperCase() + lower.slice(1);
+ if (/^[a-z]{2}$/.test(lower)) return lower.toUpperCase();
+ return lower;
+}
+
+/**
+ * Resolves a PT9 `GlossLanguage` value to the tag its glosses are keyed by: a syntactically valid
+ * BCP 47 value is case-normalized and used as-is; anything else (legacy language names like
+ * `English`) passes through verbatim and is flagged as a fallback.
+ */
+export function resolveGlossLanguageTag(rawLanguage: string): ResolvedGlossLanguage {
+ if (!BCP47_RE.test(rawLanguage)) return { tag: rawLanguage, isFallback: true };
+ return { tag: rawLanguage.split('-').map(normalizeSubtag).join('-'), isFallback: false };
+}
diff --git a/src/converters/pt9/index.ts b/src/converters/pt9/index.ts
new file mode 100644
index 00000000..f017b062
--- /dev/null
+++ b/src/converters/pt9/index.ts
@@ -0,0 +1,21 @@
+/**
+ * The PT9 interlinear converter's public surface: one function turning parsed PT9 files into the
+ * extension's analysis layer, the seam through which lexical identities resolve, and the report
+ * types describing what a conversion did. The types each stage hands the next are internal to the
+ * conversion and deliberately absent here.
+ */
+
+export { convertPt9Project } from './convertPt9Project';
+export type { Pt9ConversionInput, Pt9ConversionResult, Pt9ProjectData } from './convertPt9Project';
+
+export type { Pt9LexiconResolver } from './lexiconResolver';
+
+export type {
+ Pt9BarePayloadReport,
+ Pt9BookReport,
+ Pt9ClusterDropReason,
+ Pt9ImportReport,
+ Pt9LanguageReport,
+ Pt9MergeReport,
+ Pt9SenseReport,
+} from './report';
diff --git a/src/converters/pt9/languageAnalysisBuilder.ts b/src/converters/pt9/languageAnalysisBuilder.ts
new file mode 100644
index 00000000..d22a4961
--- /dev/null
+++ b/src/converters/pt9/languageAnalysisBuilder.ts
@@ -0,0 +1,155 @@
+import type { Book, Segment } from 'interlinearizer';
+import type { InterlinearData } from 'parsers/pt9/interlinearXmlParser';
+import { composeLexemeKeyId, LexemeKeyData } from 'parsers/pt9/lexemeKey';
+import { anchorVerseClusters, ClassifiedLexeme } from './clusterAnchoring';
+import type { Pt9GlossSource } from './pt9GlossSource';
+import { addClusterDrops, emptyBookReport, Pt9BookReport, Pt9SenseReport } from './report';
+
+/** A lexeme with its gloss resolved for one language: the facts merging needs, nothing more. */
+export interface ResolvedLexeme {
+ key: LexemeKeyData;
+ /** Composed key id, the identity two lexemes are compared by. */
+ keyId: string;
+ /**
+ * The effective sense: the cluster's explicit selection, or the default sense when the lexicon
+ * offers exactly one glossed sense in this language. Absent when neither applies.
+ */
+ senseId?: string;
+ /** Resolved gloss text for this language; absent when unresolvable. */
+ glossText?: string;
+}
+
+/** Review status a contribution carries before cross-language merging. */
+export type LangRecordStatus = 'approved' | 'suggested' | 'rejected';
+
+/** One language's analysis of one token: the unit the cross-language merger consumes. */
+export interface LangTokenRecord {
+ tag: string;
+ tokenRef: string;
+ tokenSurface: string;
+ tokenWritingSystem: string;
+ status: LangRecordStatus;
+ ambiguous: boolean;
+ word?: ResolvedLexeme;
+ parse?: { lexemes: ResolvedLexeme[]; signature: string };
+}
+
+/** One language's phrase selection, anchored to its token run. */
+export interface LangPhraseRecord {
+ tag: string;
+ phrase: ResolvedLexeme;
+ tokens: { ref: string; surface: string }[];
+ status: LangRecordStatus;
+ ambiguous: boolean;
+}
+
+/**
+ * Builds per-token and phrase contributions for one interlinear file against its book's text layer.
+ * Status follows the verse hash: present means approved, absent means suggested, and a record whose
+ * every anchored facet is excluded imports as rejected. Gloss resolution outcomes accumulate onto
+ * the shared sense report.
+ */
+export function buildLanguageBookAnalyses(args: {
+ file: InterlinearData;
+ /** Resolved tag the file's glosses are keyed by. */
+ tag: string;
+ /** The book's text layer; `undefined` when the source project has no text for this book. */
+ book: Book | undefined;
+ glossSource: Pt9GlossSource;
+ /** Shared sense-outcome counters, incremented in place. */
+ senses: Pt9SenseReport;
+}): { records: LangTokenRecord[]; phrases: LangPhraseRecord[]; bookReport: Pt9BookReport } {
+ const { file, tag, book, glossSource, senses } = args;
+ const rawLanguage = file.GlossLanguage;
+ const bookReport = emptyBookReport(file.BookId, book !== undefined);
+ const segmentById = new Map(
+ (book?.segments ?? []).map((segment) => [segment.id, segment]),
+ );
+
+ const resolveLexeme = (lexeme: ClassifiedLexeme): ResolvedLexeme => {
+ const outcome = glossSource.resolve(lexeme.key, lexeme.senseId, rawLanguage);
+ const resolved: ResolvedLexeme = {
+ key: lexeme.key,
+ keyId: composeLexemeKeyId(lexeme.key),
+ };
+ if (outcome.kind === 'none') {
+ senses.unresolvedGlossText += 1;
+ return resolved;
+ }
+ if (outcome.kind === 'specific') {
+ if (outcome.text === undefined) senses.unresolvedGlossText += 1;
+ else senses.specificResolved += 1;
+ return {
+ ...resolved,
+ senseId: outcome.senseId,
+ ...(outcome.text !== undefined && { glossText: outcome.text }),
+ };
+ }
+ senses.defaultSingleResolved += 1;
+ return {
+ ...resolved,
+ ...(outcome.senseId !== undefined && { senseId: outcome.senseId }),
+ glossText: outcome.text,
+ };
+ };
+
+ const records: LangTokenRecord[] = [];
+ const phrases: LangPhraseRecord[] = [];
+
+ Object.entries(file.Verses).forEach(([verseKey, verse]) => {
+ bookReport.versesTotal += 1;
+ if (verse.Hash !== undefined) bookReport.versesHashed += 1;
+ bookReport.punctuationEntriesIgnored += verse.Punctuations.length;
+ bookReport.clustersTotal += verse.Clusters.length;
+
+ const segment = segmentById.get(verseKey);
+ if (segment === undefined) {
+ bookReport.versesNotFound += 1;
+ bookReport.clusterDrops.verseNotFound += verse.Clusters.length;
+ return;
+ }
+
+ const baseStatus: LangRecordStatus = verse.Hash !== undefined ? 'approved' : 'suggested';
+ const anchored = anchorVerseClusters(segment, verse.Clusters);
+ addClusterDrops(bookReport.clusterDrops, anchored.dropCounts);
+ bookReport.ambiguousAnchors += anchored.ambiguousCount;
+
+ anchored.groups.forEach((group) => {
+ const excludedFacets: boolean[] = [];
+ if (group.word !== undefined) excludedFacets.push(group.word.excluded);
+ if (group.parse !== undefined) excludedFacets.push(group.parse.excluded);
+ // Every anchored group carries at least one facet, so every() never sees an empty array.
+ const status = excludedFacets.every(Boolean) ? 'rejected' : baseStatus;
+ bookReport.clustersConverted += excludedFacets.length;
+
+ records.push({
+ tag,
+ tokenRef: group.token.ref,
+ tokenSurface: group.token.surfaceText,
+ tokenWritingSystem: group.token.writingSystem,
+ status,
+ ambiguous: group.ambiguous,
+ ...(group.word !== undefined && { word: resolveLexeme(group.word.lexeme) }),
+ ...(group.parse !== undefined && {
+ parse: {
+ lexemes: group.parse.lexemes.map(resolveLexeme),
+ signature: group.parse.lexemes.map((l) => composeLexemeKeyId(l.key)).join('/'),
+ },
+ }),
+ });
+ });
+
+ anchored.phrases.forEach((phrase) => {
+ bookReport.phrasesConverted += 1;
+ phrases.push({
+ tag,
+ phrase: resolveLexeme(phrase.lexeme),
+ tokens: phrase.tokens.map((t) => ({ ref: t.ref, surface: t.surfaceText })),
+ status: phrase.excluded ? 'rejected' : baseStatus,
+ ambiguous: phrase.ambiguous,
+ });
+ });
+ });
+
+ return { records, phrases, bookReport };
+}
diff --git a/src/converters/pt9/lexiconResolver.ts b/src/converters/pt9/lexiconResolver.ts
new file mode 100644
index 00000000..96081387
--- /dev/null
+++ b/src/converters/pt9/lexiconResolver.ts
@@ -0,0 +1,31 @@
+import type { EntryRef, SenseRef } from 'interlinearizer';
+import type { LexemeKeyData } from 'parsers/pt9/lexemeKey';
+
+/**
+ * Resolves PT9 lexical identities to Lexicon-extension references.
+ *
+ * Returning `undefined` is a normal outcome rather than an error: an unresolved identity is stored
+ * with no reference at all, never a PT9-shaped one, leaving inlined gloss text as its only record.
+ *
+ * Lookups are synchronous, so an implementation backed by an asynchronous lexicon service must
+ * materialize its answers before conversion begins.
+ */
+export interface Pt9LexiconResolver {
+ /** Resolves a lexeme key to a Lexicon-extension entry, or `undefined` when unknown. */
+ resolveEntry(key: LexemeKeyData): EntryRef | undefined;
+
+ /**
+ * Resolves one sense of a lexeme to a Lexicon-extension sense, or `undefined` when unknown. PT9
+ * sense ids are only unique per entry, so the owning key travels with the id.
+ */
+ resolveSense(key: LexemeKeyData, senseId: string): SenseRef | undefined;
+}
+
+/**
+ * A resolver that resolves nothing, leaving every imported record with inlined gloss text and no
+ * lexicon references.
+ */
+export const unresolvedPt9LexiconResolver: Pt9LexiconResolver = {
+ resolveEntry: () => undefined,
+ resolveSense: () => undefined,
+};
diff --git a/src/converters/pt9/pt9GlossSource.ts b/src/converters/pt9/pt9GlossSource.ts
new file mode 100644
index 00000000..6ef4305b
--- /dev/null
+++ b/src/converters/pt9/pt9GlossSource.ts
@@ -0,0 +1,74 @@
+import type { LexiconData, LexiconSenseData } from 'parsers/pt9/lexiconXmlParser';
+import { composeLexemeKeyId, LexemeKeyData } from 'parsers/pt9/lexemeKey';
+
+/**
+ * How a lexeme's gloss resolved.
+ *
+ * - `specific`: the cluster carried an explicit sense selection. `text` is absent when the sense is
+ * missing from the lexicon (dangling) or its gloss for the language is empty.
+ * - `defaultSingle`: no selection, but the lexeme has exactly one sense with a non-empty gloss in the
+ * language, which is PT9's deterministic default. `senseId` is absent when that sense carries no
+ * id.
+ * - `none`: no selection and no single default; PT9's guessing among several glossed senses is never
+ * replicated.
+ */
+export type Pt9GlossOutcome =
+ | { kind: 'specific'; senseId: string; text?: string }
+ | { kind: 'defaultSingle'; senseId?: string; text: string }
+ | { kind: 'none' };
+
+/** Resolves gloss text for lexemes against one parsed PT9 lexicon. */
+export interface Pt9GlossSource {
+ /**
+ * Resolves the gloss for one lexeme in one gloss language. `senseId` is the cluster's selection;
+ * an absent or empty value means no selection (PT9 treats the two alike). `rawLanguage` is the
+ * file's GlossLanguage value as written; lexicon gloss languages are matched case-insensitively
+ * against it, never against resolved tags.
+ */
+ resolve(key: LexemeKeyData, senseId: string | undefined, rawLanguage: string): Pt9GlossOutcome;
+}
+
+/** The non-empty gloss text a sense carries for a language, or `undefined` when it has none. */
+function glossTextFor(sense: LexiconSenseData, rawLanguageFolded: string): string | undefined {
+ const gloss = sense.Glosses.find((g) => g.Language?.toLowerCase() === rawLanguageFolded);
+ if (gloss === undefined || gloss.Text === '') return undefined;
+ return gloss.Text;
+}
+
+/**
+ * Builds a {@link Pt9GlossSource} over a parsed lexicon. With no lexicon, every lookup with a
+ * selection resolves to a text-less `specific` outcome and every lookup without one to `none`.
+ */
+export function createPt9GlossSource(lexicon: LexiconData | undefined): Pt9GlossSource {
+ const entriesByKeyId = new Map(
+ (lexicon?.Entries ?? []).map((entry) => [composeLexemeKeyId(entry.Key), entry]),
+ );
+
+ return {
+ resolve(key, senseId, rawLanguage) {
+ const entry = entriesByKeyId.get(composeLexemeKeyId(key));
+ const rawFolded = rawLanguage.toLowerCase();
+
+ const selected = senseId !== undefined && senseId !== '' ? senseId : undefined;
+ if (selected !== undefined) {
+ const sense = entry?.Senses.find((s) => s.Id === selected);
+ if (sense === undefined) return { kind: 'specific', senseId: selected };
+ const text = glossTextFor(sense, rawFolded);
+ return { kind: 'specific', senseId: selected, ...(text !== undefined && { text }) };
+ }
+
+ if (entry === undefined) return { kind: 'none' };
+ const glossed = entry.Senses.flatMap((sense) => {
+ const text = glossTextFor(sense, rawFolded);
+ return text === undefined ? [] : [{ sense, text }];
+ });
+ if (glossed.length !== 1) return { kind: 'none' };
+ const [single] = glossed;
+ return {
+ kind: 'defaultSingle',
+ ...(single.sense.Id !== undefined && { senseId: single.sense.Id }),
+ text: single.text,
+ };
+ },
+ };
+}
diff --git a/src/converters/pt9/report.ts b/src/converters/pt9/report.ts
new file mode 100644
index 00000000..44546a34
--- /dev/null
+++ b/src/converters/pt9/report.ts
@@ -0,0 +1,165 @@
+/**
+ * Why a cluster could not be converted into an analysis record.
+ *
+ * - `verseNotFound`: the cluster's verse key has no matching segment (or its book is missing from the
+ * project text).
+ * - `formMismatch`: no token (or token run, for phrases) folds to the cluster's expected surface.
+ * Subsumes word-division and heading/footnote-scope disagreements, which are indistinguishable
+ * from plain mismatches without PT9's own text.
+ * - `lemmaOrOther`: the cluster is a Lemma or Other-type cluster, inert legacy data in modern PT9.
+ * - `duplicateCluster`: a second cluster of the same kind at the same range; only the first converts.
+ * - `unparseableLexemeId`: a lexeme id does not match PT9's `Type:Form[:Homograph]` grammar.
+ */
+export type Pt9ClusterDropReason =
+ 'verseNotFound' | 'formMismatch' | 'lemmaOrOther' | 'duplicateCluster' | 'unparseableLexemeId';
+
+/** Conversion outcome counts for one book of one gloss language. */
+export interface Pt9BookReport {
+ bookId: string;
+ /** False when the source project has no text for this book; every verse then drops. */
+ bookFound: boolean;
+ versesTotal: number;
+ /** Verses whose approval hash was present, so their records import as approved. */
+ versesHashed: number;
+ /** Verses whose key matched no segment in the book's text layer. */
+ versesNotFound: number;
+ clustersTotal: number;
+ /** Word and parse clusters that anchored and produced or enriched a token record. */
+ clustersConverted: number;
+ /** Phrase clusters that anchored and produced a phrase record. */
+ phrasesConverted: number;
+ clusterDrops: Record;
+ /** Anchors chosen among several candidate tokens rather than matched uniquely. */
+ ambiguousAnchors: number;
+ /**
+ * Punctuation entries that convert to nothing: they describe how a back translation replaces
+ * punctuation, which the analysis model has no place for.
+ */
+ punctuationEntriesIgnored: number;
+}
+
+/** Conversion outcome counts for one gloss language across its books. */
+export interface Pt9LanguageReport {
+ /** The GlossLanguage value as written in the files. */
+ rawLanguage: string;
+ /** The resolved BCP 47 tag that glosses in this language are keyed by. */
+ tag: string;
+ /** True when `rawLanguage` was not a valid tag and passed through verbatim. */
+ tagIsFallback: boolean;
+ books: Pt9BookReport[];
+}
+
+/** Cross-language merge outcomes. */
+export interface Pt9MergeReport {
+ /** Token records that carry contributions from more than one language. */
+ mergedTokenRecords: number;
+ /** Records created because parses genuinely conflicted at one token. */
+ parseConflicts: number;
+ /** Would-be-approved records demoted to candidate by the one-approved-per-token invariant. */
+ approvedDemotedToCandidate: number;
+ /** Raw language values that resolved onto one tag, grouped per collision. */
+ sameTagCollisions: string[][];
+}
+
+/** Gloss and lexicon-reference resolution outcomes. */
+export interface Pt9SenseReport {
+ /** Glosses resolved from an explicitly selected sense. */
+ specificResolved: number;
+ /** Glosses resolved through PT9's deterministic single-glossed-sense default. */
+ defaultSingleResolved: number;
+ /** Lexemes that resolved to no gloss text (dangling sense, empty gloss, or no default). */
+ unresolvedGlossText: number;
+ entryRefsResolved: number;
+ entryRefsUnresolved: number;
+ senseRefsResolved: number;
+ senseRefsUnresolved: number;
+}
+
+/** Bare word-analysis payload outcomes. */
+export interface Pt9BarePayloadReport {
+ added: number;
+ /** Analyses skipped because an identical cluster-derived record already exists. */
+ skippedExistingIdentical: number;
+ /** Analyses dropped because a lexeme id was unparseable. */
+ droppedUnparseable: number;
+ /** Analyses dropped because they carried no lexemes at all. */
+ droppedEmpty: number;
+}
+
+/** The import report returned to the caller: the quality signal for the whole conversion. */
+export interface Pt9ImportReport {
+ languages: Pt9LanguageReport[];
+ merge: Pt9MergeReport;
+ senses: Pt9SenseReport;
+ barePayloads: Pt9BarePayloadReport;
+}
+
+/** Every drop reason, for typed iteration over `clusterDrops` records. */
+const PT9_CLUSTER_DROP_REASONS: readonly Pt9ClusterDropReason[] = [
+ 'verseNotFound',
+ 'formMismatch',
+ 'lemmaOrOther',
+ 'duplicateCluster',
+ 'unparseableLexemeId',
+];
+
+/** Adds every count in `source` onto `target` in place. */
+export function addClusterDrops(
+ target: Record,
+ source: Record,
+): void {
+ PT9_CLUSTER_DROP_REASONS.forEach((reason) => {
+ target[reason] += source[reason];
+ });
+}
+
+/** A `clusterDrops` record with every reason at zero. */
+export function emptyClusterDrops(): Record {
+ return {
+ verseNotFound: 0,
+ formMismatch: 0,
+ lemmaOrOther: 0,
+ duplicateCluster: 0,
+ unparseableLexemeId: 0,
+ };
+}
+
+/** A book report with every count at zero. */
+export function emptyBookReport(bookId: string, bookFound: boolean): Pt9BookReport {
+ return {
+ bookId,
+ bookFound,
+ versesTotal: 0,
+ versesHashed: 0,
+ versesNotFound: 0,
+ clustersTotal: 0,
+ clustersConverted: 0,
+ phrasesConverted: 0,
+ clusterDrops: emptyClusterDrops(),
+ ambiguousAnchors: 0,
+ punctuationEntriesIgnored: 0,
+ };
+}
+
+/** An import report with every count at zero and no languages. */
+export function emptyPt9ImportReport(): Pt9ImportReport {
+ return {
+ languages: [],
+ merge: {
+ mergedTokenRecords: 0,
+ parseConflicts: 0,
+ approvedDemotedToCandidate: 0,
+ sameTagCollisions: [],
+ },
+ senses: {
+ specificResolved: 0,
+ defaultSingleResolved: 0,
+ unresolvedGlossText: 0,
+ entryRefsResolved: 0,
+ entryRefsUnresolved: 0,
+ senseRefsResolved: 0,
+ senseRefsUnresolved: 0,
+ },
+ barePayloads: { added: 0, skippedExistingIdentical: 0, droppedUnparseable: 0, droppedEmpty: 0 },
+ };
+}