From 01b5e87a91029b0445e85e8ecc251c8ad55363d8 Mon Sep 17 00:00:00 2001 From: Jason Naylor Date: Fri, 21 Aug 2026 14:34:21 -0700 Subject: [PATCH 1/2] Add the Paratext 9 import storage layer Add an optional pt9Import provenance field (per-file hashes and the import timestamp) to InterlinearProject, and add the storage contract for imports: savePt9Import creates the source project's single import record or replaces it wholesale, keeping only id and createdAt; updateAnalysis and updateProjectMetadata reject pt9Import-carrying projects while deleteProject stays allowed; getPt9ImportForSource finds the one import for a source. Co-Authored-By: Claude Fable 5 --- src/__tests__/services/projectStorage.test.ts | 187 ++++++++++++++++++ src/services/projectStorage.ts | 114 +++++++++++ src/types/interlinearizer.d.ts | 20 ++ 3 files changed, 321 insertions(+) diff --git a/src/__tests__/services/projectStorage.test.ts b/src/__tests__/services/projectStorage.test.ts index d21e2300..bb622e53 100644 --- a/src/__tests__/services/projectStorage.test.ts +++ b/src/__tests__/services/projectStorage.test.ts @@ -7,9 +7,11 @@ import { getDraft, getProject, getProjectsForSource, + getPt9ImportForSource, listProjects, resetQueuesForTesting, saveDraft, + savePt9Import, sweepPendingCleanup, updateAnalysis, updateProjectMetadata, @@ -1391,4 +1393,189 @@ describe('projectStorage', () => { expect(writeCallCount).toBe(2); }); }); + + describe('Paratext 9 import projects', () => { + const PT9_PROVENANCE = { + fileHashes: { 'Lexicon.xml': 'aaaa1111' }, + importedAt: '2026-08-01T00:00:00.000Z', + }; + const importedProject = { + ...makeStubProject('import-id'), + name: 'stale name', + description: 'stale description', + pt9Import: PT9_PROVENANCE, + }; + const SAVE_TIME = '2026-08-21T12:00:00.000Z'; + + /** Serves each record as the stored JSON for its key; any other key reads as never written. */ + function mockStore(records: Record): void { + __mockReadUserData.mockImplementation((_t: unknown, key: unknown) => { + if (typeof key === 'string' && key in records) + return Promise.resolve(JSON.stringify(records[key])); + return Promise.reject(enoentError()); + }); + } + + describe('freeze guard', () => { + it('rejects updateAnalysis without writing', async () => { + mockStore({ 'project:import-id': importedProject }); + + await expect(updateAnalysis(token, 'import-id', emptyAnalysis())).rejects.toThrow( + 'Paratext 9 import and is read-only', + ); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + + it('rejects updateProjectMetadata without writing', async () => { + mockStore({ 'project:import-id': importedProject }); + + await expect( + updateProjectMetadata(token, 'import-id', 'new name', undefined, ['en']), + ).rejects.toThrow('Paratext 9 import and is read-only'); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + + it('still allows deleteProject', async () => { + mockStore({ 'project:import-id': importedProject, projectIds: ['import-id'] }); + + await deleteProject(token, 'import-id'); + + expect(__mockDeleteUserData).toHaveBeenCalledWith(token, 'project:import-id'); + }); + }); + + describe('getPt9ImportForSource', () => { + it('returns the import among the source projects', async () => { + mockStore({ + projectIds: ['plain-id', 'import-id'], + 'project:plain-id': makeStubProject('plain-id'), + 'project:import-id': importedProject, + }); + + const result = await getPt9ImportForSource(token, 'src-project'); + + expect(result?.id).toBe('import-id'); + }); + + it('returns undefined when no source project carries pt9Import', async () => { + mockStore({ + projectIds: ['plain-id'], + 'project:plain-id': makeStubProject('plain-id'), + }); + + await expect(getPt9ImportForSource(token, 'src-project')).resolves.toBeUndefined(); + }); + }); + + describe('savePt9Import', () => { + const newAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'pt9:ta:GEN 1:1:0:0', surfaceText: 'hello' }], + }; + const NEW_PROVENANCE = { + fileHashes: { 'Lexicon.xml': 'bbbb2222' }, + importedAt: SAVE_TIME, + }; + + beforeEach(() => { + jest.useFakeTimers().setSystemTime(new Date(SAVE_TIME)); + }); + + it('creates the import and indexes it when the source has none', async () => { + mockStore({ projectIds: [] }); + + const project = await savePt9Import( + token, + 'src-project', + 'Paratext 9 Interlinear', + 'Imported from Paratext 9.', + ['en', 'fr'], + newAnalysis, + NEW_PROVENANCE, + ); + + expect(project).toMatchObject({ + id: '00000000-0000-0000-0000-000000000001', + createdAt: SAVE_TIME, + updatedAt: SAVE_TIME, + name: 'Paratext 9 Interlinear', + description: 'Imported from Paratext 9.', + sourceProjectId: 'src-project', + analysisLanguages: ['en', 'fr'], + analysis: newAnalysis, + pt9Import: NEW_PROVENANCE, + }); + expect(__mockWriteUserData).toHaveBeenCalledWith( + token, + 'projectIds', + JSON.stringify(['00000000-0000-0000-0000-000000000001']), + ); + }); + + it('replaces the existing import wholesale, keeping only id and createdAt', async () => { + mockStore({ + projectIds: ['import-id'], + 'project:import-id': { ...importedProject, targetProjectId: 'stray-target' }, + }); + + const project = await savePt9Import( + token, + 'src-project', + 'Paratext 9 Interlinear', + 'Imported from Paratext 9.', + ['en'], + newAnalysis, + NEW_PROVENANCE, + ); + + expect(project.id).toBe('import-id'); + expect(__mockWriteUserData).toHaveBeenCalledTimes(1); + expect(__mockWriteUserData).toHaveBeenCalledWith( + token, + 'project:import-id', + JSON.stringify({ + id: 'import-id', + modelVersion: CURRENT_MODEL_VERSION, + createdAt: importedProject.createdAt, + updatedAt: SAVE_TIME, + name: 'Paratext 9 Interlinear', + description: 'Imported from Paratext 9.', + sourceProjectId: 'src-project', + analysisLanguages: ['en'], + analysis: newAnalysis, + pt9Import: NEW_PROVENANCE, + }), + ); + }); + + it('creates a fresh record when the import is deleted between lookup and write', async () => { + let importReads = 0; + __mockReadUserData.mockImplementation((_t: unknown, key: unknown) => { + if (key === 'projectIds') return Promise.resolve(JSON.stringify(['import-id'])); + if (key === 'project:import-id') { + importReads += 1; + if (importReads === 1) return Promise.resolve(JSON.stringify(importedProject)); + } + return Promise.reject(enoentError()); + }); + + const project = await savePt9Import( + token, + 'src-project', + 'Paratext 9 Interlinear', + 'Imported from Paratext 9.', + ['en'], + newAnalysis, + NEW_PROVENANCE, + ); + + expect(project.id).toBe('00000000-0000-0000-0000-000000000001'); + expect(__mockWriteUserData).toHaveBeenCalledWith( + token, + 'project:00000000-0000-0000-0000-000000000001', + expect.stringContaining('"pt9Import"'), + ); + }); + }); + }); }); diff --git a/src/services/projectStorage.ts b/src/services/projectStorage.ts index 89b47ff7..7c71e838 100644 --- a/src/services/projectStorage.ts +++ b/src/services/projectStorage.ts @@ -147,6 +147,11 @@ function projectKey(id: string): string { return `project:${id}`; } +/** The error rejecting a user write to a Paratext 9 import, whose content only sync replaces. */ +function pt9ImportReadOnlyError(id: string): Error { + return new Error(`Project ${id} is a Paratext 9 import and is read-only`); +} + /** Returns the storage key for a source project's draft. */ function draftKey(sourceProjectId: string): string { return `draft:${sourceProjectId}`; @@ -386,6 +391,21 @@ export async function createProject( ...(targetProjectId !== undefined && { links: [] }), }; + await persistNewProject(token, project); + return project; +} + +/** + * Writes a newly built project record and appends its ID to the stored index, keeping the two in + * step: on an index-write failure the record is rolled back (or, failing that, recorded for a later + * {@link sweepPendingCleanup}) before the index error is rethrown, so no successful return leaves an + * indexed-but-missing or written-but-unindexed project behind. + */ +async function persistNewProject( + token: ExecutionToken, + project: InterlinearProject, +): Promise { + const { id } = project; await papi.storage.writeUserData(token, projectKey(id), JSON.stringify(project)); try { await enqueueIndexOp(async () => { @@ -406,7 +426,95 @@ export async function createProject( } throw indexError; } +} + +/** + * Reads the single Paratext 9 import project for the given source project, or `undefined` when the + * source has none. At most one project per source carries `pt9Import` (the import replaces it in + * place rather than creating another), so the first match is the only match. + * + * @throws {SyntaxError} If a project's storage value contains invalid JSON. + * @throws {Error} If a stored record was written by a newer build. + * @throws If `papi.storage.readUserData` rejects for any non-ENOENT reason. + */ +export async function getPt9ImportForSource( + token: ExecutionToken, + sourceProjectId: string, +): Promise { + const projects = await getProjectsForSource(token, sourceProjectId); + return projects.find((project) => project.pt9Import !== undefined); +} + +/** + * Persists the outcome of a Paratext 9 interlinear import: creates the source project's import + * record, or replaces it wholesale when one exists. This is the only write path for + * `pt9Import`-carrying projects - the public update functions reject them - and every field except + * `id` and `createdAt` is rebuilt from the arguments on replace, so nothing from the previous + * import survives a sync. + * + * @param token - The execution token for storage access. + * @param sourceProjectId - The Platform.Bible project ID the interlinear data was imported from. + * @param name - Fixed, already-localized project name the import stamps on every run. + * @param description - Fixed, already-localized project description, stamped like `name`. + * @param analysisLanguages - Resolved gloss-language tags from the conversion. + * @param analysis - The converted analysis layer. + * @param pt9Import - Provenance to store: per-file hashes and the import timestamp. + * @throws {SyntaxError} If a read storage value contains invalid JSON. + * @throws {Error} If a stored record was written by a newer build. + * @throws If `papi.storage.readUserData` or `papi.storage.writeUserData` rejects for a non-ENOENT + * reason. On a create, an index-write failure rolls back the record as in {@link createProject}. + */ +export async function savePt9Import( + token: ExecutionToken, + sourceProjectId: string, + name: string, + description: string, + analysisLanguages: string[], + analysis: TextAnalysis, + pt9Import: NonNullable, +): Promise { + const buildNew = (): InterlinearProject => { + const now = new Date().toISOString(); + return { + id: crypto.randomUUID(), + modelVersion: CURRENT_MODEL_VERSION, + createdAt: now, + updatedAt: now, + name, + description, + sourceProjectId, + analysisLanguages, + analysis, + pt9Import, + }; + }; + + const existing = await getPt9ImportForSource(token, sourceProjectId); + if (existing) { + const replaced = await enqueueProjectOp(existing.id, async () => { + const current = await getProject(token, existing.id); + if (!current) return undefined; + const updated: InterlinearProject = { + id: current.id, + modelVersion: CURRENT_MODEL_VERSION, + createdAt: current.createdAt, + updatedAt: new Date().toISOString(), + name, + description, + sourceProjectId, + analysisLanguages, + analysis, + pt9Import, + }; + await papi.storage.writeUserData(token, projectKey(current.id), JSON.stringify(updated)); + return updated; + }); + if (replaced) return replaced; + // The import was deleted between the lookup and the queued write; fall through and create. + } + const project = buildNew(); + await persistNewProject(token, project); return project; } @@ -519,6 +627,8 @@ export async function getProjectsForSource( * @returns The updated project record, or `undefined` if no project with the given ID exists. * @throws {SyntaxError} If the project's storage value contains invalid JSON. * @throws {Error} If the stored record was written by a newer build; nothing is written. + * @throws {Error} If the project is a Paratext 9 import (`pt9Import` present); imports are + * read-only and only {@link savePt9Import} replaces their content. * @throws If `papi.storage.readUserData` or `papi.storage.writeUserData` rejects for a non-ENOENT * reason. */ @@ -531,6 +641,7 @@ export async function updateAnalysis( return enqueueProjectOp(id, async () => { const project = await getProject(token, id); if (!project) return undefined; + if (project.pt9Import) throw pt9ImportReadOnlyError(id); const updated: InterlinearProject = { ...project, modelVersion: CURRENT_MODEL_VERSION, @@ -560,6 +671,8 @@ export async function updateAnalysis( * @returns The updated project record, or `undefined` if no project with the given ID exists. * @throws {SyntaxError} If the project's storage value contains invalid JSON. * @throws {Error} If the stored record was written by a newer build; nothing is written. + * @throws {Error} If the project is a Paratext 9 import (`pt9Import` present); imports are + * read-only, their name and description included - the import stamps fixed values. * @throws If `papi.storage.readUserData` or `papi.storage.writeUserData` rejects for a non-ENOENT * reason. */ @@ -574,6 +687,7 @@ export async function updateProjectMetadata( return enqueueProjectOp(id, async () => { const project = await getProject(token, id); if (!project) return undefined; + if (project.pt9Import) throw pt9ImportReadOnlyError(id); const updated: InterlinearProject = { ...project, modelVersion: CURRENT_MODEL_VERSION, diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index 08ec116c..ee8c81c6 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -1314,6 +1314,26 @@ declare module 'interlinearizer' { * {@link SegmentationDelta}. */ segmentation?: SegmentationDelta; + + /** + * Provenance for a project whose analysis was produced by the Paratext 9 interlinear import, + * rather than authored by the user. Present only on imported projects; absent for user-created + * ones. Its presence freezes the project - storage rejects every user write, and the view + * renders it read-only - and identifies it for sync: a repeat import for the same + * `sourceProjectId` replaces this project's analysis in place instead of creating a new + * project. At most one project per source carries this field. + */ + pt9Import?: { + /** + * SHA-256 hex of each imported source file, keyed by project-relative path, exactly as the + * source projectInterface reported it at import time. A later import compares a fresh + * manifest against this to decide whether the source changed since this import. + */ + fileHashes: Record; + + /** ISO 8601 timestamp of the import that produced the current analysis. */ + importedAt: string; + }; } /** From 4bd01f4c0e06ce46380d9eb40ab443a523be248a Mon Sep 17 00:00:00 2001 From: Jason Naylor Date: Fri, 21 Aug 2026 14:46:29 -0700 Subject: [PATCH 2/2] Add the Paratext 9 import service and commands Add pt9ImportService.importPt9Project: fetch the source project's PT9 interlinear files through the platformScripture.Pt9Interlinear projectInterface, parse them, rebuild the text layer for each referenced book from the project's USJ, convert, and persist the outcome as the source's single frozen import project with a fixed localized name and description. An empty source aborts a first import and leaves an existing import unchanged with a warning. Register two commands: interlinearizer.importPt9Project (import or sync) and interlinearizer.createEditableCopy, which clones an import's analysis into a new editable project without import provenance. Co-Authored-By: Claude Fable 5 --- __mocks__/papi-backend.ts | 10 + contributions/localizedStrings.json | 8 +- src/__tests__/main.test.ts | 116 ++++++++ src/__tests__/services/projectStorage.test.ts | 53 ++++ .../services/pt9ImportService.test.ts | 254 ++++++++++++++++++ src/main.ts | 135 ++++++++++ src/services/projectStorage.ts | 43 +++ src/services/pt9ImportService.ts | 133 +++++++++ src/types/interlinearizer.d.ts | 36 +++ user-questions.md | 10 + 10 files changed, 797 insertions(+), 1 deletion(-) create mode 100644 src/__tests__/services/pt9ImportService.test.ts create mode 100644 src/services/pt9ImportService.ts diff --git a/__mocks__/papi-backend.ts b/__mocks__/papi-backend.ts index 38c2eba9..26bfef0b 100644 --- a/__mocks__/papi-backend.ts +++ b/__mocks__/papi-backend.ts @@ -15,6 +15,8 @@ const mockReadUserData = jest.fn(); const mockWriteUserData = jest.fn(); const mockDeleteUserData = jest.fn(); const mockNotificationsSend = jest.fn(); +const mockProjectDataProvidersGet = jest.fn(); +const mockGetLocalizedString = jest.fn(); const mockLogger = { debug: jest.fn(), error: jest.fn(), @@ -35,6 +37,12 @@ const papi = { notifications: { send: mockNotificationsSend, }, + projectDataProviders: { + get: mockProjectDataProvidersGet, + }, + localization: { + getLocalizedString: mockGetLocalizedString, + }, storage: { readUserData: mockReadUserData, writeUserData: mockWriteUserData, @@ -69,6 +77,8 @@ const defaultExport = { __mockWriteUserData: mockWriteUserData, __mockDeleteUserData: mockDeleteUserData, __mockNotificationsSend: mockNotificationsSend, + __mockProjectDataProvidersGet: mockProjectDataProvidersGet, + __mockGetLocalizedString: mockGetLocalizedString, __mockLogger: mockLogger, }; diff --git a/contributions/localizedStrings.json b/contributions/localizedStrings.json index dd7c949c..0663350e 100644 --- a/contributions/localizedStrings.json +++ b/contributions/localizedStrings.json @@ -142,7 +142,13 @@ "%interlinearizer_error_update_project_failed%": "Could not update the interlinearizer project. Please try again.", "%interlinearizer_error_load_projects_failed%": "Could not load interlinear projects. Please try again.", "%interlinearizer_error_save_draft_failed%": "Could not save your working draft. Please try again.", - "%interlinearizer_error_save_analysis_failed%": "Could not save your analysis. Please try again." + "%interlinearizer_error_save_analysis_failed%": "Could not save your analysis. Please try again.", + + "%interlinearizer_pt9Import_name%": "Paratext 9 Interlinear", + "%interlinearizer_pt9Import_description%": "Imported from this project's Paratext 9 interlinear data. Read-only; synced from the Paratext 9 files.", + "%interlinearizer_error_pt9Import_failed%": "Could not import the Paratext 9 interlinear data. Please try again.", + "%interlinearizer_warning_pt9Import_sourceEmpty%": "The project's Paratext 9 interlinear files are missing, so the imported data was left as it was.", + "%interlinearizer_error_createEditableCopy_failed%": "Could not copy the imported project. Please try again." } } } diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index e4dd7abb..5be6f5c5 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -5,10 +5,12 @@ import papiBackendMock from '@papi/backend'; import { activate, deactivate } from '@main'; import type { InterlinearizerOpenOptions } from '@main'; import * as projectStorage from '../services/projectStorage'; +import * as pt9ImportService from '../services/pt9ImportService'; import { emptyAnalysis, emptyDraft } from '../types/empty-factories'; import { createTestActivationContext, makeStubProject } from './test-helpers'; jest.mock('../services/projectStorage'); +jest.mock('../services/pt9ImportService'); /** Shape of the Jest-mocked @papi/backend default export used in these tests. */ interface PapiBackendTestMock { @@ -144,6 +146,18 @@ const getUpdateProjectMetadataHandler = () => ) => Promise >('interlinearizer.updateProjectMetadata'); +/** Activates the extension and returns the `interlinearizer.importPt9Project` handler. */ +const getImportPt9ProjectHandler = () => + activateAndGetHandler<(sourceProjectId: string) => Promise>( + 'interlinearizer.importPt9Project', + ); + +/** Activates the extension and returns the `interlinearizer.createEditableCopy` handler. */ +const getCreateEditableCopyHandler = () => + activateAndGetHandler<(id: string, name: string, description?: string) => Promise>( + 'interlinearizer.createEditableCopy', + ); + /** Activates the extension and returns the `interlinearizer.getProject` handler. */ const getGetProjectHandler = () => activateAndGetHandler<(id: string) => Promise>('interlinearizer.getProject'); @@ -1151,4 +1165,106 @@ describe('main', () => { expect(__mockLogger.debug).toHaveBeenCalledWith('Interlinearizer extension is deactivating!'); }); }); + + describe('interlinearizer.importPt9Project command', () => { + const mockImport = jest.mocked(pt9ImportService.importPt9Project); + + it('registers the interlinearizer.importPt9Project command', async () => { + const context = createTestActivationContext(); + + await activate(context); + + expect(__mockRegisterCommand).toHaveBeenCalledWith( + 'interlinearizer.importPt9Project', + expect.any(Function), + expect.any(Object), + ); + }); + + it('returns the import result as JSON without a warning when data was imported', async () => { + mockImport.mockResolvedValue({ outcome: 'imported', projectId: 'import-id' }); + const handler = await getImportPt9ProjectHandler(); + + const result = await handler('src-project'); + + expect(mockImport).toHaveBeenCalledWith(expect.anything(), 'src-project'); + expect(JSON.parse(result)).toStrictEqual({ outcome: 'imported', projectId: 'import-id' }); + expect(__mockNotificationsSend).not.toHaveBeenCalled(); + }); + + it('sends a warning notification when the stored import was kept', async () => { + mockImport.mockResolvedValue({ outcome: 'staleKept', projectId: 'import-id' }); + const handler = await getImportPt9ProjectHandler(); + + const result = await handler('src-project'); + + expect(JSON.parse(result)).toStrictEqual({ outcome: 'staleKept', projectId: 'import-id' }); + expect(__mockNotificationsSend).toHaveBeenCalledWith({ + message: '%interlinearizer_warning_pt9Import_sourceEmpty%', + severity: 'warning', + }); + }); + + it('logs the error, sends an error notification, and rethrows when the import fails', async () => { + mockImport.mockRejectedValue(new Error('nothing to import')); + const handler = await getImportPt9ProjectHandler(); + + await expect(handler('src-project')).rejects.toThrow('nothing to import'); + expect(__mockLogger.error).toHaveBeenCalledWith( + 'Interlinearizer: failed to import Paratext 9 interlinear data', + expect.any(Error), + ); + expect(__mockNotificationsSend).toHaveBeenCalledWith({ + message: '%interlinearizer_error_pt9Import_failed%', + severity: 'error', + }); + }); + }); + + describe('interlinearizer.createEditableCopy command', () => { + const mockCopy = jest.mocked(projectStorage.createEditableCopy); + + it('registers the interlinearizer.createEditableCopy command', async () => { + const context = createTestActivationContext(); + + await activate(context); + + expect(__mockRegisterCommand).toHaveBeenCalledWith( + 'interlinearizer.createEditableCopy', + expect.any(Function), + expect.any(Object), + ); + }); + + it('returns the created copy as JSON', async () => { + const copy = { ...makeStubProject('copy-id'), name: 'My Copy' }; + mockCopy.mockResolvedValue(copy); + const handler = await getCreateEditableCopyHandler(); + + const result = await handler('import-id', 'My Copy', 'my description'); + + expect(mockCopy).toHaveBeenCalledWith( + expect.anything(), + 'import-id', + 'My Copy', + 'my description', + ); + expect(JSON.parse(result)).toStrictEqual(copy); + }); + + it('logs the error, sends an error notification, and rethrows when the copy fails', async () => { + mockCopy.mockRejectedValue(new Error('not a Paratext 9 import')); + const handler = await getCreateEditableCopyHandler(); + + await expect(handler('plain-id', 'My Copy')).rejects.toThrow('not a Paratext 9 import'); + expect(__mockLogger.error).toHaveBeenCalledWith( + 'Interlinearizer: failed to create an editable copy', + expect.any(Error), + ); + expect(__mockNotificationsSend).toHaveBeenCalledWith({ + message: '%interlinearizer_error_createEditableCopy_failed%', + severity: 'error', + }); + }); + }); }); diff --git a/src/__tests__/services/projectStorage.test.ts b/src/__tests__/services/projectStorage.test.ts index bb622e53..12dcfaef 100644 --- a/src/__tests__/services/projectStorage.test.ts +++ b/src/__tests__/services/projectStorage.test.ts @@ -2,6 +2,7 @@ import papiBackendMock from '@papi/backend'; import { + createEditableCopy, createProject, deleteProject, getDraft, @@ -1577,5 +1578,57 @@ describe('projectStorage', () => { ); }); }); + + describe('createEditableCopy', () => { + it('creates an editable project carrying the analysis and no pt9Import', async () => { + mockStore({ + projectIds: ['import-id'], + 'project:import-id': importedProject, + }); + + const copy = await createEditableCopy(token, 'import-id', 'My Copy', 'my description'); + + expect(copy).toMatchObject({ + id: '00000000-0000-0000-0000-000000000001', + name: 'My Copy', + description: 'my description', + sourceProjectId: importedProject.sourceProjectId, + analysisLanguages: importedProject.analysisLanguages, + analysis: importedProject.analysis, + }); + expect(copy).not.toHaveProperty('pt9Import'); + expect(__mockWriteUserData).toHaveBeenCalledWith( + token, + 'projectIds', + JSON.stringify(['import-id', '00000000-0000-0000-0000-000000000001']), + ); + }); + + it('omits the description when none is given', async () => { + mockStore({ projectIds: ['import-id'], 'project:import-id': importedProject }); + + const copy = await createEditableCopy(token, 'import-id', 'My Copy'); + + expect(copy).not.toHaveProperty('description'); + }); + + it('throws when the project does not exist', async () => { + mockStore({}); + + await expect(createEditableCopy(token, 'missing', 'My Copy')).rejects.toThrow( + 'does not exist', + ); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + + it('throws when the project is not a Paratext 9 import', async () => { + mockStore({ 'project:plain-id': makeStubProject('plain-id') }); + + await expect(createEditableCopy(token, 'plain-id', 'My Copy')).rejects.toThrow( + 'not a Paratext 9 import', + ); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + }); }); }); diff --git a/src/__tests__/services/pt9ImportService.test.ts b/src/__tests__/services/pt9ImportService.test.ts new file mode 100644 index 00000000..c3cf3525 --- /dev/null +++ b/src/__tests__/services/pt9ImportService.test.ts @@ -0,0 +1,254 @@ +/// + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import papiBackendMock from '@papi/backend'; +import { importPt9Project } from '../../services/pt9ImportService'; +import { resetQueuesForTesting } from '../../services/projectStorage'; +import { createTestActivationContext, makeStubProject } from '../test-helpers'; + +/** + * The backend-mock jest fns this suite drives: the PAPI boundary (project data providers, + * localization, storage) around the real parsers, converter, and storage module. + */ +interface BackendMock { + __mockProjectDataProvidersGet: jest.Mock; + __mockGetLocalizedString: jest.Mock; + __mockReadUserData: jest.Mock; + __mockWriteUserData: jest.Mock; + __mockDeleteUserData: jest.Mock; + __mockLogger: { debug: jest.Mock; error: jest.Mock; info: jest.Mock; warn: jest.Mock }; +} + +function isBackendMock(m: unknown): m is BackendMock { + return ( + !!m && + typeof m === 'object' && + '__mockProjectDataProvidersGet' in m && + '__mockGetLocalizedString' in m && + '__mockReadUserData' in m && + '__mockWriteUserData' in m + ); +} + +if (!isBackendMock(papiBackendMock)) throw new Error('Expected mocked @papi/backend'); +const { + __mockProjectDataProvidersGet, + __mockGetLocalizedString, + __mockReadUserData, + __mockWriteUserData, + __mockDeleteUserData, + __mockLogger, +} = papiBackendMock; + +const token = createTestActivationContext().executionToken; + +const IMPORT_TIME = '2026-08-21T15:00:00.000Z'; + +/** Reads one of the coherent PT9 fixtures the converter tests are built on. */ +function readFixture(name: string): string { + return fs.readFileSync(path.join(__dirname, '..', '..', '..', 'test-data', name), 'utf-8'); +} + +/** The fixture set as the projectInterface serves it: project-relative path to text and hash. */ +function fixtureFileSet(): Record { + return { + 'Interlinear_en/Interlinear_en_MAT.xml': { + text: readFixture('Interlinear_en_MAT.xml'), + sha256: 'hash-interlinear', + }, + 'Lexicon.xml': { text: readFixture('Lexicon.xml'), sha256: 'hash-lexicon' }, + 'WordAnalyses.xml': { text: readFixture('WordAnalyses.xml'), sha256: 'hash-word-analyses' }, + 'InterlinearSetup.xml': { text: readFixture('InterlinearSetup.xml'), sha256: 'hash-setup' }, + }; +} + +/** A USJ book whose verse texts match what the fixture interlinear data anchors against. */ +const MAT_USJ = { + content: [ + { type: 'book', code: 'MAT', content: [] }, + { type: 'chapter', number: '1', sid: 'MAT 1' }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'MAT 1:1', number: '1' }, + 'hello aokaybe abe abc this is a footnote with a note تمان oj', + { type: 'verse', sid: 'MAT 1:2', number: '2' }, + 'oooo dearly', + { type: 'verse', sid: 'MAT 1:9', number: '9' }, + 'hello', + ], + }, + ], +}; + +/** Serves fake PDPs for the three projectInterfaces the service consumes. */ +function mockPdps({ + files = fixtureFileSet(), + usj = MAT_USJ, + languageTag = 'en', +}: { + files?: Record; + usj?: unknown; + languageTag?: unknown; +} = {}): void { + __mockProjectDataProvidersGet.mockImplementation((projectInterface: unknown) => { + if (projectInterface === 'platformScripture.Pt9Interlinear') + return Promise.resolve({ getPt9InterlinearFiles: jest.fn().mockResolvedValue(files) }); + if (projectInterface === 'platformScripture.USJ_Book') + return Promise.resolve({ getBookUSJ: jest.fn().mockResolvedValue(usj) }); + return Promise.resolve({ getSetting: jest.fn().mockResolvedValue(languageTag) }); + }); +} + +/** The localized values the import resolves and stamps. */ +const LOCALIZED: Record = { + '%interlinearizer_pt9Import_name%': 'Paratext 9 Interlinear', + '%interlinearizer_pt9Import_description%': 'Imported from Paratext 9.', +}; + +/** Constructs the ENOENT error `papi.storage.readUserData` throws for a never-written key. */ +function enoentError(): Error { + return Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' }); +} + +/** The project record JSON written under a `project:` key, parsed; throws when none was written. */ +function writtenProject(): ReturnType { + const call = __mockWriteUserData.mock.calls.find( + (c: unknown[]) => typeof c[1] === 'string' && c[1].startsWith('project:'), + ); + if (!call || typeof call[2] !== 'string') throw new Error('Expected a project write'); + return JSON.parse(call[2]); +} + +describe('importPt9Project', () => { + beforeEach(() => { + resetQueuesForTesting(); + __mockReadUserData.mockRejectedValue(enoentError()); + __mockWriteUserData.mockResolvedValue(undefined); + __mockDeleteUserData.mockResolvedValue(undefined); + __mockGetLocalizedString.mockImplementation(({ localizeKey }: { localizeKey: string }) => + Promise.resolve(LOCALIZED[localizeKey] ?? localizeKey), + ); + mockPdps(); + jest.useFakeTimers().setSystemTime(new Date(IMPORT_TIME)); + jest.spyOn(crypto, 'randomUUID').mockReturnValue('00000000-0000-0000-0000-000000000001'); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('imports the fixture set end to end and persists the frozen project', async () => { + const result = await importPt9Project(token, 'src-project'); + + expect(result.outcome).toBe('imported'); + expect(result.projectId).toBe('00000000-0000-0000-0000-000000000001'); + const [language] = result.report?.languages ?? []; + expect(language.tag).toBe('en'); + expect(language.books[0]).toMatchObject({ + bookId: 'MAT', + bookFound: true, + versesTotal: 36, + clustersConverted: 24, + }); + + const project = writtenProject(); + expect(project).toMatchObject({ + name: 'Paratext 9 Interlinear', + description: 'Imported from Paratext 9.', + sourceProjectId: 'src-project', + analysisLanguages: ['en'], + pt9Import: { + importedAt: IMPORT_TIME, + fileHashes: { + 'Interlinear_en/Interlinear_en_MAT.xml': 'hash-interlinear', + 'Lexicon.xml': 'hash-lexicon', + 'WordAnalyses.xml': 'hash-word-analyses', + 'InterlinearSetup.xml': 'hash-setup', + }, + }, + }); + expect(project.analysis.tokenAnalyses).toHaveLength(18); + }); + + it('replaces the existing import on sync, keeping its id', async () => { + const existing = { + ...makeStubProject('import-id'), + pt9Import: { fileHashes: { 'Lexicon.xml': 'old' }, importedAt: '2026-08-01T00:00:00.000Z' }, + }; + __mockReadUserData.mockImplementation((_t: unknown, key: unknown) => { + if (key === 'projectIds') return Promise.resolve(JSON.stringify(['import-id'])); + if (key === 'project:import-id') return Promise.resolve(JSON.stringify(existing)); + return Promise.reject(enoentError()); + }); + + const result = await importPt9Project(token, 'src-project'); + + expect(result).toMatchObject({ outcome: 'imported', projectId: 'import-id' }); + }); + + it('skips a book the project has no USJ for and reports it missing', async () => { + // eslint-disable-next-line no-null/no-null -- null defeats the option's MAT_USJ default, which undefined would trigger + mockPdps({ usj: null }); + + const result = await importPt9Project(token, 'src-project'); + + expect(result.outcome).toBe('imported'); + expect(result.report?.languages[0].books[0].bookFound).toBe(false); + expect(__mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('no USJ for book MAT')); + }); + + it('aborts without writing when the source has no interlinear data and no import exists', async () => { + mockPdps({ files: {} }); + + await expect(importPt9Project(token, 'src-project')).rejects.toThrow( + 'no Paratext 9 interlinear data to import', + ); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + + it('keeps the stored import untouched when the source files have disappeared', async () => { + mockPdps({ files: {} }); + const existing = { + ...makeStubProject('import-id'), + pt9Import: { fileHashes: { 'Lexicon.xml': 'old' }, importedAt: '2026-08-01T00:00:00.000Z' }, + }; + __mockReadUserData.mockImplementation((_t: unknown, key: unknown) => { + if (key === 'projectIds') return Promise.resolve(JSON.stringify(['import-id'])); + if (key === 'project:import-id') return Promise.resolve(JSON.stringify(existing)); + return Promise.reject(enoentError()); + }); + + const result = await importPt9Project(token, 'src-project'); + + expect(result).toStrictEqual({ outcome: 'staleKept', projectId: 'import-id' }); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + expect(__mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('keeping the stored import'), + ); + }); + + it('falls back to the und writing system when the language tag setting is unavailable', async () => { + mockPdps({ languageTag: '' }); + + await importPt9Project(token, 'src-project'); + + const project = writtenProject(); + const bare = project.analysis.tokenAnalyses.find( + (a) => a.producer === 'pt9-import:word-analyses', + ); + expect(bare?.morphemes?.[0].writingSystem).toBe('und'); + }); + + it('propagates a parse failure without writing', async () => { + const files = fixtureFileSet(); + files['Lexicon.xml'] = { text: ' { + try { + const result = await pt9ImportService.importPt9Project(executionToken, sourceProjectId); + if (result.outcome === 'staleKept') { + await papi.notifications + .send({ + message: '%interlinearizer_warning_pt9Import_sourceEmpty%', + severity: 'warning', + }) + .catch(() => {}); + } + return JSON.stringify(result); + } catch (e) { + logger.error('Interlinearizer: failed to import Paratext 9 interlinear data', e); + await papi.notifications + .send({ + message: '%interlinearizer_error_pt9Import_failed%', + severity: 'error', + }) + .catch(() => {}); + throw e; + } +} + +/** + * Creates an editable copy of a Paratext 9 import project. Returns the created project as a JSON + * string. + * + * @param interlinearProjectId - UUID of the Paratext 9 import to copy. + * @param name - User-facing name for the copy, chosen in the copy dialog. + * @param description - Optional user-facing description for the copy. + * @throws If the project does not exist, is not a Paratext 9 import, or storage fails. The error is + * logged and an error notification is sent before rethrowing so the frontend `catch` block can + * suppress it without sending a second notification. + */ +async function createEditableCopy( + interlinearProjectId: string, + name: string, + description?: string, +): Promise { + try { + const project = await projectStorage.createEditableCopy( + executionToken, + interlinearProjectId, + name, + description, + ); + return JSON.stringify(project); + } catch (e) { + logger.error('Interlinearizer: failed to create an editable copy', e); + await papi.notifications + .send({ + message: '%interlinearizer_error_createEditableCopy_failed%', + severity: 'error', + }) + .catch(() => {}); + throw e; + } +} + /** * Loads the interlinearizer project with the given ID, including its full `TextAnalysis`. The * WebView calls this when the active project changes to load the stored analysis into the @@ -707,6 +778,68 @@ export async function activate(context: ExecutionActivationContext): Promise { + const source = await getProject(token, projectId); + if (!source) throw new Error(`Project ${projectId} does not exist`); + if (!source.pt9Import) + throw new Error(`Project ${projectId} is not a Paratext 9 import; it is already editable`); + + const now = new Date().toISOString(); + const project: InterlinearProject = { + id: crypto.randomUUID(), + modelVersion: CURRENT_MODEL_VERSION, + createdAt: now, + updatedAt: now, + name, + ...(description !== undefined && { description }), + sourceProjectId: source.sourceProjectId, + analysisLanguages: source.analysisLanguages, + analysis: source.analysis, + }; + await persistNewProject(token, project); + return project; +} + /** * A project as storage may actually hold it. The record type declares both times as required, which * a project written before it carried a modification time — or one damaged outside the extension — diff --git a/src/services/pt9ImportService.ts b/src/services/pt9ImportService.ts new file mode 100644 index 00000000..e6332f01 --- /dev/null +++ b/src/services/pt9ImportService.ts @@ -0,0 +1,133 @@ +import papi, { logger } from '@papi/backend'; +import type { ExecutionToken } from '@papi/core'; +import type { Book } from 'interlinearizer'; +import { InterlinearXmlParser } from 'parsers/pt9/interlinearXmlParser'; +import { InterlinearSetupXmlParser } from 'parsers/pt9/interlinearSetupXmlParser'; +import { LexiconXmlParser } from 'parsers/pt9/lexiconXmlParser'; +import { WordAnalysesXmlParser } from 'parsers/pt9/wordAnalysesXmlParser'; +import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; +import { tokenizeBook } from 'parsers/papi/bookTokenizer'; +import { convertPt9Project, Pt9ImportReport, Pt9ProjectData } from '../converters/pt9'; +import * as projectStorage from './projectStorage'; + +/** The outcome of one import run, returned to the caller as the command's JSON payload. */ +export interface Pt9ImportResult { + /** + * `imported` when a conversion ran and its outcome was persisted. `staleKept` when the source's + * interlinear files have disappeared while an earlier import exists: the stored import is left + * untouched rather than replaced with nothing, and only an explicit delete removes it. + */ + outcome: 'imported' | 'staleKept'; + + /** The id of the created, replaced, or kept import project. */ + projectId: string; + + /** The conversion's report; absent when no conversion ran (`staleKept`). */ + report?: Pt9ImportReport; +} + +/** + * Resolves the writing system tag for the source project's text, falling back to `und` when the + * project setting is unavailable. + */ +async function getWritingSystem(sourceProjectId: string): Promise { + const basePdp = await papi.projectDataProviders.get('platform.base', sourceProjectId); + const languageTag = await basePdp.getSetting('platform.languageTag'); + return typeof languageTag === 'string' && languageTag !== '' ? languageTag : 'und'; +} + +/** + * Imports the source project's Paratext 9 interlinear data into the extension's model, serving both + * first import and sync: fetches the raw files through the read-only Pt9Interlinear + * projectInterface, parses them, rebuilds the text layer for every book they reference from the + * project's USJ, converts, and persists the outcome as the source's single frozen import project - + * created on first run, replaced wholesale on later runs. The stored name and description are the + * fixed localized values, resolved at import time. + * + * A book the source project has no USJ for is skipped and counted in the report rather than failing + * the import. + * + * @throws {Error} If the source project has no Paratext 9 interlinear data and no earlier import + * exists - nothing is created for an empty source. + * @throws If a file fails to parse, the conversion rejects the input, or persistence fails. Nothing + * has been written unless persistence itself failed. + */ +export async function importPt9Project( + token: ExecutionToken, + sourceProjectId: string, +): Promise { + const pt9Pdp = await papi.projectDataProviders.get( + 'platformScripture.Pt9Interlinear', + sourceProjectId, + ); + const files = await pt9Pdp.getPt9InterlinearFiles(); + // Path-sorted for a deterministic parse order; gloss-language order follows it. + const paths = Object.keys(files).sort(); + + if (paths.length === 0) { + const existing = await projectStorage.getPt9ImportForSource(token, sourceProjectId); + if (existing) { + logger.warn( + `Interlinearizer: project ${sourceProjectId} has no Paratext 9 interlinear files; keeping the stored import ${existing.id} unchanged`, + ); + return { outcome: 'staleKept', projectId: existing.id }; + } + throw new Error(`Project ${sourceProjectId} has no Paratext 9 interlinear data to import`); + } + + const data: Pt9ProjectData = { interlinear: [] }; + paths.forEach((path) => { + const { text } = files[path]; + if (path === 'Lexicon.xml') data.lexicon = new LexiconXmlParser().parse(text); + else if (path === 'WordAnalyses.xml') + data.wordAnalyses = new WordAnalysesXmlParser().parse(text); + else if (path === 'InterlinearSetup.xml') + data.setups = new InterlinearSetupXmlParser().parse(text); + // The projectInterface serves only the four known patterns, so every other path is a + // per-language interlinear file. + else data.interlinear.push(new InterlinearXmlParser().parse(text)); + }); + + const bookIds = [...new Set(data.interlinear.map((file) => file.BookId))]; + const writingSystem = await getWritingSystem(sourceProjectId); + const usjPdp = await papi.projectDataProviders.get('platformScripture.USJ_Book', sourceProjectId); + const books: Book[] = ( + await Promise.all( + bookIds.map(async (bookId): Promise => { + const usj = await usjPdp.getBookUSJ({ book: bookId, chapterNum: 1, verseNum: 1 }); + if (!usj) { + logger.warn( + `Interlinearizer: project ${sourceProjectId} has no USJ for book ${bookId}; its interlinear data is skipped`, + ); + return []; + } + return [tokenizeBook(extractBookFromUsj(usj, writingSystem))]; + }), + ) + ).flat(); + + const importedAt = new Date().toISOString(); + const { analysis, analysisLanguages, report } = convertPt9Project({ data, books, importedAt }); + + const fileHashes = Object.fromEntries(paths.map((path) => [path, files[path].sha256])); + const [name, description] = await Promise.all([ + papi.localization.getLocalizedString({ localizeKey: '%interlinearizer_pt9Import_name%' }), + papi.localization.getLocalizedString({ + localizeKey: '%interlinearizer_pt9Import_description%', + }), + ]); + + const project = await projectStorage.savePt9Import( + token, + sourceProjectId, + name, + description, + analysisLanguages, + analysis, + { fileHashes, importedAt }, + ); + logger.info( + `Interlinearizer: imported Paratext 9 interlinear data from ${sourceProjectId} into ${project.id}`, + ); + return { outcome: 'imported', projectId: project.id, report }; +} diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index ee8c81c6..ae7f539d 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -222,6 +222,42 @@ declare module 'papi-shared-types' { analysisLanguages: string[], targetProjectId?: string, ) => Promise; + + /** + * Imports the source project's Paratext 9 interlinear data, serving both first import and sync: + * creates the source's single frozen import project, or replaces its content wholesale when one + * exists. When the source's interlinear files have disappeared but an earlier import exists, + * the stored import is kept unchanged instead. + * + * @param sourceProjectId - Platform.Bible project ID whose Paratext 9 interlinear files to + * import. + * @returns A JSON string of `{ outcome, projectId, report? }`: `outcome` is `'imported'` or + * `'staleKept'`, `projectId` the import project's id, and `report` the conversion report + * (present only when `outcome` is `'imported'`). + * @throws If the source has no Paratext 9 interlinear data and no earlier import exists, if a + * file fails to parse, or if persistence fails. The error is logged and an error notification + * is sent before rethrowing so callers do not need to send a second notification. + */ + 'interlinearizer.importPt9Project': (sourceProjectId: string) => Promise; + + /** + * Creates an editable project from a Paratext 9 import: a new project carrying the import's + * analysis verbatim and no import provenance, so it never syncs and is edited like any other + * project. The import itself is untouched. + * + * @param interlinearProjectId - UUID of the Paratext 9 import to copy. + * @param name - User-facing name for the copy, chosen in the copy dialog. + * @param description - Optional user-facing description for the copy. + * @returns The created project as a JSON string. + * @throws If the project does not exist or is not a Paratext 9 import, or if storage fails. The + * error is logged and an error notification is sent before rethrowing so callers do not need + * to send a second notification. + */ + 'interlinearizer.createEditableCopy': ( + interlinearProjectId: string, + name: string, + description?: string, + ) => Promise; } } diff --git a/user-questions.md b/user-questions.md index 593afaea..e954cda7 100644 --- a/user-questions.md +++ b/user-questions.md @@ -293,3 +293,13 @@ Decisions made during development that we'd like reviewed: protection was removed in favor of uniform, predictable behavior.) - When a heading is merged into a neighbor, its free translation follows the hide-and-restore behavior of item 2 — confirm that parallels hold for headings too. + +4. **Paratext 9 import: fixed project name and description.** A project imported from Paratext 9 + interlinear data is read-only, and its name and description are not editable; the import stamps + fixed, localizable values instead (there is at most one such import per source project, and the + project picker only lists projects for the current source, so the name never needs to + disambiguate between imports). Proposed wording, to be reviewed: name "Paratext 9 Interlinear"; + description "Imported from this project's Paratext 9 interlinear data. Read-only; synced from + the Paratext 9 files." Are these the right words, and are there localization concerns with + stamping the resolved string at import time (the label stays in the language the UI had when the + import ran, until a sync re-resolves it)?