diff --git a/__mocks__/papi-backend.ts b/__mocks__/papi-backend.ts index 38c2eba9..26bfef0b 100644 --- a/__mocks__/papi-backend.ts +++ b/__mocks__/papi-backend.ts @@ -15,6 +15,8 @@ const mockReadUserData = jest.fn(); const mockWriteUserData = jest.fn(); const mockDeleteUserData = jest.fn(); const mockNotificationsSend = jest.fn(); +const mockProjectDataProvidersGet = jest.fn(); +const mockGetLocalizedString = jest.fn(); const mockLogger = { debug: jest.fn(), error: jest.fn(), @@ -35,6 +37,12 @@ const papi = { notifications: { send: mockNotificationsSend, }, + projectDataProviders: { + get: mockProjectDataProvidersGet, + }, + localization: { + getLocalizedString: mockGetLocalizedString, + }, storage: { readUserData: mockReadUserData, writeUserData: mockWriteUserData, @@ -69,6 +77,8 @@ const defaultExport = { __mockWriteUserData: mockWriteUserData, __mockDeleteUserData: mockDeleteUserData, __mockNotificationsSend: mockNotificationsSend, + __mockProjectDataProvidersGet: mockProjectDataProvidersGet, + __mockGetLocalizedString: mockGetLocalizedString, __mockLogger: mockLogger, }; diff --git a/contributions/localizedStrings.json b/contributions/localizedStrings.json index dd7c949c..0663350e 100644 --- a/contributions/localizedStrings.json +++ b/contributions/localizedStrings.json @@ -142,7 +142,13 @@ "%interlinearizer_error_update_project_failed%": "Could not update the interlinearizer project. Please try again.", "%interlinearizer_error_load_projects_failed%": "Could not load interlinear projects. Please try again.", "%interlinearizer_error_save_draft_failed%": "Could not save your working draft. Please try again.", - "%interlinearizer_error_save_analysis_failed%": "Could not save your analysis. Please try again." + "%interlinearizer_error_save_analysis_failed%": "Could not save your analysis. Please try again.", + + "%interlinearizer_pt9Import_name%": "Paratext 9 Interlinear", + "%interlinearizer_pt9Import_description%": "Imported from this project's Paratext 9 interlinear data. Read-only; synced from the Paratext 9 files.", + "%interlinearizer_error_pt9Import_failed%": "Could not import the Paratext 9 interlinear data. Please try again.", + "%interlinearizer_warning_pt9Import_sourceEmpty%": "The project's Paratext 9 interlinear files are missing, so the imported data was left as it was.", + "%interlinearizer_error_createEditableCopy_failed%": "Could not copy the imported project. Please try again." } } } diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index e4dd7abb..5be6f5c5 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -5,10 +5,12 @@ import papiBackendMock from '@papi/backend'; import { activate, deactivate } from '@main'; import type { InterlinearizerOpenOptions } from '@main'; import * as projectStorage from '../services/projectStorage'; +import * as pt9ImportService from '../services/pt9ImportService'; import { emptyAnalysis, emptyDraft } from '../types/empty-factories'; import { createTestActivationContext, makeStubProject } from './test-helpers'; jest.mock('../services/projectStorage'); +jest.mock('../services/pt9ImportService'); /** Shape of the Jest-mocked @papi/backend default export used in these tests. */ interface PapiBackendTestMock { @@ -144,6 +146,18 @@ const getUpdateProjectMetadataHandler = () => ) => Promise >('interlinearizer.updateProjectMetadata'); +/** Activates the extension and returns the `interlinearizer.importPt9Project` handler. */ +const getImportPt9ProjectHandler = () => + activateAndGetHandler<(sourceProjectId: string) => Promise>( + 'interlinearizer.importPt9Project', + ); + +/** Activates the extension and returns the `interlinearizer.createEditableCopy` handler. */ +const getCreateEditableCopyHandler = () => + activateAndGetHandler<(id: string, name: string, description?: string) => Promise>( + 'interlinearizer.createEditableCopy', + ); + /** Activates the extension and returns the `interlinearizer.getProject` handler. */ const getGetProjectHandler = () => activateAndGetHandler<(id: string) => Promise>('interlinearizer.getProject'); @@ -1151,4 +1165,106 @@ describe('main', () => { expect(__mockLogger.debug).toHaveBeenCalledWith('Interlinearizer extension is deactivating!'); }); }); + + describe('interlinearizer.importPt9Project command', () => { + const mockImport = jest.mocked(pt9ImportService.importPt9Project); + + it('registers the interlinearizer.importPt9Project command', async () => { + const context = createTestActivationContext(); + + await activate(context); + + expect(__mockRegisterCommand).toHaveBeenCalledWith( + 'interlinearizer.importPt9Project', + expect.any(Function), + expect.any(Object), + ); + }); + + it('returns the import result as JSON without a warning when data was imported', async () => { + mockImport.mockResolvedValue({ outcome: 'imported', projectId: 'import-id' }); + const handler = await getImportPt9ProjectHandler(); + + const result = await handler('src-project'); + + expect(mockImport).toHaveBeenCalledWith(expect.anything(), 'src-project'); + expect(JSON.parse(result)).toStrictEqual({ outcome: 'imported', projectId: 'import-id' }); + expect(__mockNotificationsSend).not.toHaveBeenCalled(); + }); + + it('sends a warning notification when the stored import was kept', async () => { + mockImport.mockResolvedValue({ outcome: 'staleKept', projectId: 'import-id' }); + const handler = await getImportPt9ProjectHandler(); + + const result = await handler('src-project'); + + expect(JSON.parse(result)).toStrictEqual({ outcome: 'staleKept', projectId: 'import-id' }); + expect(__mockNotificationsSend).toHaveBeenCalledWith({ + message: '%interlinearizer_warning_pt9Import_sourceEmpty%', + severity: 'warning', + }); + }); + + it('logs the error, sends an error notification, and rethrows when the import fails', async () => { + mockImport.mockRejectedValue(new Error('nothing to import')); + const handler = await getImportPt9ProjectHandler(); + + await expect(handler('src-project')).rejects.toThrow('nothing to import'); + expect(__mockLogger.error).toHaveBeenCalledWith( + 'Interlinearizer: failed to import Paratext 9 interlinear data', + expect.any(Error), + ); + expect(__mockNotificationsSend).toHaveBeenCalledWith({ + message: '%interlinearizer_error_pt9Import_failed%', + severity: 'error', + }); + }); + }); + + describe('interlinearizer.createEditableCopy command', () => { + const mockCopy = jest.mocked(projectStorage.createEditableCopy); + + it('registers the interlinearizer.createEditableCopy command', async () => { + const context = createTestActivationContext(); + + await activate(context); + + expect(__mockRegisterCommand).toHaveBeenCalledWith( + 'interlinearizer.createEditableCopy', + expect.any(Function), + expect.any(Object), + ); + }); + + it('returns the created copy as JSON', async () => { + const copy = { ...makeStubProject('copy-id'), name: 'My Copy' }; + mockCopy.mockResolvedValue(copy); + const handler = await getCreateEditableCopyHandler(); + + const result = await handler('import-id', 'My Copy', 'my description'); + + expect(mockCopy).toHaveBeenCalledWith( + expect.anything(), + 'import-id', + 'My Copy', + 'my description', + ); + expect(JSON.parse(result)).toStrictEqual(copy); + }); + + it('logs the error, sends an error notification, and rethrows when the copy fails', async () => { + mockCopy.mockRejectedValue(new Error('not a Paratext 9 import')); + const handler = await getCreateEditableCopyHandler(); + + await expect(handler('plain-id', 'My Copy')).rejects.toThrow('not a Paratext 9 import'); + expect(__mockLogger.error).toHaveBeenCalledWith( + 'Interlinearizer: failed to create an editable copy', + expect.any(Error), + ); + expect(__mockNotificationsSend).toHaveBeenCalledWith({ + message: '%interlinearizer_error_createEditableCopy_failed%', + severity: 'error', + }); + }); + }); }); diff --git a/src/__tests__/services/projectStorage.test.ts b/src/__tests__/services/projectStorage.test.ts index d21e2300..12dcfaef 100644 --- a/src/__tests__/services/projectStorage.test.ts +++ b/src/__tests__/services/projectStorage.test.ts @@ -2,14 +2,17 @@ import papiBackendMock from '@papi/backend'; import { + createEditableCopy, createProject, deleteProject, getDraft, getProject, getProjectsForSource, + getPt9ImportForSource, listProjects, resetQueuesForTesting, saveDraft, + savePt9Import, sweepPendingCleanup, updateAnalysis, updateProjectMetadata, @@ -1391,4 +1394,241 @@ describe('projectStorage', () => { expect(writeCallCount).toBe(2); }); }); + + describe('Paratext 9 import projects', () => { + const PT9_PROVENANCE = { + fileHashes: { 'Lexicon.xml': 'aaaa1111' }, + importedAt: '2026-08-01T00:00:00.000Z', + }; + const importedProject = { + ...makeStubProject('import-id'), + name: 'stale name', + description: 'stale description', + pt9Import: PT9_PROVENANCE, + }; + const SAVE_TIME = '2026-08-21T12:00:00.000Z'; + + /** Serves each record as the stored JSON for its key; any other key reads as never written. */ + function mockStore(records: Record): void { + __mockReadUserData.mockImplementation((_t: unknown, key: unknown) => { + if (typeof key === 'string' && key in records) + return Promise.resolve(JSON.stringify(records[key])); + return Promise.reject(enoentError()); + }); + } + + describe('freeze guard', () => { + it('rejects updateAnalysis without writing', async () => { + mockStore({ 'project:import-id': importedProject }); + + await expect(updateAnalysis(token, 'import-id', emptyAnalysis())).rejects.toThrow( + 'Paratext 9 import and is read-only', + ); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + + it('rejects updateProjectMetadata without writing', async () => { + mockStore({ 'project:import-id': importedProject }); + + await expect( + updateProjectMetadata(token, 'import-id', 'new name', undefined, ['en']), + ).rejects.toThrow('Paratext 9 import and is read-only'); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + + it('still allows deleteProject', async () => { + mockStore({ 'project:import-id': importedProject, projectIds: ['import-id'] }); + + await deleteProject(token, 'import-id'); + + expect(__mockDeleteUserData).toHaveBeenCalledWith(token, 'project:import-id'); + }); + }); + + describe('getPt9ImportForSource', () => { + it('returns the import among the source projects', async () => { + mockStore({ + projectIds: ['plain-id', 'import-id'], + 'project:plain-id': makeStubProject('plain-id'), + 'project:import-id': importedProject, + }); + + const result = await getPt9ImportForSource(token, 'src-project'); + + expect(result?.id).toBe('import-id'); + }); + + it('returns undefined when no source project carries pt9Import', async () => { + mockStore({ + projectIds: ['plain-id'], + 'project:plain-id': makeStubProject('plain-id'), + }); + + await expect(getPt9ImportForSource(token, 'src-project')).resolves.toBeUndefined(); + }); + }); + + describe('savePt9Import', () => { + const newAnalysis = { + ...emptyAnalysis(), + tokenAnalyses: [{ ...FIXTURE_STAMPS, id: 'pt9:ta:GEN 1:1:0:0', surfaceText: 'hello' }], + }; + const NEW_PROVENANCE = { + fileHashes: { 'Lexicon.xml': 'bbbb2222' }, + importedAt: SAVE_TIME, + }; + + beforeEach(() => { + jest.useFakeTimers().setSystemTime(new Date(SAVE_TIME)); + }); + + it('creates the import and indexes it when the source has none', async () => { + mockStore({ projectIds: [] }); + + const project = await savePt9Import( + token, + 'src-project', + 'Paratext 9 Interlinear', + 'Imported from Paratext 9.', + ['en', 'fr'], + newAnalysis, + NEW_PROVENANCE, + ); + + expect(project).toMatchObject({ + id: '00000000-0000-0000-0000-000000000001', + createdAt: SAVE_TIME, + updatedAt: SAVE_TIME, + name: 'Paratext 9 Interlinear', + description: 'Imported from Paratext 9.', + sourceProjectId: 'src-project', + analysisLanguages: ['en', 'fr'], + analysis: newAnalysis, + pt9Import: NEW_PROVENANCE, + }); + expect(__mockWriteUserData).toHaveBeenCalledWith( + token, + 'projectIds', + JSON.stringify(['00000000-0000-0000-0000-000000000001']), + ); + }); + + it('replaces the existing import wholesale, keeping only id and createdAt', async () => { + mockStore({ + projectIds: ['import-id'], + 'project:import-id': { ...importedProject, targetProjectId: 'stray-target' }, + }); + + const project = await savePt9Import( + token, + 'src-project', + 'Paratext 9 Interlinear', + 'Imported from Paratext 9.', + ['en'], + newAnalysis, + NEW_PROVENANCE, + ); + + expect(project.id).toBe('import-id'); + expect(__mockWriteUserData).toHaveBeenCalledTimes(1); + expect(__mockWriteUserData).toHaveBeenCalledWith( + token, + 'project:import-id', + JSON.stringify({ + id: 'import-id', + modelVersion: CURRENT_MODEL_VERSION, + createdAt: importedProject.createdAt, + updatedAt: SAVE_TIME, + name: 'Paratext 9 Interlinear', + description: 'Imported from Paratext 9.', + sourceProjectId: 'src-project', + analysisLanguages: ['en'], + analysis: newAnalysis, + pt9Import: NEW_PROVENANCE, + }), + ); + }); + + it('creates a fresh record when the import is deleted between lookup and write', async () => { + let importReads = 0; + __mockReadUserData.mockImplementation((_t: unknown, key: unknown) => { + if (key === 'projectIds') return Promise.resolve(JSON.stringify(['import-id'])); + if (key === 'project:import-id') { + importReads += 1; + if (importReads === 1) return Promise.resolve(JSON.stringify(importedProject)); + } + return Promise.reject(enoentError()); + }); + + const project = await savePt9Import( + token, + 'src-project', + 'Paratext 9 Interlinear', + 'Imported from Paratext 9.', + ['en'], + newAnalysis, + NEW_PROVENANCE, + ); + + expect(project.id).toBe('00000000-0000-0000-0000-000000000001'); + expect(__mockWriteUserData).toHaveBeenCalledWith( + token, + 'project:00000000-0000-0000-0000-000000000001', + expect.stringContaining('"pt9Import"'), + ); + }); + }); + + describe('createEditableCopy', () => { + it('creates an editable project carrying the analysis and no pt9Import', async () => { + mockStore({ + projectIds: ['import-id'], + 'project:import-id': importedProject, + }); + + const copy = await createEditableCopy(token, 'import-id', 'My Copy', 'my description'); + + expect(copy).toMatchObject({ + id: '00000000-0000-0000-0000-000000000001', + name: 'My Copy', + description: 'my description', + sourceProjectId: importedProject.sourceProjectId, + analysisLanguages: importedProject.analysisLanguages, + analysis: importedProject.analysis, + }); + expect(copy).not.toHaveProperty('pt9Import'); + expect(__mockWriteUserData).toHaveBeenCalledWith( + token, + 'projectIds', + JSON.stringify(['import-id', '00000000-0000-0000-0000-000000000001']), + ); + }); + + it('omits the description when none is given', async () => { + mockStore({ projectIds: ['import-id'], 'project:import-id': importedProject }); + + const copy = await createEditableCopy(token, 'import-id', 'My Copy'); + + expect(copy).not.toHaveProperty('description'); + }); + + it('throws when the project does not exist', async () => { + mockStore({}); + + await expect(createEditableCopy(token, 'missing', 'My Copy')).rejects.toThrow( + 'does not exist', + ); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + + it('throws when the project is not a Paratext 9 import', async () => { + mockStore({ 'project:plain-id': makeStubProject('plain-id') }); + + await expect(createEditableCopy(token, 'plain-id', 'My Copy')).rejects.toThrow( + 'not a Paratext 9 import', + ); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + }); + }); }); diff --git a/src/__tests__/services/pt9ImportService.test.ts b/src/__tests__/services/pt9ImportService.test.ts new file mode 100644 index 00000000..c3cf3525 --- /dev/null +++ b/src/__tests__/services/pt9ImportService.test.ts @@ -0,0 +1,254 @@ +/// + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import papiBackendMock from '@papi/backend'; +import { importPt9Project } from '../../services/pt9ImportService'; +import { resetQueuesForTesting } from '../../services/projectStorage'; +import { createTestActivationContext, makeStubProject } from '../test-helpers'; + +/** + * The backend-mock jest fns this suite drives: the PAPI boundary (project data providers, + * localization, storage) around the real parsers, converter, and storage module. + */ +interface BackendMock { + __mockProjectDataProvidersGet: jest.Mock; + __mockGetLocalizedString: jest.Mock; + __mockReadUserData: jest.Mock; + __mockWriteUserData: jest.Mock; + __mockDeleteUserData: jest.Mock; + __mockLogger: { debug: jest.Mock; error: jest.Mock; info: jest.Mock; warn: jest.Mock }; +} + +function isBackendMock(m: unknown): m is BackendMock { + return ( + !!m && + typeof m === 'object' && + '__mockProjectDataProvidersGet' in m && + '__mockGetLocalizedString' in m && + '__mockReadUserData' in m && + '__mockWriteUserData' in m + ); +} + +if (!isBackendMock(papiBackendMock)) throw new Error('Expected mocked @papi/backend'); +const { + __mockProjectDataProvidersGet, + __mockGetLocalizedString, + __mockReadUserData, + __mockWriteUserData, + __mockDeleteUserData, + __mockLogger, +} = papiBackendMock; + +const token = createTestActivationContext().executionToken; + +const IMPORT_TIME = '2026-08-21T15:00:00.000Z'; + +/** Reads one of the coherent PT9 fixtures the converter tests are built on. */ +function readFixture(name: string): string { + return fs.readFileSync(path.join(__dirname, '..', '..', '..', 'test-data', name), 'utf-8'); +} + +/** The fixture set as the projectInterface serves it: project-relative path to text and hash. */ +function fixtureFileSet(): Record { + return { + 'Interlinear_en/Interlinear_en_MAT.xml': { + text: readFixture('Interlinear_en_MAT.xml'), + sha256: 'hash-interlinear', + }, + 'Lexicon.xml': { text: readFixture('Lexicon.xml'), sha256: 'hash-lexicon' }, + 'WordAnalyses.xml': { text: readFixture('WordAnalyses.xml'), sha256: 'hash-word-analyses' }, + 'InterlinearSetup.xml': { text: readFixture('InterlinearSetup.xml'), sha256: 'hash-setup' }, + }; +} + +/** A USJ book whose verse texts match what the fixture interlinear data anchors against. */ +const MAT_USJ = { + content: [ + { type: 'book', code: 'MAT', content: [] }, + { type: 'chapter', number: '1', sid: 'MAT 1' }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'MAT 1:1', number: '1' }, + 'hello aokaybe abe abc this is a footnote with a note تمان oj', + { type: 'verse', sid: 'MAT 1:2', number: '2' }, + 'oooo dearly', + { type: 'verse', sid: 'MAT 1:9', number: '9' }, + 'hello', + ], + }, + ], +}; + +/** Serves fake PDPs for the three projectInterfaces the service consumes. */ +function mockPdps({ + files = fixtureFileSet(), + usj = MAT_USJ, + languageTag = 'en', +}: { + files?: Record; + usj?: unknown; + languageTag?: unknown; +} = {}): void { + __mockProjectDataProvidersGet.mockImplementation((projectInterface: unknown) => { + if (projectInterface === 'platformScripture.Pt9Interlinear') + return Promise.resolve({ getPt9InterlinearFiles: jest.fn().mockResolvedValue(files) }); + if (projectInterface === 'platformScripture.USJ_Book') + return Promise.resolve({ getBookUSJ: jest.fn().mockResolvedValue(usj) }); + return Promise.resolve({ getSetting: jest.fn().mockResolvedValue(languageTag) }); + }); +} + +/** The localized values the import resolves and stamps. */ +const LOCALIZED: Record = { + '%interlinearizer_pt9Import_name%': 'Paratext 9 Interlinear', + '%interlinearizer_pt9Import_description%': 'Imported from Paratext 9.', +}; + +/** Constructs the ENOENT error `papi.storage.readUserData` throws for a never-written key. */ +function enoentError(): Error { + return Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' }); +} + +/** The project record JSON written under a `project:` key, parsed; throws when none was written. */ +function writtenProject(): ReturnType { + const call = __mockWriteUserData.mock.calls.find( + (c: unknown[]) => typeof c[1] === 'string' && c[1].startsWith('project:'), + ); + if (!call || typeof call[2] !== 'string') throw new Error('Expected a project write'); + return JSON.parse(call[2]); +} + +describe('importPt9Project', () => { + beforeEach(() => { + resetQueuesForTesting(); + __mockReadUserData.mockRejectedValue(enoentError()); + __mockWriteUserData.mockResolvedValue(undefined); + __mockDeleteUserData.mockResolvedValue(undefined); + __mockGetLocalizedString.mockImplementation(({ localizeKey }: { localizeKey: string }) => + Promise.resolve(LOCALIZED[localizeKey] ?? localizeKey), + ); + mockPdps(); + jest.useFakeTimers().setSystemTime(new Date(IMPORT_TIME)); + jest.spyOn(crypto, 'randomUUID').mockReturnValue('00000000-0000-0000-0000-000000000001'); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('imports the fixture set end to end and persists the frozen project', async () => { + const result = await importPt9Project(token, 'src-project'); + + expect(result.outcome).toBe('imported'); + expect(result.projectId).toBe('00000000-0000-0000-0000-000000000001'); + const [language] = result.report?.languages ?? []; + expect(language.tag).toBe('en'); + expect(language.books[0]).toMatchObject({ + bookId: 'MAT', + bookFound: true, + versesTotal: 36, + clustersConverted: 24, + }); + + const project = writtenProject(); + expect(project).toMatchObject({ + name: 'Paratext 9 Interlinear', + description: 'Imported from Paratext 9.', + sourceProjectId: 'src-project', + analysisLanguages: ['en'], + pt9Import: { + importedAt: IMPORT_TIME, + fileHashes: { + 'Interlinear_en/Interlinear_en_MAT.xml': 'hash-interlinear', + 'Lexicon.xml': 'hash-lexicon', + 'WordAnalyses.xml': 'hash-word-analyses', + 'InterlinearSetup.xml': 'hash-setup', + }, + }, + }); + expect(project.analysis.tokenAnalyses).toHaveLength(18); + }); + + it('replaces the existing import on sync, keeping its id', async () => { + const existing = { + ...makeStubProject('import-id'), + pt9Import: { fileHashes: { 'Lexicon.xml': 'old' }, importedAt: '2026-08-01T00:00:00.000Z' }, + }; + __mockReadUserData.mockImplementation((_t: unknown, key: unknown) => { + if (key === 'projectIds') return Promise.resolve(JSON.stringify(['import-id'])); + if (key === 'project:import-id') return Promise.resolve(JSON.stringify(existing)); + return Promise.reject(enoentError()); + }); + + const result = await importPt9Project(token, 'src-project'); + + expect(result).toMatchObject({ outcome: 'imported', projectId: 'import-id' }); + }); + + it('skips a book the project has no USJ for and reports it missing', async () => { + // eslint-disable-next-line no-null/no-null -- null defeats the option's MAT_USJ default, which undefined would trigger + mockPdps({ usj: null }); + + const result = await importPt9Project(token, 'src-project'); + + expect(result.outcome).toBe('imported'); + expect(result.report?.languages[0].books[0].bookFound).toBe(false); + expect(__mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('no USJ for book MAT')); + }); + + it('aborts without writing when the source has no interlinear data and no import exists', async () => { + mockPdps({ files: {} }); + + await expect(importPt9Project(token, 'src-project')).rejects.toThrow( + 'no Paratext 9 interlinear data to import', + ); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + }); + + it('keeps the stored import untouched when the source files have disappeared', async () => { + mockPdps({ files: {} }); + const existing = { + ...makeStubProject('import-id'), + pt9Import: { fileHashes: { 'Lexicon.xml': 'old' }, importedAt: '2026-08-01T00:00:00.000Z' }, + }; + __mockReadUserData.mockImplementation((_t: unknown, key: unknown) => { + if (key === 'projectIds') return Promise.resolve(JSON.stringify(['import-id'])); + if (key === 'project:import-id') return Promise.resolve(JSON.stringify(existing)); + return Promise.reject(enoentError()); + }); + + const result = await importPt9Project(token, 'src-project'); + + expect(result).toStrictEqual({ outcome: 'staleKept', projectId: 'import-id' }); + expect(__mockWriteUserData).not.toHaveBeenCalled(); + expect(__mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('keeping the stored import'), + ); + }); + + it('falls back to the und writing system when the language tag setting is unavailable', async () => { + mockPdps({ languageTag: '' }); + + await importPt9Project(token, 'src-project'); + + const project = writtenProject(); + const bare = project.analysis.tokenAnalyses.find( + (a) => a.producer === 'pt9-import:word-analyses', + ); + expect(bare?.morphemes?.[0].writingSystem).toBe('und'); + }); + + it('propagates a parse failure without writing', async () => { + const files = fixtureFileSet(); + files['Lexicon.xml'] = { text: ' { + try { + const result = await pt9ImportService.importPt9Project(executionToken, sourceProjectId); + if (result.outcome === 'staleKept') { + await papi.notifications + .send({ + message: '%interlinearizer_warning_pt9Import_sourceEmpty%', + severity: 'warning', + }) + .catch(() => {}); + } + return JSON.stringify(result); + } catch (e) { + logger.error('Interlinearizer: failed to import Paratext 9 interlinear data', e); + await papi.notifications + .send({ + message: '%interlinearizer_error_pt9Import_failed%', + severity: 'error', + }) + .catch(() => {}); + throw e; + } +} + +/** + * Creates an editable copy of a Paratext 9 import project. Returns the created project as a JSON + * string. + * + * @param interlinearProjectId - UUID of the Paratext 9 import to copy. + * @param name - User-facing name for the copy, chosen in the copy dialog. + * @param description - Optional user-facing description for the copy. + * @throws If the project does not exist, is not a Paratext 9 import, or storage fails. The error is + * logged and an error notification is sent before rethrowing so the frontend `catch` block can + * suppress it without sending a second notification. + */ +async function createEditableCopy( + interlinearProjectId: string, + name: string, + description?: string, +): Promise { + try { + const project = await projectStorage.createEditableCopy( + executionToken, + interlinearProjectId, + name, + description, + ); + return JSON.stringify(project); + } catch (e) { + logger.error('Interlinearizer: failed to create an editable copy', e); + await papi.notifications + .send({ + message: '%interlinearizer_error_createEditableCopy_failed%', + severity: 'error', + }) + .catch(() => {}); + throw e; + } +} + /** * Loads the interlinearizer project with the given ID, including its full `TextAnalysis`. The * WebView calls this when the active project changes to load the stored analysis into the @@ -707,6 +778,68 @@ export async function activate(context: ExecutionActivationContext): Promise { + const { id } = project; await papi.storage.writeUserData(token, projectKey(id), JSON.stringify(project)); try { await enqueueIndexOp(async () => { @@ -406,7 +426,138 @@ export async function createProject( } throw indexError; } +} + +/** + * Reads the single Paratext 9 import project for the given source project, or `undefined` when the + * source has none. At most one project per source carries `pt9Import` (the import replaces it in + * place rather than creating another), so the first match is the only match. + * + * @throws {SyntaxError} If a project's storage value contains invalid JSON. + * @throws {Error} If a stored record was written by a newer build. + * @throws If `papi.storage.readUserData` rejects for any non-ENOENT reason. + */ +export async function getPt9ImportForSource( + token: ExecutionToken, + sourceProjectId: string, +): Promise { + const projects = await getProjectsForSource(token, sourceProjectId); + return projects.find((project) => project.pt9Import !== undefined); +} + +/** + * Persists the outcome of a Paratext 9 interlinear import: creates the source project's import + * record, or replaces it wholesale when one exists. This is the only write path for + * `pt9Import`-carrying projects - the public update functions reject them - and every field except + * `id` and `createdAt` is rebuilt from the arguments on replace, so nothing from the previous + * import survives a sync. + * + * @param token - The execution token for storage access. + * @param sourceProjectId - The Platform.Bible project ID the interlinear data was imported from. + * @param name - Fixed, already-localized project name the import stamps on every run. + * @param description - Fixed, already-localized project description, stamped like `name`. + * @param analysisLanguages - Resolved gloss-language tags from the conversion. + * @param analysis - The converted analysis layer. + * @param pt9Import - Provenance to store: per-file hashes and the import timestamp. + * @throws {SyntaxError} If a read storage value contains invalid JSON. + * @throws {Error} If a stored record was written by a newer build. + * @throws If `papi.storage.readUserData` or `papi.storage.writeUserData` rejects for a non-ENOENT + * reason. On a create, an index-write failure rolls back the record as in {@link createProject}. + */ +export async function savePt9Import( + token: ExecutionToken, + sourceProjectId: string, + name: string, + description: string, + analysisLanguages: string[], + analysis: TextAnalysis, + pt9Import: NonNullable, +): Promise { + const buildNew = (): InterlinearProject => { + const now = new Date().toISOString(); + return { + id: crypto.randomUUID(), + modelVersion: CURRENT_MODEL_VERSION, + createdAt: now, + updatedAt: now, + name, + description, + sourceProjectId, + analysisLanguages, + analysis, + pt9Import, + }; + }; + + const existing = await getPt9ImportForSource(token, sourceProjectId); + if (existing) { + const replaced = await enqueueProjectOp(existing.id, async () => { + const current = await getProject(token, existing.id); + if (!current) return undefined; + const updated: InterlinearProject = { + id: current.id, + modelVersion: CURRENT_MODEL_VERSION, + createdAt: current.createdAt, + updatedAt: new Date().toISOString(), + name, + description, + sourceProjectId, + analysisLanguages, + analysis, + pt9Import, + }; + await papi.storage.writeUserData(token, projectKey(current.id), JSON.stringify(updated)); + return updated; + }); + if (replaced) return replaced; + // The import was deleted between the lookup and the queued write; fall through and create. + } + + const project = buildNew(); + await persistNewProject(token, project); + return project; +} + +/** + * Creates an editable project from a Paratext 9 import: a new record with a fresh id and timestamps + * carrying the import's analysis and analysis languages verbatim, the same source project, and no + * `pt9Import` - so the copy is an ordinary editable project that never syncs. The import itself is + * untouched. + * + * @param token - The execution token for storage access. + * @param projectId - The Paratext 9 import to copy. + * @param name - User-facing name for the copy, chosen in the copy dialog. + * @param description - Optional user-facing description for the copy. + * @throws {Error} If no project with the given ID exists, or if it is not a Paratext 9 import - + * only frozen imports need an editable copy; ordinary projects are already editable. + * @throws {SyntaxError} If the project's storage value contains invalid JSON. + * @throws If storage reads or writes reject for a non-ENOENT reason. On an index-write failure the + * new record rolls back as in {@link createProject}. + */ +export async function createEditableCopy( + token: ExecutionToken, + projectId: string, + name: string, + description?: string, +): Promise { + const source = await getProject(token, projectId); + if (!source) throw new Error(`Project ${projectId} does not exist`); + if (!source.pt9Import) + throw new Error(`Project ${projectId} is not a Paratext 9 import; it is already editable`); + const now = new Date().toISOString(); + const project: InterlinearProject = { + id: crypto.randomUUID(), + modelVersion: CURRENT_MODEL_VERSION, + createdAt: now, + updatedAt: now, + name, + ...(description !== undefined && { description }), + sourceProjectId: source.sourceProjectId, + analysisLanguages: source.analysisLanguages, + analysis: source.analysis, + }; + await persistNewProject(token, project); return project; } @@ -519,6 +670,8 @@ export async function getProjectsForSource( * @returns The updated project record, or `undefined` if no project with the given ID exists. * @throws {SyntaxError} If the project's storage value contains invalid JSON. * @throws {Error} If the stored record was written by a newer build; nothing is written. + * @throws {Error} If the project is a Paratext 9 import (`pt9Import` present); imports are + * read-only and only {@link savePt9Import} replaces their content. * @throws If `papi.storage.readUserData` or `papi.storage.writeUserData` rejects for a non-ENOENT * reason. */ @@ -531,6 +684,7 @@ export async function updateAnalysis( return enqueueProjectOp(id, async () => { const project = await getProject(token, id); if (!project) return undefined; + if (project.pt9Import) throw pt9ImportReadOnlyError(id); const updated: InterlinearProject = { ...project, modelVersion: CURRENT_MODEL_VERSION, @@ -560,6 +714,8 @@ export async function updateAnalysis( * @returns The updated project record, or `undefined` if no project with the given ID exists. * @throws {SyntaxError} If the project's storage value contains invalid JSON. * @throws {Error} If the stored record was written by a newer build; nothing is written. + * @throws {Error} If the project is a Paratext 9 import (`pt9Import` present); imports are + * read-only, their name and description included - the import stamps fixed values. * @throws If `papi.storage.readUserData` or `papi.storage.writeUserData` rejects for a non-ENOENT * reason. */ @@ -574,6 +730,7 @@ export async function updateProjectMetadata( return enqueueProjectOp(id, async () => { const project = await getProject(token, id); if (!project) return undefined; + if (project.pt9Import) throw pt9ImportReadOnlyError(id); const updated: InterlinearProject = { ...project, modelVersion: CURRENT_MODEL_VERSION, diff --git a/src/services/pt9ImportService.ts b/src/services/pt9ImportService.ts new file mode 100644 index 00000000..e6332f01 --- /dev/null +++ b/src/services/pt9ImportService.ts @@ -0,0 +1,133 @@ +import papi, { logger } from '@papi/backend'; +import type { ExecutionToken } from '@papi/core'; +import type { Book } from 'interlinearizer'; +import { InterlinearXmlParser } from 'parsers/pt9/interlinearXmlParser'; +import { InterlinearSetupXmlParser } from 'parsers/pt9/interlinearSetupXmlParser'; +import { LexiconXmlParser } from 'parsers/pt9/lexiconXmlParser'; +import { WordAnalysesXmlParser } from 'parsers/pt9/wordAnalysesXmlParser'; +import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; +import { tokenizeBook } from 'parsers/papi/bookTokenizer'; +import { convertPt9Project, Pt9ImportReport, Pt9ProjectData } from '../converters/pt9'; +import * as projectStorage from './projectStorage'; + +/** The outcome of one import run, returned to the caller as the command's JSON payload. */ +export interface Pt9ImportResult { + /** + * `imported` when a conversion ran and its outcome was persisted. `staleKept` when the source's + * interlinear files have disappeared while an earlier import exists: the stored import is left + * untouched rather than replaced with nothing, and only an explicit delete removes it. + */ + outcome: 'imported' | 'staleKept'; + + /** The id of the created, replaced, or kept import project. */ + projectId: string; + + /** The conversion's report; absent when no conversion ran (`staleKept`). */ + report?: Pt9ImportReport; +} + +/** + * Resolves the writing system tag for the source project's text, falling back to `und` when the + * project setting is unavailable. + */ +async function getWritingSystem(sourceProjectId: string): Promise { + const basePdp = await papi.projectDataProviders.get('platform.base', sourceProjectId); + const languageTag = await basePdp.getSetting('platform.languageTag'); + return typeof languageTag === 'string' && languageTag !== '' ? languageTag : 'und'; +} + +/** + * Imports the source project's Paratext 9 interlinear data into the extension's model, serving both + * first import and sync: fetches the raw files through the read-only Pt9Interlinear + * projectInterface, parses them, rebuilds the text layer for every book they reference from the + * project's USJ, converts, and persists the outcome as the source's single frozen import project - + * created on first run, replaced wholesale on later runs. The stored name and description are the + * fixed localized values, resolved at import time. + * + * A book the source project has no USJ for is skipped and counted in the report rather than failing + * the import. + * + * @throws {Error} If the source project has no Paratext 9 interlinear data and no earlier import + * exists - nothing is created for an empty source. + * @throws If a file fails to parse, the conversion rejects the input, or persistence fails. Nothing + * has been written unless persistence itself failed. + */ +export async function importPt9Project( + token: ExecutionToken, + sourceProjectId: string, +): Promise { + const pt9Pdp = await papi.projectDataProviders.get( + 'platformScripture.Pt9Interlinear', + sourceProjectId, + ); + const files = await pt9Pdp.getPt9InterlinearFiles(); + // Path-sorted for a deterministic parse order; gloss-language order follows it. + const paths = Object.keys(files).sort(); + + if (paths.length === 0) { + const existing = await projectStorage.getPt9ImportForSource(token, sourceProjectId); + if (existing) { + logger.warn( + `Interlinearizer: project ${sourceProjectId} has no Paratext 9 interlinear files; keeping the stored import ${existing.id} unchanged`, + ); + return { outcome: 'staleKept', projectId: existing.id }; + } + throw new Error(`Project ${sourceProjectId} has no Paratext 9 interlinear data to import`); + } + + const data: Pt9ProjectData = { interlinear: [] }; + paths.forEach((path) => { + const { text } = files[path]; + if (path === 'Lexicon.xml') data.lexicon = new LexiconXmlParser().parse(text); + else if (path === 'WordAnalyses.xml') + data.wordAnalyses = new WordAnalysesXmlParser().parse(text); + else if (path === 'InterlinearSetup.xml') + data.setups = new InterlinearSetupXmlParser().parse(text); + // The projectInterface serves only the four known patterns, so every other path is a + // per-language interlinear file. + else data.interlinear.push(new InterlinearXmlParser().parse(text)); + }); + + const bookIds = [...new Set(data.interlinear.map((file) => file.BookId))]; + const writingSystem = await getWritingSystem(sourceProjectId); + const usjPdp = await papi.projectDataProviders.get('platformScripture.USJ_Book', sourceProjectId); + const books: Book[] = ( + await Promise.all( + bookIds.map(async (bookId): Promise => { + const usj = await usjPdp.getBookUSJ({ book: bookId, chapterNum: 1, verseNum: 1 }); + if (!usj) { + logger.warn( + `Interlinearizer: project ${sourceProjectId} has no USJ for book ${bookId}; its interlinear data is skipped`, + ); + return []; + } + return [tokenizeBook(extractBookFromUsj(usj, writingSystem))]; + }), + ) + ).flat(); + + const importedAt = new Date().toISOString(); + const { analysis, analysisLanguages, report } = convertPt9Project({ data, books, importedAt }); + + const fileHashes = Object.fromEntries(paths.map((path) => [path, files[path].sha256])); + const [name, description] = await Promise.all([ + papi.localization.getLocalizedString({ localizeKey: '%interlinearizer_pt9Import_name%' }), + papi.localization.getLocalizedString({ + localizeKey: '%interlinearizer_pt9Import_description%', + }), + ]); + + const project = await projectStorage.savePt9Import( + token, + sourceProjectId, + name, + description, + analysisLanguages, + analysis, + { fileHashes, importedAt }, + ); + logger.info( + `Interlinearizer: imported Paratext 9 interlinear data from ${sourceProjectId} into ${project.id}`, + ); + return { outcome: 'imported', projectId: project.id, report }; +} diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index 08ec116c..ae7f539d 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -222,6 +222,42 @@ declare module 'papi-shared-types' { analysisLanguages: string[], targetProjectId?: string, ) => Promise; + + /** + * Imports the source project's Paratext 9 interlinear data, serving both first import and sync: + * creates the source's single frozen import project, or replaces its content wholesale when one + * exists. When the source's interlinear files have disappeared but an earlier import exists, + * the stored import is kept unchanged instead. + * + * @param sourceProjectId - Platform.Bible project ID whose Paratext 9 interlinear files to + * import. + * @returns A JSON string of `{ outcome, projectId, report? }`: `outcome` is `'imported'` or + * `'staleKept'`, `projectId` the import project's id, and `report` the conversion report + * (present only when `outcome` is `'imported'`). + * @throws If the source has no Paratext 9 interlinear data and no earlier import exists, if a + * file fails to parse, or if persistence fails. The error is logged and an error notification + * is sent before rethrowing so callers do not need to send a second notification. + */ + 'interlinearizer.importPt9Project': (sourceProjectId: string) => Promise; + + /** + * Creates an editable project from a Paratext 9 import: a new project carrying the import's + * analysis verbatim and no import provenance, so it never syncs and is edited like any other + * project. The import itself is untouched. + * + * @param interlinearProjectId - UUID of the Paratext 9 import to copy. + * @param name - User-facing name for the copy, chosen in the copy dialog. + * @param description - Optional user-facing description for the copy. + * @returns The created project as a JSON string. + * @throws If the project does not exist or is not a Paratext 9 import, or if storage fails. The + * error is logged and an error notification is sent before rethrowing so callers do not need + * to send a second notification. + */ + 'interlinearizer.createEditableCopy': ( + interlinearProjectId: string, + name: string, + description?: string, + ) => Promise; } } @@ -1314,6 +1350,26 @@ declare module 'interlinearizer' { * {@link SegmentationDelta}. */ segmentation?: SegmentationDelta; + + /** + * Provenance for a project whose analysis was produced by the Paratext 9 interlinear import, + * rather than authored by the user. Present only on imported projects; absent for user-created + * ones. Its presence freezes the project - storage rejects every user write, and the view + * renders it read-only - and identifies it for sync: a repeat import for the same + * `sourceProjectId` replaces this project's analysis in place instead of creating a new + * project. At most one project per source carries this field. + */ + pt9Import?: { + /** + * SHA-256 hex of each imported source file, keyed by project-relative path, exactly as the + * source projectInterface reported it at import time. A later import compares a fresh + * manifest against this to decide whether the source changed since this import. + */ + fileHashes: Record; + + /** ISO 8601 timestamp of the import that produced the current analysis. */ + importedAt: string; + }; } /** diff --git a/user-questions.md b/user-questions.md index 593afaea..e954cda7 100644 --- a/user-questions.md +++ b/user-questions.md @@ -293,3 +293,13 @@ Decisions made during development that we'd like reviewed: protection was removed in favor of uniform, predictable behavior.) - When a heading is merged into a neighbor, its free translation follows the hide-and-restore behavior of item 2 — confirm that parallels hold for headings too. + +4. **Paratext 9 import: fixed project name and description.** A project imported from Paratext 9 + interlinear data is read-only, and its name and description are not editable; the import stamps + fixed, localizable values instead (there is at most one such import per source project, and the + project picker only lists projects for the current source, so the name never needs to + disambiguate between imports). Proposed wording, to be reviewed: name "Paratext 9 Interlinear"; + description "Imported from this project's Paratext 9 interlinear data. Read-only; synced from + the Paratext 9 files." Are these the right words, and are there localization concerns with + stamping the resolved string at import time (the label stays in the language the UI had when the + import ran, until a sync re-resolves it)?