From f5070ee602615c9081eb8d65446eca17a43f11b7 Mon Sep 17 00:00:00 2001 From: yashin4112 Date: Mon, 8 Jun 2026 10:38:39 +0530 Subject: [PATCH 1/6] fix: prevent stale entry chunk files from clobbering current migration data API: - Add clearStaleEntries() to wipe the entries/ subtree before each (test) migration run, so orphaned chunk files from previous runs can't overwrite current entry data during the update step - removeEntriesFromDatabase now reads index.json to process only the current chunk files, falling back to globbing for legacy data - field-attacher: use customLogger instead of console.info for content type creation logs - runCli: switch to writeUidMapping util for uid mapping UI: - Remove the auto-mapped content mapping merge flow (AutoMappedMergeConfirmModal, persistAutoMappedContentMapper, handleUpdateAutoMappedContentMapping, shouldPromptShowAutoMappedMerge) and the related Auto-mapped status/pill constants --- api/src/services/migration.service.ts | 9 +- api/src/services/runCli.service.ts | 3 +- api/src/utils/entry-update.utils.ts | 50 ++- api/src/utils/field-attacher.utils.ts | 4 +- .../Common/SaveChangesModal/index.tsx | 12 +- .../ContentMapper/contentMapper.interface.ts | 2 - ui/src/components/ContentMapper/index.scss | 94 ++-- ui/src/components/ContentMapper/index.tsx | 418 +----------------- ui/src/context/app/app.interface.ts | 4 +- ui/src/pages/Migration/index.tsx | 52 +-- ui/src/utilities/constants.ts | 3 - ui/tests/unit/utilities/constants.test.ts | 1 - upload-api/.env | 2 + 13 files changed, 127 insertions(+), 527 deletions(-) create mode 100644 upload-api/.env diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 0298c6780..5d8605821 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -50,7 +50,7 @@ import { import { aemService } from './aem.service.js'; import { requestWithSsoTokenRefresh } from '../utils/sso-request.utils.js'; import { utilsUpdateCli } from './updateEntryCli.service.js'; -import { enrichConfigWithAssetMapping, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; +import { clearStaleEntries, enrichConfigWithAssetMapping, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; import { removeExistingAssets, saveAssetMetadata } from '../utils/asset-update.utils.js'; /** @@ -427,6 +427,9 @@ const startTestMigration = async (req: Request): Promise => { }; await copyLogsToTestStack(project?.current_test_stack_id, loggerPath); + // Clear any stale entries from a previous run before re-transforming, so orphaned + // chunk files cannot clobber this run's entry data during the update step. + clearStaleEntries(project?.current_test_stack_id, loggerPath); const contentTypes = await fieldAttacher({ orgId, projectId: safeTestProjectId, @@ -849,6 +852,10 @@ const startMigration = async (req: Request): Promise => { await copyLogsToStack(project?.destination_stack_id, loggerPath); + // Clear any stale entries from a previous run before re-transforming, so orphaned + // chunk files cannot clobber this run's entry data during the update step. + clearStaleEntries(project?.destination_stack_id, loggerPath); + const contentTypes = await fieldAttacher({ orgId, projectId: safeFinalProjectId, diff --git a/api/src/services/runCli.service.ts b/api/src/services/runCli.service.ts index dbb9dc3b1..c814117f8 100644 --- a/api/src/services/runCli.service.ts +++ b/api/src/services/runCli.service.ts @@ -19,8 +19,7 @@ interface TestStack { isMigrated: boolean; } import { setBasicAuthConfig, setOAuthConfig } from '../utils/config-handler.util.js'; -import getUidMapperDb from '../models/uidMapper.js'; -import customLogger from '../utils/custom-logger.utils.js'; +import writeUidMapping from '../utils/uid-mapper.utils.js'; /** * Determines log level based on message content without removing ANSI codes diff --git a/api/src/utils/entry-update.utils.ts b/api/src/utils/entry-update.utils.ts index 2677bba82..540f32cc8 100644 --- a/api/src/utils/entry-update.utils.ts +++ b/api/src/utils/entry-update.utils.ts @@ -3,6 +3,7 @@ import ProjectModelLowdb from "../models/project-lowdb.js"; import path from "path"; import fs from "node:fs"; import { MIGRATION_DATA_CONFIG, DATABASE_FILES } from "../constants/index.js"; +import { sanitizeStackId, assertResolvedPathUnderBase } from "./sanitize-path.utils.js"; /** * Helper function to write log entries to file @@ -19,6 +20,38 @@ const writeLogEntry = (message: string, methodName: string, loggerPath?: string) } }; +/** + * Deletes the transformed entries tree for a stack before a fresh import. + * + * Each import run writes entry chunk files with fresh random UUID names and overwrites + * index.json, but never removes the previous run's chunk files. Those orphans carry + * stale (previous-iteration) content that can later clobber current data during update. + * Wiping the entries tree up front guarantees the importer starts from a clean slate. + * + * Scope is limited to the `entries/` subtree only — assets, references, environments, + * locales and content-type creation (driven by the lowdb mappers, not this folder) are + * untouched. + */ +export const clearStaleEntries = (stackId: string, loggerPath?: string): void => { + const safeStackId = sanitizeStackId(stackId); + if (!safeStackId) { + writeLogEntry(`Invalid stackId, skipping stale entries cleanup.`, "clearStaleEntries", loggerPath); + return; + } + + const dataBase = path.resolve(process.cwd(), MIGRATION_DATA_CONFIG.DATA); + const entriesDir = path.join(dataBase, safeStackId, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME); + assertResolvedPathUnderBase(dataBase, entriesDir); + + if (!fs.existsSync(entriesDir)) { + writeLogEntry(`No existing entries directory to clear: ${entriesDir}`, "clearStaleEntries", loggerPath); + return; + } + + fs.rmSync(entriesDir, { recursive: true, force: true }); + writeLogEntry(`Cleared stale entries directory before import: ${entriesDir}`, "clearStaleEntries", loggerPath); +}; + export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: string): Promise => { const entriesToUpdate: Record> = {}; @@ -72,8 +105,21 @@ export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: for (const localeDir of localeDirs) { const localePath = path.join(ctPath, localeDir.name); - const jsonFiles = fs.readdirSync(localePath) - ?.filter((file) => file?.endsWith(".json") && file !== "index.json"); + // Respect index.json — only the chunk files it lists are current. Each import run + // writes chunk files with fresh random UUID names and overwrites index.json, but does + // not delete prior runs' chunk files. Globbing all *.json would pick up those orphans, + // whose stale (previous-iteration) content can then clobber the current entry data. + const indexPath = path.join(localePath, MIGRATION_DATA_CONFIG.ENTRIES_MASTER_FILE); + let jsonFiles: string[]; + if (fs.existsSync(indexPath)) { + const indexData = JSON.parse(fs.readFileSync(indexPath, "utf-8")); + jsonFiles = Object.values(indexData) + .filter((file): file is string => typeof file === "string" && file.endsWith(".json")); + } else { + // Legacy data without an index.json — fall back to globbing. + jsonFiles = fs.readdirSync(localePath) + ?.filter((file) => file?.endsWith(".json") && file !== MIGRATION_DATA_CONFIG.ENTRIES_MASTER_FILE); + } for (const jsonFile of jsonFiles) { const filePath = path.join(localePath, jsonFile); diff --git a/api/src/utils/field-attacher.utils.ts b/api/src/utils/field-attacher.utils.ts index cd77dc392..9daaf6598 100644 --- a/api/src/utils/field-attacher.utils.ts +++ b/api/src/utils/field-attacher.utils.ts @@ -4,6 +4,7 @@ import getFieldMapperDb from "../models/FieldMapper.js"; import { contenTypeMaker } from "./content-type-creator.utils.js"; import { shouldSkipContentTypeCreation } from "./content-type-checker.utils.js"; import { sanitizeProjectId } from "./sanitize-path.utils.js"; +import customLogger from "./custom-logger.utils.js"; export const fieldAttacher = async ({ projectId, orgId, destinationStackId, region, user_id, is_sso }: any) => { const safeProjectId = sanitizeProjectId(projectId); @@ -44,10 +45,11 @@ export const fieldAttacher = async ({ projectId, orgId, destinationStackId, regi else { const shouldSkip = await shouldSkipContentTypeCreation(safeProjectId, contentType?.otherCmsUid, iteration); if (!shouldSkip) { - console.info(`Creating new content type: ${contentType.otherCmsUid}`); + customLogger(`Creating new content type: ${contentType.otherCmsUid}`, safeProjectId, 'info', iteration ); await contenTypeMaker({ contentType, destinationStackId, projectId: safeProjectId, newStack: projectData?.stackDetails?.isNewStack, keyMapper: projectData?.mapperKeys, region, user_id, is_sso }) } else { console.info(`Skipping content type creation: ${contentType.otherCmsUid} (already exists from previous iteration)`); + customLogger(`Skipping content type creation: ${contentType.otherCmsUid} (already exists from previous iteration)`, safeProjectId, 'info', iteration ); } } contentTypes?.push?.(contentType); diff --git a/ui/src/components/Common/SaveChangesModal/index.tsx b/ui/src/components/Common/SaveChangesModal/index.tsx index 6c299da5e..67b3104d2 100644 --- a/ui/src/components/Common/SaveChangesModal/index.tsx +++ b/ui/src/components/Common/SaveChangesModal/index.tsx @@ -13,7 +13,7 @@ interface Props { otherCmsTitle?: string; saveContentType?: () => void; openContentType?: () => void; - changeStep?: () => void | Promise; + changeStep?: () => void; dropdownStateChange: () => void; } @@ -47,24 +47,24 @@ const SaveChangesModal = (props: Props) => { ))} @@ -3663,34 +3333,6 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref:
    {filteredContentTypes?.map?.((content: ContentType, index: number) => { const icon = STATUS_ICON_Mapping[content?.status] || ''; - const existingCTSide = asContentTypeListArray( - newMigrationData?.content_mapping?.existingCT - ); - const existingGlobalSide = asContentTypeListArray( - newMigrationData?.content_mapping?.existingGlobal - ); - let rowDestinationModels: ContentTypeList[] = getDestinationModelsForRow( - content, - existingCTSide, - existingGlobalSide - ); - if ( - !rowDestinationModels?.length && - contentModels?.length && - selectedContentType?.contentstackUid === content?.contentstackUid && - ((content?.type === 'content_type' && isContentType) || - (content?.type !== 'content_type' && !isContentType)) - ) { - rowDestinationModels = contentModels; - } - const showAutoMappedBadge = - !isNewStack && - isContentTypeAutoMapped( - content?.contentstackUid, - rowDestinationModels, - combinedContentTypeMapping, - uidAutoMapSuppressedForSourceUids - ); const format = (str: string) => { const frags = str?.split('_'); @@ -3728,38 +3370,14 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref:
    - - {showAutoMappedBadge && ( - - e.preventDefault()} - > - - - - )} + {icon && ( )} - + diff --git a/ui/src/context/app/app.interface.ts b/ui/src/context/app/app.interface.ts index 48c4e6ba7..dc34b0c00 100644 --- a/ui/src/context/app/app.interface.ts +++ b/ui/src/context/app/app.interface.ts @@ -191,8 +191,8 @@ export interface IDestinationStack { csLocale: string[]; } export interface IContentMapper { - existingGlobal: ContentTypeList[]; - existingCT: ContentTypeList[]; + existingGlobal: ContentTypeList[] | (() => ContentTypeList[]); + existingCT: ContentTypeList[] | (() => ContentTypeList[]); content_type_mapping: ContentTypeMap; isDropDownChanged?: boolean; otherCmsTitle?: string; diff --git a/ui/src/pages/Migration/index.tsx b/ui/src/pages/Migration/index.tsx index 3cfbc47a1..5658e2e0c 100644 --- a/ui/src/pages/Migration/index.tsx +++ b/ui/src/pages/Migration/index.tsx @@ -750,22 +750,6 @@ const Migration = () => { * Calls when click Continue button on Content Mapper step and handles to proceed to Test Migration */ const handleOnClickContentMapper = async (event: MouseEvent) => { - const persistAutoMappedContentMapper = async (): Promise => { - try { - await saveRef?.current?.handleUpdateAutoMappedContentMapping?.(); - return true; - } catch { - Notification({ - notificationContent: { - text: 'Could not save content type mapping. Please try again.' - }, - notificationProps: { position: 'bottom-center', hideProgressBar: true }, - type: 'error' - }); - return false; - } - }; - if (newMigrationData?.content_mapping?.isDropDownChanged) { setIsModalOpen(true); @@ -777,7 +761,6 @@ const Migration = () => { otherCmsTitle={newMigrationData?.content_mapping?.otherCmsTitle} saveContentType={saveRef?.current?.handleSaveContentType} changeStep={async () => { - if (!(await persistAutoMappedContentMapper())) return; const url = `/projects/${projectId}/migration/steps/4`; navigate(url, { replace: true }); @@ -793,35 +776,12 @@ const Migration = () => { } }); } else { - const finishContentMapperNavigation = async () => { - if (!(await persistAutoMappedContentMapper())) return; - await updateCurrentStepData(selectedOrganisation.value, projectId); - setIsLoading(false); - event?.preventDefault?.(); - handleStepChange(3); - const url = `/projects/${projectId}/migration/steps/4`; - navigate(url, { replace: true }); - }; - - if (saveRef?.current?.shouldPromptShowAutoMappedMerge?.()) { - return cbModal({ - component: (props: ModalObj) => ( - { - props.closeModal(); - await finishContentMapperNavigation(); - }} - /> - ), - modalProps: { - size: 'xsmall', - shouldCloseOnOverlayClick: false - } - }); - } - - await finishContentMapperNavigation(); + await updateCurrentStepData(selectedOrganisation.value, projectId); + setIsLoading(false); + event?.preventDefault?.(); + handleStepChange(3); + const url = `/projects/${projectId}/migration/steps/4`; + navigate(url, { replace: true }); } }; diff --git a/ui/src/utilities/constants.ts b/ui/src/utilities/constants.ts index 6f831b92e..04050d38d 100644 --- a/ui/src/utilities/constants.ts +++ b/ui/src/utilities/constants.ts @@ -106,7 +106,6 @@ export const CONTENT_MAPPING_STATUS: ObjectType = { '2': 'Updated', '3': 'Failed', '4': 'All', - '5': 'Auto-mapped', }; export const STATUS_ICON_Mapping: { [key: string]: string } = { '1': 'CheckedCircle', @@ -198,5 +197,3 @@ export const EXECUTION_LOGS_UI_TEXT = { export const EXECUTION_LOGS_ERROR_TEXT = { ERROR: 'Error in Getting Migration Logs' } - -export const AUTO_MAPPED_PILL_ITEMS = [{ id: 'auto-mapped', text: 'Auto-mapped' }]; \ No newline at end of file diff --git a/ui/tests/unit/utilities/constants.test.ts b/ui/tests/unit/utilities/constants.test.ts index 543c7f638..7df7686bf 100644 --- a/ui/tests/unit/utilities/constants.test.ts +++ b/ui/tests/unit/utilities/constants.test.ts @@ -111,7 +111,6 @@ describe('utilities/constants', () => { expect(CONTENT_MAPPING_STATUS['1']).toBe('Mapped'); expect(CONTENT_MAPPING_STATUS['2']).toBe('Updated'); expect(CONTENT_MAPPING_STATUS['3']).toBe('Failed'); - expect(CONTENT_MAPPING_STATUS['5']).toBe('Auto-mapped'); expect(CONTENT_MAPPING_STATUS['4']).toBe('All'); }); diff --git a/upload-api/.env b/upload-api/.env new file mode 100644 index 000000000..d86f70a1c --- /dev/null +++ b/upload-api/.env @@ -0,0 +1,2 @@ +PORT=4002 +NODE_BACKEND_API=http://localhost:5001 \ No newline at end of file From 29c31501eb729cebee98f315a49a2ccb4467f484 Mon Sep 17 00:00:00 2001 From: yashin4112 Date: Mon, 8 Jun 2026 10:43:46 +0530 Subject: [PATCH 2/6] test: add unit tests for writeUidMapping utility function --- api/tests/unit/utils/uid-mapper.utils.test.ts | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 api/tests/unit/utils/uid-mapper.utils.test.ts diff --git a/api/tests/unit/utils/uid-mapper.utils.test.ts b/api/tests/unit/utils/uid-mapper.utils.test.ts new file mode 100644 index 000000000..d4c2623e8 --- /dev/null +++ b/api/tests/unit/utils/uid-mapper.utils.test.ts @@ -0,0 +1,154 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const { + mockProjectRead, + mockChainGet, + mockExistsSync, + mockReadFileSync, + mockUidRead, + mockUidWrite, + mockGetUidMapperDb, + mockCustomLogger, +} = vi.hoisted(() => ({ + mockProjectRead: vi.fn(), + mockChainGet: vi.fn(), + mockExistsSync: vi.fn(), + mockReadFileSync: vi.fn(), + mockUidRead: vi.fn(), + mockUidWrite: vi.fn(), + mockGetUidMapperDb: vi.fn(), + mockCustomLogger: vi.fn(), +})); + +vi.mock('../../../src/models/project-lowdb.js', () => ({ + default: { + read: mockProjectRead, + chain: { get: mockChainGet }, + }, +})); + +vi.mock('../../../src/models/uidMapper.js', () => ({ + default: mockGetUidMapperDb, +})); + +vi.mock('../../../src/utils/custom-logger.utils.js', () => ({ + default: mockCustomLogger, +})); + +vi.mock('fs', () => ({ + default: { + existsSync: mockExistsSync, + readFileSync: mockReadFileSync, + }, +})); + +import writeUidMapping from '../../../src/utils/uid-mapper.utils'; + +describe('uid-mapper.utils - writeUidMapping', () => { + const uidDb = { + read: mockUidRead, + write: mockUidWrite, + data: {} as Record, + }; + + beforeEach(() => { + vi.clearAllMocks(); + mockProjectRead.mockResolvedValue(undefined); + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue({ + id: 'p1', + destination_stack_id: 'stack1', + }), + }), + }); + mockCustomLogger.mockResolvedValue(undefined); + mockUidRead.mockResolvedValue(undefined); + mockUidWrite.mockResolvedValue(undefined); + mockGetUidMapperDb.mockReturnValue(uidDb); + }); + + it('writes combined asset and entry mapping when both files have data', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockImplementation((p: string) => { + if (p.includes('assets')) return JSON.stringify({ a1: 'asset-uid' }); + return JSON.stringify({ e1: 'entry-uid' }); + }); + + await writeUidMapping('/backup', 'p1', 1); + + expect(mockGetUidMapperDb).toHaveBeenCalledWith('p1', 1); + expect(uidDb.data).toEqual({ + assets: { a1: 'asset-uid' }, + entry: { e1: 'entry-uid' }, + }); + expect(mockUidWrite).toHaveBeenCalled(); + }); + + it('does not write when entry mapper file is missing', async () => { + // asset file exists with data, entry file does not exist + mockExistsSync.mockImplementation((p: string) => p.includes('assets')); + mockReadFileSync.mockReturnValue(JSON.stringify({ a1: 'asset-uid' })); + + await writeUidMapping('/backup', 'p1', 1); + + expect(mockUidWrite).not.toHaveBeenCalled(); + }); + + it('falls back to previous iteration for assets when current asset data is empty', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockImplementation((p: string) => { + if (p.includes('mapper') && p.includes('assets')) return JSON.stringify({}); + if (p.includes('database')) return JSON.stringify({ assets: { prev: 'a' } }); + return JSON.stringify({ e1: 'entry-uid' }); + }); + + await writeUidMapping('/backup', 'p1', 2); + + expect(uidDb.data).toMatchObject({ assets: { prev: 'a' } }); + expect(mockUidWrite).toHaveBeenCalled(); + }); + + it('falls back to previous iteration for entries when current entry data is empty', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockImplementation((p: string) => { + if (p.includes('mapper') && p.includes('assets')) return JSON.stringify({ a1: 'x' }); + if (p.includes('mapper') && p.includes('entries')) return JSON.stringify({}); + if (p.includes('database')) return JSON.stringify({ entry: { prevE: 'e' } }); + return JSON.stringify({}); + }); + + await writeUidMapping('/backup', 'p1', 2); + + expect(uidDb.data).toMatchObject({ entry: { prevE: 'e' } }); + expect(mockUidWrite).toHaveBeenCalled(); + }); + + it('handles missing project data gracefully (undefined destination stack)', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(undefined), + }), + }); + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue(JSON.stringify({ k: 'v' })); + + await expect(writeUidMapping('/backup', 'p1', 1)).resolves.toBeUndefined(); + expect(mockUidWrite).toHaveBeenCalled(); + }); + + it('catches and logs errors without throwing', async () => { + const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockImplementation(() => { + throw new Error('boom'); + }); + + await expect(writeUidMapping('/backup', 'p1', 1)).resolves.toBeUndefined(); + expect(consoleSpy).toHaveBeenCalledWith( + 'Error writing UID mapping file:', + expect.any(Error), + ); + consoleSpy.mockRestore(); + }); +}); \ No newline at end of file From 3a5e9f573ec2a9a1c9647e1fa6402f6d5a68cc3f Mon Sep 17 00:00:00 2001 From: yashin4112 Date: Mon, 8 Jun 2026 11:08:25 +0530 Subject: [PATCH 3/6] fix: disable file format icon button in LoadFileFormat component to prevent it from editing --- ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx b/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx index 391886ab7..c9d524fbd 100644 --- a/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx +++ b/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx @@ -92,6 +92,7 @@ const LoadFileFormat = (_props: LoadFileFormatProps) => { aria-label="File format icon" /> } + disabled={true} />
    From 03a3683fad0c19e95311fa14ea38d8b086217fcc Mon Sep 17 00:00:00 2001 From: yashin4112 Date: Mon, 8 Jun 2026 12:21:24 +0530 Subject: [PATCH 4/6] fix: resolve review comments on entry-update, field-attacher, and SaveChangesModal --- api/src/utils/entry-update.utils.ts | 21 +- api/src/utils/field-attacher.utils.ts | 5 +- .../unit/utils/entry-update.utils.test.ts | 196 +++++++++++++++++- .../Common/SaveChangesModal/index.tsx | 14 +- 4 files changed, 218 insertions(+), 18 deletions(-) diff --git a/api/src/utils/entry-update.utils.ts b/api/src/utils/entry-update.utils.ts index 540f32cc8..489fc4c1b 100644 --- a/api/src/utils/entry-update.utils.ts +++ b/api/src/utils/entry-update.utils.ts @@ -112,9 +112,24 @@ export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: const indexPath = path.join(localePath, MIGRATION_DATA_CONFIG.ENTRIES_MASTER_FILE); let jsonFiles: string[]; if (fs.existsSync(indexPath)) { - const indexData = JSON.parse(fs.readFileSync(indexPath, "utf-8")); - jsonFiles = Object.values(indexData) - .filter((file): file is string => typeof file === "string" && file.endsWith(".json")); + // Guard against a missing/corrupt/non-object index.json — a parse failure or + // unexpected shape here would otherwise throw and abort the entire removal step. + let indexData: unknown; + try { + indexData = JSON.parse(fs.readFileSync(indexPath, "utf-8")); + } catch (err) { + writeLogEntry(`Failed to parse index.json at ${indexPath}, skipping locale: ${(err as Error)?.message}`, "removeEntriesFromDatabase", loggerPath); + continue; + } + if (!indexData || typeof indexData !== "object") { + writeLogEntry(`index.json at ${indexPath} is not an object, skipping locale.`, "removeEntriesFromDatabase", loggerPath); + continue; + } + jsonFiles = Object.values(indexData as Record) + // Sanitize to path.basename — index values are trusted verbatim otherwise, + // so an unexpected value could introduce extra path segments. + .filter((file): file is string => typeof file === "string" && file.endsWith(".json")) + .map((file) => path.basename(file)); } else { // Legacy data without an index.json — fall back to globbing. jsonFiles = fs.readdirSync(localePath) diff --git a/api/src/utils/field-attacher.utils.ts b/api/src/utils/field-attacher.utils.ts index 9daaf6598..2c84fdd91 100644 --- a/api/src/utils/field-attacher.utils.ts +++ b/api/src/utils/field-attacher.utils.ts @@ -45,11 +45,10 @@ export const fieldAttacher = async ({ projectId, orgId, destinationStackId, regi else { const shouldSkip = await shouldSkipContentTypeCreation(safeProjectId, contentType?.otherCmsUid, iteration); if (!shouldSkip) { - customLogger(`Creating new content type: ${contentType.otherCmsUid}`, safeProjectId, 'info', iteration ); + await customLogger(safeProjectId, destinationStackId, 'info', `Creating new content type: ${contentType.otherCmsUid}`); await contenTypeMaker({ contentType, destinationStackId, projectId: safeProjectId, newStack: projectData?.stackDetails?.isNewStack, keyMapper: projectData?.mapperKeys, region, user_id, is_sso }) } else { - console.info(`Skipping content type creation: ${contentType.otherCmsUid} (already exists from previous iteration)`); - customLogger(`Skipping content type creation: ${contentType.otherCmsUid} (already exists from previous iteration)`, safeProjectId, 'info', iteration ); + await customLogger(safeProjectId, destinationStackId, 'info', `Skipping content type creation: ${contentType.otherCmsUid} (already exists from previous iteration)`); } } contentTypes?.push?.(contentType); diff --git a/api/tests/unit/utils/entry-update.utils.test.ts b/api/tests/unit/utils/entry-update.utils.test.ts index dc9e39ed9..31ed59996 100644 --- a/api/tests/unit/utils/entry-update.utils.test.ts +++ b/api/tests/unit/utils/entry-update.utils.test.ts @@ -11,6 +11,9 @@ const { mockMkdirSync, mockReaddirSync, mockAppendFileSync, + mockRmSync, + mockSanitizeStackId, + mockAssertResolvedPathUnderBase, } = vi.hoisted(() => ({ mockProjectRead: vi.fn(), mockChainGet: vi.fn(), @@ -22,6 +25,9 @@ const { mockMkdirSync: vi.fn(), mockReaddirSync: vi.fn(), mockAppendFileSync: vi.fn(), + mockRmSync: vi.fn(), + mockSanitizeStackId: vi.fn(), + mockAssertResolvedPathUnderBase: vi.fn(), })); vi.mock('../../../src/models/project-lowdb.js', () => ({ @@ -46,9 +52,16 @@ vi.mock('node:fs', () => ({ mkdirSync: mockMkdirSync, appendFileSync: mockAppendFileSync, readdirSync: mockReaddirSync, + rmSync: mockRmSync, }, })); +vi.mock('../../../src/utils/sanitize-path.utils.js', () => ({ + sanitizeStackId: mockSanitizeStackId, + assertResolvedPathUnderBase: mockAssertResolvedPathUnderBase, + getSafePath: (p: string) => p, +})); + describe('entry-update.utils', () => { beforeEach(() => { vi.clearAllMocks(); @@ -68,6 +81,52 @@ describe('entry-update.utils', () => { { otherCmsEntryUid: 'legacy-key', isUpdate: true, contentstackEntryUid: 'cs-uid' }, ], }); + mockSanitizeStackId.mockImplementation((id: string) => id); + mockAssertResolvedPathUnderBase.mockReturnValue(undefined); + }); + + describe('clearStaleEntries', () => { + it('skips cleanup and logs when stackId is invalid', async () => { + mockSanitizeStackId.mockReturnValue(''); + const { clearStaleEntries } = await import('../../../src/utils/entry-update.utils.js'); + + clearStaleEntries('../../evil', '/tmp/mig.log'); + + expect(mockRmSync).not.toHaveBeenCalled(); + expect(mockAppendFileSync).toHaveBeenCalledWith( + '/tmp/mig.log', + expect.stringContaining('Invalid stackId') + ); + }); + + it('does nothing when entries directory does not exist', async () => { + mockExistsSync.mockReturnValue(false); + const { clearStaleEntries } = await import('../../../src/utils/entry-update.utils.js'); + + clearStaleEntries('stack1', '/tmp/mig.log'); + + expect(mockRmSync).not.toHaveBeenCalled(); + expect(mockAppendFileSync).toHaveBeenCalledWith( + '/tmp/mig.log', + expect.stringContaining('No existing entries directory') + ); + }); + + it('removes the entries directory when it exists', async () => { + mockExistsSync.mockReturnValue(true); + const { clearStaleEntries } = await import('../../../src/utils/entry-update.utils.js'); + + clearStaleEntries('stack1', '/tmp/mig.log'); + + expect(mockRmSync).toHaveBeenCalledWith( + expect.stringContaining('entries'), + { recursive: true, force: true } + ); + expect(mockAppendFileSync).toHaveBeenCalledWith( + '/tmp/mig.log', + expect.stringContaining('Cleared stale entries directory') + ); + }); }); it('removeEntriesFromDatabase returns null when stackId missing', async () => { @@ -93,15 +152,20 @@ describe('entry-update.utils', () => { }); it('removeEntriesFromDatabase walks dirs, updates json, writes config', async () => { - mockExistsSync.mockReturnValue(true); + // entriesDir + content-type/locale dirs exist, but no index.json → glob fallback. + mockExistsSync.mockImplementation((p: string) => !String(p).endsWith('index.json')); const dirent = (name: string, isDir: boolean) => ({ name, isDirectory: () => isDir, }); - mockReaddirSync - .mockReturnValueOnce([dirent('ct1', true)]) - .mockReturnValueOnce([dirent('en', true)]) - .mockReturnValueOnce(['page.json']); + // Branch on the options arg instead of queued returns so the once-queue can't + // leak into a later test. + mockReaddirSync.mockImplementation((p: string, opts?: { withFileTypes?: boolean }) => { + if (opts?.withFileTypes) { + return String(p).endsWith('ct1') ? [dirent('en', true)] : [dirent('ct1', true)]; + } + return ['page.json']; + }); mockReadFileSync.mockReturnValue( JSON.stringify({ 'legacy-key': { title: 'Hello' }, keep: { x: 1 } }) ); @@ -114,6 +178,128 @@ describe('entry-update.utils', () => { expect(mockMkdirSync).toHaveBeenCalled(); }); + describe('removeEntriesFromDatabase index.json-driven chunk selection', () => { + const dirent = (name: string, isDir: boolean) => ({ + name, + isDirectory: () => isDir, + }); + + it('only processes chunk files listed in index.json (ignores stale orphans)', async () => { + // entriesDir exists + index.json exists + the listed chunk exists + mockExistsSync.mockReturnValue(true); + mockReaddirSync.mockImplementation((p: string, opts?: { withFileTypes?: boolean }) => + opts?.withFileTypes + ? (String(p).endsWith('ct1') ? [dirent('en', true)] : [dirent('ct1', true)]) + : [] + ); + // index.json lists only current.json; orphan.json is NOT listed and must be ignored + mockReadFileSync.mockImplementation((p: string) => { + if (String(p).endsWith('index.json')) { + return JSON.stringify({ '1': 'current.json' }); + } + return JSON.stringify({ 'legacy-key': { title: 'Hello' } }); + }); + + const { removeEntriesFromDatabase } = await import('../../../src/utils/entry-update.utils.js'); + const result = await removeEntriesFromDatabase('p1', '/tmp/mig.log'); + + expect(result).toMatch(/updated-entries\.json$/); + // The chunk read must be the index-listed file, never the orphan. + const readPaths = mockReadFileSync.mock.calls.map((c) => String(c[0])); + expect(readPaths.some((p) => p.endsWith('current.json'))).toBe(true); + expect(readPaths.some((p) => p.endsWith('orphan.json'))).toBe(false); + }); + + it('skips the locale and logs when index.json is corrupt', async () => { + mockExistsSync.mockReturnValue(true); + mockReaddirSync.mockImplementation((p: string, opts?: { withFileTypes?: boolean }) => + opts?.withFileTypes + ? (String(p).endsWith('ct1') ? [dirent('en', true)] : [dirent('ct1', true)]) + : [] + ); + mockReadFileSync.mockImplementation((p: string) => { + if (String(p).endsWith('index.json')) return '{ not valid json'; + return JSON.stringify({ 'legacy-key': { title: 'Hello' } }); + }); + + const { removeEntriesFromDatabase } = await import('../../../src/utils/entry-update.utils.js'); + const result = await removeEntriesFromDatabase('p1', '/tmp/mig.log'); + + // Whole step must not abort; config still written, locale skipped. + expect(result).toMatch(/updated-entries\.json$/); + expect(mockAppendFileSync).toHaveBeenCalledWith( + '/tmp/mig.log', + expect.stringContaining('Failed to parse index.json') + ); + // No chunk file was read because the locale was skipped. + const readPaths = mockReadFileSync.mock.calls.map((c) => String(c[0])); + expect(readPaths.some((p) => p.endsWith('current.json'))).toBe(false); + }); + + it('skips the locale and logs when index.json is not an object', async () => { + mockExistsSync.mockReturnValue(true); + mockReaddirSync.mockImplementation((p: string, opts?: { withFileTypes?: boolean }) => + opts?.withFileTypes + ? (String(p).endsWith('ct1') ? [dirent('en', true)] : [dirent('ct1', true)]) + : [] + ); + mockReadFileSync.mockImplementation((p: string) => { + if (String(p).endsWith('index.json')) return JSON.stringify('a string, not an object'); + return JSON.stringify({ 'legacy-key': { title: 'Hello' } }); + }); + + const { removeEntriesFromDatabase } = await import('../../../src/utils/entry-update.utils.js'); + const result = await removeEntriesFromDatabase('p1', '/tmp/mig.log'); + + expect(result).toMatch(/updated-entries\.json$/); + expect(mockAppendFileSync).toHaveBeenCalledWith( + '/tmp/mig.log', + expect.stringContaining('is not an object') + ); + }); + + // readdirSync is called two ways: `{ withFileTypes: true }` for the dir walks + // (content-type dirs, then locale dirs) and with no options for the glob fallback. + // Branch on the options arg so call ordering can't desync the mocks. + const dirWalkImpl = (globResult: string[] = ['page.json', 'index.json']) => + (p: string, opts?: { withFileTypes?: boolean }) => { + if (opts?.withFileTypes) { + return String(p).endsWith('ct1') ? [dirent('en', true)] : [dirent('ct1', true)]; + } + return globResult; + }; + + it('falls back to globbing when index.json is absent (legacy data)', async () => { + // entriesDir exists, but index.json does not → glob the locale dir + mockExistsSync.mockImplementation((p: string) => !String(p).endsWith('index.json')); + mockReaddirSync.mockImplementation(dirWalkImpl(['page.json', 'index.json'])); + mockReadFileSync.mockReturnValue(JSON.stringify({ 'legacy-key': { title: 'Hello' } })); + + const { removeEntriesFromDatabase } = await import('../../../src/utils/entry-update.utils.js'); + const result = await removeEntriesFromDatabase('p1', '/tmp/mig.log'); + + expect(result).toMatch(/updated-entries\.json$/); + const readPaths = mockReadFileSync.mock.calls.map((c) => String(c[0])); + // index.json itself is filtered out of the glob; only page.json is read. + expect(readPaths.some((p) => p.endsWith('page.json'))).toBe(true); + }); + + it('throws when an index-listed chunk file does not exist on disk', async () => { + mockExistsSync.mockReturnValue(true); + mockReaddirSync.mockImplementation(dirWalkImpl()); + mockReadFileSync.mockImplementation((p: string) => { + if (String(p).endsWith('index.json')) return JSON.stringify({ '1': 'ghost.json' }); + // Reading the listed-but-missing chunk fails like a real fs.readFileSync ENOENT + const err: NodeJS.ErrnoException = new Error('ENOENT: no such file'); + err.code = 'ENOENT'; + throw err; + }); + + const { removeEntriesFromDatabase } = await import('../../../src/utils/entry-update.utils.js'); + await expect(removeEntriesFromDatabase('p1', '/tmp/mig.log')).rejects.toThrow(/ENOENT/); + }); + }); + it('enrichConfigWithAssetMapping covers iteration 1 (no old path branch)', async () => { mockExistsSync.mockReturnValue(false); const { enrichConfigWithAssetMapping } = await import('../../../src/utils/entry-update.utils.js'); diff --git a/ui/src/components/Common/SaveChangesModal/index.tsx b/ui/src/components/Common/SaveChangesModal/index.tsx index 67b3104d2..61a0cfe8f 100644 --- a/ui/src/components/Common/SaveChangesModal/index.tsx +++ b/ui/src/components/Common/SaveChangesModal/index.tsx @@ -11,9 +11,9 @@ interface Props { closeModal: () => void; isopen?: (flag: boolean) => void; otherCmsTitle?: string; - saveContentType?: () => void; + saveContentType?: () => void | Promise; openContentType?: () => void; - changeStep?: () => void; + changeStep?: () => void | Promise; dropdownStateChange: () => void; } @@ -47,24 +47,24 @@ const SaveChangesModal = (props: Props) => {