Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions apps/web/src/backend/api/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1267,7 +1267,7 @@ export function createApiRoutes(connectionModule: ConnectionModule, connectionSt
res.status(400).json({ error: 'dialect and script are required' });
return;
}
const id = await dataMigrateHistory.start((req as AuthedRequest).userId!, {
const started = await dataMigrateHistory.start((req as AuthedRequest).userId!, {
dialect: body.dialect,
sourceHost: body.sourceHost,
targetHost: body.targetHost,
Expand All @@ -1287,7 +1287,7 @@ export function createApiRoutes(connectionModule: ConnectionModule, connectionSt
script: body.script,
snapshotJson: body.snapshotJson,
});
res.json({ id });
res.json(started);
}
);

Expand Down
32 changes: 32 additions & 0 deletions apps/web/src/backend/modules/data-migrate-history.module.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/**
* Fox Schema (foxschema)
* Copyright 2024-2026 Huy Phan <huyplb@gmail.com>
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, expect, it } from 'vitest';
import {
DATA_MIGRATE_MAX_TEXT_LEN,
storeableSnapshotJson,
} from './data-migrate-history.module';

describe('storeableSnapshotJson', () => {
it('stores intact JSON under the size limit', () => {
const json = JSON.stringify({ version: 1, rows: [], columns: ['id'] });
expect(storeableSnapshotJson(json)).toEqual({ json, stored: true });
});

it('omits oversized snapshots instead of appending a truncation marker', () => {
const json = `${'{"rows":['}${'1,'.repeat(DATA_MIGRATE_MAX_TEXT_LEN)}]`;
expect(json.length).toBeGreaterThan(DATA_MIGRATE_MAX_TEXT_LEN);
expect(storeableSnapshotJson(json)).toEqual({ json: undefined, stored: false });
});

it('rejects non-JSON so History never offers a corrupt Restore payload', () => {
const truncated = `${JSON.stringify({ rows: [1], columns: ['id'] }).slice(0, 20)}\n… (truncated)`;
expect(storeableSnapshotJson(truncated)).toEqual({ json: undefined, stored: false });
});

it('treats missing snapshot as not stored', () => {
expect(storeableSnapshotJson(undefined)).toEqual({ json: undefined, stored: false });
});
});
36 changes: 29 additions & 7 deletions apps/web/src/backend/modules/data-migrate-history.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,32 @@ interface Row {
}

const MAX_RUNS_PER_USER = 200;
const MAX_TEXT_LEN = 1_000_000;
/** Bound script / snapshot blobs so one run cannot bloat the metadata DB. */
export const DATA_MIGRATE_MAX_TEXT_LEN = 1_000_000;

function cap(text: string | undefined, max = MAX_TEXT_LEN): string | undefined {
/** Truncate free-form script text for display; never used for machine-parsed JSON. */
function capScript(text: string | undefined, max = DATA_MIGRATE_MAX_TEXT_LEN): string | undefined {
if (text == null) return text;
return text.length > max ? `${text.slice(0, max)}\n… (truncated)` : text;
return text.length > max ? `${text.slice(0, max)}\n-- … (truncated)` : text;
}

/**
* Store snapshot JSON only when it fits intact. Appending a truncation marker
* (the old `cap()` behavior) produced invalid JSON, so History still offered
* Restore while `JSON.parse` always failed — a silent loss of the undo safety net.
*/
export function storeableSnapshotJson(
text: string | undefined,
max = DATA_MIGRATE_MAX_TEXT_LEN
): { json: string | undefined; stored: boolean } {
if (text == null) return { json: undefined, stored: false };
if (text.length > max) return { json: undefined, stored: false };
try {
JSON.parse(text);
} catch {
return { json: undefined, stored: false };
}
return { json: text, stored: true };
}

function parseOps(raw: string | null): { insert: boolean; update: boolean; delete: boolean } {
Expand Down Expand Up @@ -105,8 +126,9 @@ export class DataMigrateHistoryStore {
script: string;
snapshotJson?: string;
}
): Promise<string> {
): Promise<{ id: string; snapshotStored: boolean }> {
const id = randomUUID();
const snapshot = storeableSnapshotJson(input.snapshotJson);
const store = await getStore();
await store.run(
`INSERT INTO data_migrate_runs
Expand All @@ -126,13 +148,13 @@ export class DataMigrateHistoryStore {
JSON.stringify(input.opsEnabled),
input.includeIdentity ? 1 : 0,
JSON.stringify(input.keyColumns),
cap(input.script) ?? null,
cap(input.snapshotJson) ?? null,
capScript(input.script) ?? null,
snapshot.json ?? null,
new Date().toISOString(),
]
);
await this.prune(userId);
return id;
return { id, snapshotStored: snapshot.stored };
}

private async prune(userId: string): Promise<void> {
Expand Down
5 changes: 2 additions & 3 deletions apps/web/src/frontend/api/dataMigrateApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,11 @@ export async function apiStartDataMigrate(input: {
keyColumns: string[];
script: string;
snapshotJson?: string;
}): Promise<string> {
const { id } = await request<{ id: string }>('/data-migrations/start', {
}): Promise<{ id: string; snapshotStored: boolean }> {
return request<{ id: string; snapshotStored: boolean }>('/data-migrations/start', {
method: 'POST',
body: JSON.stringify(input),
});
return id;
}

export async function apiFinishDataMigrate(
Expand Down
107 changes: 94 additions & 13 deletions apps/web/src/frontend/components/sql-editor/DataMigrateBar.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
type DataMigrateRunDetail,
type DataMigrateRunSummary,
} from '../../api/dataMigrateApi';
import { buildDataMigratePlans, buildDestSnapshotJson, buildRestorePlansFromSnapshot } from '../../lib/dataMigratePlans';
import { buildDataMigratePlans, buildDestSnapshotJson, buildRestorePlansFromSnapshot, isUsableDataMigrateSnapshot, snapshotTargetConnectionId } from '../../lib/dataMigratePlans';
import {
classifyRowsByKey,
DATA_MIGRATE_ROW_CAP,
Expand Down Expand Up @@ -150,6 +150,8 @@ export const DataMigrateBar: React.FC<Props> = ({
const [backupEnabled, setBackupEnabled] = useState(true);
const [lastBackup, setLastBackup] = useState<{
runId: string | null;
/** Destination that was migrated — Restore must not follow a later Destination switch. */
connectionId: string;
snapshotJson: string;
results: DataMigrateOpResult[];
tableName: string;
Expand Down Expand Up @@ -434,6 +436,7 @@ export const DataMigrateBar: React.FC<Props> = ({
? buildDestSnapshotJson({
tableName,
dialect: dest.dialect,
connectionId: dest.connectionId,
destColumns: dest.columns,
sourceColumns: source.columns,
keyNames,
Expand All @@ -457,8 +460,9 @@ export const DataMigrateBar: React.FC<Props> = ({
);

let runId: string | null = null;
let snapshotStored = false;
try {
runId = await apiStartDataMigrate({
const started = await apiStartDataMigrate({
dialect: dest.dialect,
sourceHost: sourceConn?.host || source.label,
targetHost: destConn?.host || dest.label,
Expand All @@ -472,6 +476,15 @@ export const DataMigrateBar: React.FC<Props> = ({
script,
snapshotJson,
});
runId = started.id;
snapshotStored = started.snapshotStored;
if (backupEnabled && snapshotJson && !snapshotStored) {
toast({
tone: 'warning',
title: 'Backup too large for History',
body: 'The pre-apply snapshot exceeds the 1MB History limit, so durable Restore after reload is unavailable. Use Restore from the toast while this session still has the in-memory backup.',
});
}
} catch (e) {
toast({
tone: 'warning',
Expand Down Expand Up @@ -557,13 +570,19 @@ export const DataMigrateBar: React.FC<Props> = ({
!rolledBack &&
succeeded.length > 0;

// Capture at apply time — toast onAction must not read a later Destination switch.
const migratedConnectionId = dest.connectionId;
const migratedDialect = dest.dialect;
const migratedLabel = dest.label;

if (canOfferRestore && snapshotJson) {
setLastBackup({
runId,
connectionId: migratedConnectionId,
snapshotJson,
results,
tableName,
dialect: dest.dialect,
dialect: migratedDialect,
});
} else {
setLastBackup(null);
Expand All @@ -583,7 +602,13 @@ export const DataMigrateBar: React.FC<Props> = ({
: `Finished with ${failCount} failure(s)`,
body:
failCount === 0
? `Destination: ${dest.label}.${backupEnabled ? ' Backup snapshot saved to history.' : ''}`
? `Destination: ${migratedLabel}.${
backupEnabled
? snapshotStored
? ' Backup snapshot saved to history.'
: ' Backup kept in-session only (too large for History).'
: ''
}`
: `Failed: ${failedKeys.join('; ')}${
results.filter((r) => r.status === 'FAILED').length > 5 ? '…' : ''
}. ${rolledBack ? 'Transaction rolled back. ' : ''}${
Expand All @@ -594,10 +619,11 @@ export const DataMigrateBar: React.FC<Props> = ({
? () =>
void restoreFromBackup({
runId,
connectionId: migratedConnectionId,
snapshotJson: snapshotJson!,
results,
tableName,
dialect: dest.dialect,
dialect: migratedDialect,
})
: () => void openHistory(),
durationMs: 12_000,
Expand All @@ -607,6 +633,7 @@ export const DataMigrateBar: React.FC<Props> = ({

const restoreFromBackup = async (backup: {
runId: string | null;
connectionId: string;
snapshotJson: string;
results: DataMigrateOpResult[];
tableName: string;
Expand All @@ -621,6 +648,31 @@ export const DataMigrateBar: React.FC<Props> = ({
return;
}

// Prefer the connection recorded at migrate time (snapshot / caller). Never
// silently follow a later Compare Destination switch — that would reverse
// DML onto the wrong database (same failure class as Lokee #256).
const targetConnectionId =
backup.connectionId.trim() ||
snapshotTargetConnectionId(backup.snapshotJson) ||
'';
if (!targetConnectionId) {
toast({
tone: 'warning',
title: 'Cannot restore',
body: 'This backup does not record which destination was migrated. Re-run migrate with Backup on, or restore only while the original Destination is selected.',
});
return;
}
const targetConn = connections.find((c) => c.id === targetConnectionId);
if (!targetConn) {
toast({
tone: 'warning',
title: 'Cannot restore',
body: 'The destination credential for this backup is no longer available.',
});
return;
}

const { plans, errors } = buildRestorePlansFromSnapshot({
snapshotJson: backup.snapshotJson,
successfulOps,
Expand All @@ -647,12 +699,12 @@ export const DataMigrateBar: React.FC<Props> = ({

let restoreRunId: string | null = null;
try {
restoreRunId = await apiStartDataMigrate({
const started = await apiStartDataMigrate({
dialect: backup.dialect,
sourceHost: 'backup-restore',
targetHost: destConn?.host || dest.label,
database: destConn?.database,
schema: destConn?.schema,
targetHost: targetConn.host || targetConn.name || targetConnectionId,
database: targetConn.database,
schema: targetConn.schema,
tableName: backup.tableName,
rowCount: plans.length,
opsEnabled: { insert: true, update: true, delete: true },
Expand All @@ -664,16 +716,17 @@ export const DataMigrateBar: React.FC<Props> = ({
].join('\n\n'),
snapshotJson: backup.snapshotJson,
});
restoreRunId = started.id;
} catch {
/* history best-effort */
}

try {
const out = await apiExecuteDataMigrate(
{
connectionId: dest.connectionId,
password: sessionPasswords[dest.connectionId] || undefined,
schema: destConn?.schema?.trim() || undefined,
connectionId: targetConnectionId,
password: sessionPasswords[targetConnectionId] || undefined,
schema: targetConn.schema?.trim() || undefined,
},
plans.map((p) => ({
op: p.op,
Expand Down Expand Up @@ -1113,7 +1166,7 @@ export const DataMigrateBar: React.FC<Props> = ({
<pre className="mt-1 max-h-40 overflow-auto rounded bg-slate-900 p-2 text-[10px] text-slate-400">
{historyDetail.snapshotJson || '(none — Backup was off)'}
</pre>
{historyDetail.snapshotJson &&
{isUsableDataMigrateSnapshot(historyDetail.snapshotJson) &&
historyDetail.results.some((r) => r.status === 'SUCCESS') &&
historyDetail.status !== 'SUCCESS' && (
<button
Expand All @@ -1122,9 +1175,30 @@ export const DataMigrateBar: React.FC<Props> = ({
disabled={restoring || applying}
className="mt-2 inline-flex items-center gap-1 rounded-md border border-amber-500/50 bg-amber-950/40 px-2 py-1 text-[11px] font-semibold text-amber-200 hover:bg-amber-900/50 disabled:opacity-40"
onClick={() => {
const connectionId =
snapshotTargetConnectionId(historyDetail.snapshotJson!) ||
// Legacy backups (pre-connectionId): only restore when the
// current Destination still matches the recorded target.
(historyDetail.targetHost &&
destConn &&
(destConn.host === historyDetail.targetHost ||
dest.label === historyDetail.targetHost) &&
(!historyDetail.database ||
destConn.database === historyDetail.database)
? dest.connectionId
: '');
if (!connectionId) {
toast({
tone: 'warning',
title: 'Cannot restore',
body: 'Select the original Destination credential for this backup before restoring.',
});
return;
}
setHistoryOpen(false);
void restoreFromBackup({
runId: historyDetail.id,
connectionId,
snapshotJson: historyDetail.snapshotJson!,
results: historyDetail.results,
tableName: historyDetail.tableName || tableName,
Expand All @@ -1136,6 +1210,13 @@ export const DataMigrateBar: React.FC<Props> = ({
Restore from this backup
</button>
)}
{historyDetail.snapshotJson &&
!isUsableDataMigrateSnapshot(historyDetail.snapshotJson) && (
<p className="mt-2 text-[11px] text-rose-300">
Snapshot is not valid JSON — Restore is unavailable (likely an
older truncated History entry).
</p>
)}
</div>
<div>
<span className="text-slate-500">Script</span>
Expand Down
Loading
Loading