From be92141de933a965eb4baf5401d586bc31f5d263 Mon Sep 17 00:00:00 2001 From: LeaningLearner <158067205+LeaningLearner@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:45:51 +0800 Subject: [PATCH] fix: harden router supervisor and autostart recovery --- package-lock.json | 19 + src/autostart.mjs | 27 ++ src/catalog.mjs | 27 ++ src/cli.mjs | 480 +++++++++++++++++------- src/supervisor.mjs | 594 +++++++++++++++++++++++++++++- test/autostart.test.mjs | 300 +++++++++++++++ test/config.test.mjs | 45 ++- test/supervisor-recovery.test.mjs | 304 +++++++++++++++ test/supervisor.test.mjs | 367 +++++++++++++++++- 9 files changed, 2016 insertions(+), 147 deletions(-) create mode 100644 package-lock.json create mode 100644 test/supervisor-recovery.test.mjs diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..191cbeb --- /dev/null +++ b/package-lock.json @@ -0,0 +1,19 @@ +{ + "name": "dscodex", + "version": "1.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "dscodex", + "version": "1.1.0", + "license": "MIT", + "bin": { + "dscodex": "src/cli.mjs" + }, + "engines": { + "node": ">=24.5" + } + } + } +} diff --git a/src/autostart.mjs b/src/autostart.mjs index 85553af..f6a8681 100644 --- a/src/autostart.mjs +++ b/src/autostart.mjs @@ -7,6 +7,33 @@ export const WINDOWS_TASK = "DSCodex"; export const WINDOWS_RESTART_COUNT = 255; export const WINDOWS_RESTART_INTERVAL_MINUTES = 1; +// Cleanup must remain best-effort: a corrupt pid file or failed authenticated +// shutdown cannot prevent the service manager and generated artifact from being +// removed. Report every failure only after all requested steps have run. +export async function cleanupAutostart({ + stopRouter, + deactivateManager, + removeArtifact, + reloadManager, + message = "Failed to clean up DSCodex autostart", +}) { + const failures = []; + for (const operation of [stopRouter, deactivateManager, removeArtifact, reloadManager]) { + if (!operation) continue; + try { + await operation(); + } catch (error) { + failures.push(error); + } + } + if (failures.length) { + const details = failures.map((error) => ( + error instanceof Error ? error.message : String(error) + )).join("; "); + throw new AggregateError(failures, `${message}: ${details}`); + } +} + export function autostartKind(platform = process.platform) { if (platform === "darwin") return "launchd"; if (platform === "win32") return "schtasks"; diff --git a/src/catalog.mjs b/src/catalog.mjs index 272ff36..8cc71d3 100644 --- a/src/catalog.mjs +++ b/src/catalog.mjs @@ -24,6 +24,17 @@ const NATIVE_ENTRY_DEFAULTS = { supports_reasoning_summaries: false, }; +// These fields are required by the current Codex model-catalog parser. Keep +// this deliberately narrower than the full catalog shape: native entries may +// gain optional fields independently, while doctor only needs to know that the +// merged catalog is parseable and still contains both DSCodex models. +const REQUIRED_ENTRY_FIELD_TYPES = Object.freeze({ + slug: "string", + base_instructions: "string", + prefer_websockets: "boolean", + supports_reasoning_summaries: "boolean", +}); + function backfillNativeEntry(model) { const entry = clone(model); for (const [key, value] of Object.entries(NATIVE_ENTRY_DEFAULTS)) { @@ -113,6 +124,22 @@ export function buildCatalog(cache) { }; } +export function isCatalogReady(catalog) { + if (!Array.isArray(catalog?.models) || catalog.models.length === 0) return false; + + const slugs = new Set(); + for (const model of catalog.models) { + if (!model || typeof model !== "object" || Array.isArray(model)) return false; + if (Object.entries(REQUIRED_ENTRY_FIELD_TYPES).some(([field, type]) => ( + typeof model[field] !== type + ))) return false; + if (model.slug.trim().length === 0 || slugs.has(model.slug)) return false; + slugs.add(model.slug); + } + + return DEEPSEEK_MODELS.every((model) => slugs.has(model.pickerSlug)); +} + export function writeCatalog({ catalogPath, catalog }) { const temporary = `${catalogPath}.dscodex-tmp-${process.pid}`; writeFileSync(temporary, `${JSON.stringify(catalog, null, 2)}\n`, { mode: 0o600 }); diff --git a/src/cli.mjs b/src/cli.mjs index 51ab374..c480e0a 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -16,7 +16,7 @@ import { request as httpRequest } from "node:http"; import { dirname, join } from "node:path"; import { createInterface } from "node:readline"; import { fileURLToPath } from "node:url"; -import { buildCatalog, syncCatalog } from "./catalog.mjs"; +import { buildCatalog, isCatalogReady, syncCatalog } from "./catalog.mjs"; import { ensureManagedRouterBinding, install, @@ -41,7 +41,13 @@ import { validateProxyUrl, } from "./proxy-config.mjs"; import { createProxyServer } from "./proxy.mjs"; -import { superviseRouter } from "./supervisor.mjs"; +import { + authenticateSupervisorOwner, + readSupervisorState, + removeSupervisorState, + requestSupervisorStop, + superviseRouter, +} from "./supervisor.mjs"; import { LAUNCHD_LABEL, SYSTEMD_UNIT, @@ -51,6 +57,7 @@ import { buildSystemdUnit, buildWindowsRegisterScript, buildWindowsVbs, + cleanupAutostart, encodeWindowsVbs, launchdPlistPath, systemdUnitPath, @@ -342,11 +349,7 @@ function loadModels(paths) { function catalogReady(paths) { try { - const models = loadModels(paths); - return models.length > 0 && models.every((model) => ( - typeof model?.slug === "string" - && typeof model.base_instructions === "string" - )); + return isCatalogReady({ models: loadModels(paths) }); } catch { return false; } @@ -523,7 +526,7 @@ async function serve(port) { }); } -async function stopInstance(paths) { +async function stopRouterInstance(paths) { const state = readPidState(paths); if (!state) return "none"; const expected = { pid: state.pid, instanceId: state.instanceId }; @@ -559,6 +562,113 @@ async function stopInstance(paths) { throw new Error(`Router PID ${state.pid} did not exit after authenticated shutdown`); } +function sameSupervisor(left, right) { + return left?.pid === right?.pid && left?.instanceId === right?.instanceId; +} + +async function stopInstance(paths) { + const failures = []; + const attemptedRouters = new Set(); + let supervisor = null; + let sawSupervisor = false; + let result = "none"; + + try { + supervisor = readSupervisorState(paths.log); + if (supervisor) { + sawSupervisor = true; + // The request is scoped to this exact random instance token and never + // signals the PID. Leave it even when the fresh liveness challenge times + // out: a temporarily stalled real owner will honor it when it resumes, + // while a recycled unrelated PID cannot observe or act on it. + requestSupervisorStop(paths.log, supervisor); + if (!(await authenticateSupervisorOwner(paths.log, supervisor))) { + removeSupervisorState(paths.log, supervisor); + supervisor = null; + } + } + } catch (error) { + failures.push(error); + } + + try { + const state = readPidState(paths); + if (state) attemptedRouters.add(state.instanceId); + result = await stopRouterInstance(paths); + } catch (error) { + failures.push(error); + } + + // A stop request can race with a supervised child publishing server.pid. + // Keep watching the authenticated supervisor instance and, if a fresh child + // appears, stop that child through the normal authenticated HTTP endpoint. + if (supervisor) { + let supervisorStopped = false; + for (let attempt = 0; attempt < 80; attempt += 1) { + let current; + try { + current = readSupervisorState(paths.log); + } catch (error) { + failures.push(error); + break; + } + if (!current) { + supervisorStopped = true; + break; + } + if (!sameSupervisor(current, supervisor)) { + supervisor = current; + sawSupervisor = true; + try { + requestSupervisorStop(paths.log, current); + } catch (error) { + failures.push(error); + break; + } + } + let supervisorAuthenticated; + try { + supervisorAuthenticated = await authenticateSupervisorOwner(paths.log, current); + } catch (error) { + failures.push(error); + break; + } + if (!supervisorAuthenticated) { + try { + removeSupervisorState(paths.log, current); + } catch (error) { + failures.push(error); + } + // Re-read on the next pass: a replacement may have published between + // the failed challenge and conditional cleanup of this generation. + await new Promise((resolve) => setTimeout(resolve, 25)); + continue; + } + try { + const router = readPidState(paths); + if (router && !attemptedRouters.has(router.instanceId)) { + attemptedRouters.add(router.instanceId); + result = await stopRouterInstance(paths); + } + } catch (error) { + failures.push(error); + break; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + if (!supervisorStopped) { + failures.push(new Error(`Supervisor PID ${supervisor.pid} did not acknowledge the authenticated stop request`)); + } + } + + if (failures.length) { + throw new AggregateError(failures, failures.map((error) => ( + error instanceof Error ? error.message : String(error) + )).join("; ")); + } + return sawSupervisor && (result === "none" || result === "stale") ? "stopped" : result; +} + async function start(port) { const { paths } = runtime(); const binding = ensureManagedRouterBinding({ paths, port }); @@ -629,27 +739,22 @@ async function waitForWindowsTaskState(expected, attempts = 100) { } async function restoreManualRouter(paths, port, cause, rollback) { - const failures = [cause]; if (rollback) { try { await rollback(); } catch (rollbackError) { - failures.push(rollbackError); + throw new AggregateError( + [cause, rollbackError], + `Autostart failed and rollback was incomplete, so the previous manual router was not restarted: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}`, + ); } } try { await start(port); } catch (restoreError) { - failures.push(restoreError); throw new AggregateError( - failures, - `Autostart failed and the previous manual router could not be restored; inspect ${paths.log}`, - ); - } - if (failures.length > 1) { - throw new AggregateError( - failures, - "Autostart failed; the previous manual router was restored, but autostart rollback also failed", + [cause, restoreError], + `Autostart failed and the previous manual router could not be restored; inspect ${paths.log}: ${restoreError instanceof Error ? restoreError.message : String(restoreError)}`, ); } throw new Error( @@ -658,6 +763,18 @@ async function restoreManualRouter(paths, port, cause, rollback) { ); } +async function throwAfterRollback(cause, rollback) { + try { + await rollback(); + } catch (rollbackError) { + throw new AggregateError( + [cause, rollbackError], + `Autostart failed and rollback was incomplete: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}`, + ); + } + throw cause; +} + async function stop() { const { paths } = runtime(); const result = await stopInstance(paths); @@ -673,16 +790,107 @@ function autostartFile(paths) { return join(paths.stateDir, "autostart-run.vbs"); } +function windowsTaskExists() { + try { + execFileSync("schtasks", ["/query", "/tn", WINDOWS_TASK], { stdio: ["ignore", "ignore", "ignore"] }); + return true; + } catch { + return false; + } +} + function autostartEnabled(paths) { - if (autostartKind() === "schtasks") { - try { - execFileSync("schtasks", ["/query", "/tn", WINDOWS_TASK], { stdio: ["ignore", "ignore", "ignore"] }); - return true; - } catch { - return false; + if (autostartKind() === "schtasks") return windowsTaskExists(); + return existsSync(autostartFile(paths)); +} + +function launchdLoaded(uid) { + try { + execFileSync("/bin/launchctl", ["print", `gui/${uid}/${LAUNCHD_LABEL}`], { + stdio: ["ignore", "ignore", "ignore"], + }); + return true; + } catch { + return false; + } +} + +function bootoutLaunchd(uid) { + if (!launchdLoaded(uid)) return; + try { + execFileSync("/bin/launchctl", ["bootout", `gui/${uid}/${LAUNCHD_LABEL}`], { + stdio: ["ignore", "ignore", "pipe"], + }); + } catch (error) { + // A service can finish between print and bootout; only surface the error if + // launchd still reports that our label is loaded. + if (launchdLoaded(uid)) throw error; + } +} + +function removeAutostartArtifact(file) { + try { + unlinkSync(file); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } +} + +function deactivateWindowsTask() { + const failures = []; + try { + execFileSync("schtasks", ["/end", "/tn", WINDOWS_TASK], { stdio: ["ignore", "ignore", "ignore"] }); + } catch (error) { + if (windowsTaskInfo()?.state === "Running") failures.push(error); + } + try { + execFileSync("schtasks", ["/delete", "/tn", WINDOWS_TASK, "/f"], { stdio: ["ignore", "ignore", "pipe"] }); + } catch (error) { + if (windowsTaskExists()) failures.push(error); + } + if (failures.length) { + throw new AggregateError(failures, failures.map((error) => ( + error instanceof Error ? error.message : String(error) + )).join("; ")); + } +} + +async function cleanupManagedAutostart(paths, { + kind, + file, + stopRouter = false, + deactivateManager = true, + managerExpected = false, + message, +}) { + const uid = kind === "launchd" ? process.getuid() : null; + let managerCleanup = null; + if (deactivateManager) { + if (kind === "launchd") { + managerCleanup = () => bootoutLaunchd(uid); + } else if (kind === "schtasks") { + managerCleanup = () => deactivateWindowsTask(); + } else { + managerCleanup = () => { + try { + execFileSync("systemctl", ["--user", "disable", "--now", SYSTEMD_UNIT], { + stdio: ["ignore", "ignore", "pipe"], + }); + } catch (error) { + if (managerExpected) throw error; + } + }; } } - return existsSync(autostartFile(paths)); + await cleanupAutostart({ + stopRouter: stopRouter ? () => stopInstance(paths) : null, + deactivateManager: managerCleanup, + removeArtifact: () => removeAutostartArtifact(file), + reloadManager: kind === "systemd" && managerExpected + ? () => execFileSync("systemctl", ["--user", "daemon-reload"], { stdio: ["ignore", "ignore", "pipe"] }) + : null, + message, + }); } // The generated plist/unit/VBS never embeds the DeepSeek key: the router resolves @@ -698,6 +906,8 @@ async function autostartEnable(paths, port) { requireProxyRuntime(paths); const wasRunning = Boolean(await health(port, routerToken)); if (kind === "schtasks") { + const previousTaskExisted = autostartEnabled(paths); + const previousArtifact = existsSync(file) ? readFileSync(file) : null; writeFileSync(file, encodeWindowsVbs(buildWindowsVbs({ nodePath: nodePath(), cliPath, @@ -706,41 +916,40 @@ async function autostartEnable(paths, port) { })), { mode: 0o600 }); // Register before touching a healthy manual router. A permissions/policy // failure must never turn a failed autostart attempt into an outage. - execFileSync(powershellPath(), [ - "-NoProfile", - "-NonInteractive", - "-Command", - buildWindowsRegisterScript({ taskName: WINDOWS_TASK, vbsPath: file }), - ], { stdio: ["ignore", "ignore", "pipe"] }); + try { + execFileSync(powershellPath(), [ + "-NoProfile", + "-NonInteractive", + "-Command", + buildWindowsRegisterScript({ taskName: WINDOWS_TASK, vbsPath: file }), + ], { stdio: ["ignore", "ignore", "pipe"] }); + } catch (error) { + const restorePreviousArtifact = async () => { + if (previousArtifact) writeFileSync(file, previousArtifact, { mode: 0o600 }); + else removeAutostartArtifact(file); + }; + // Never delete a task that existed before this registration attempt. If + // there was no previous task, clean up only when a post-failure query can + // positively identify a partially registered new task. + const rollbackRegistration = !previousTaskExisted && autostartEnabled(paths) + ? () => cleanupManagedAutostart(paths, { + kind, + file, + managerExpected: true, + message: "Failed to roll back Windows autostart registration", + }) + : restorePreviousArtifact; + return throwAfterRollback(error, rollbackRegistration); + } let manualStopped = false; - const rollback = async () => { - const failures = []; - if (manualStopped) { - // A task instance may have started but missed the readiness deadline. - // Stop it through the authenticated endpoint before removing its owner. - try { - await stopInstance(paths); - } catch (error) { - failures.push(error); - } - } - try { - execFileSync("schtasks", ["/end", "/tn", WINDOWS_TASK], { stdio: ["ignore", "ignore", "ignore"] }); - } catch { - // The task may never have started. - } - try { - execFileSync("schtasks", ["/delete", "/tn", WINDOWS_TASK, "/f"], { stdio: ["ignore", "ignore", "pipe"] }); - } catch (error) { - failures.push(error); - } - try { - unlinkSync(file); - } catch (error) { - if (error?.code !== "ENOENT") failures.push(error); - } - if (failures.length) throw new AggregateError(failures, "Failed to roll back Windows autostart"); - }; + let taskStarted = false; + const rollback = () => cleanupManagedAutostart(paths, { + kind, + file, + stopRouter: taskStarted, + managerExpected: true, + message: "Failed to roll back Windows autostart", + }); try { if (wasRunning) { await stopInstance(paths); @@ -759,6 +968,7 @@ async function autostartEnable(paths, port) { await waitForWindowsTaskState("Ready"); // Take effect now, not only at the next logon. execFileSync("schtasks", ["/run", "/tn", WINDOWS_TASK], { stdio: ["ignore", "ignore", "pipe"] }); + taskStarted = true; const ready = await waitForHealth(port, routerToken); if (!ready) throw new Error(`Autostart is installed but the router did not become ready; inspect ${paths.log}`); const task = windowsTaskInfo(); @@ -772,95 +982,70 @@ async function autostartEnable(paths, port) { if (manualStopped || (wasRunning && !(await health(port, routerToken)))) { return restoreManualRouter(paths, port, error, rollback); } - await rollback(); - throw error; + return throwAfterRollback(error, rollback); } } // launchd/systemd start the service during registration, so release the port // immediately before handing ownership to the service manager. - if (wasRunning) await stopInstance(paths); - if (kind === "launchd") { - mkdirSync(dirname(file), { recursive: true }); - writeFileSync(file, buildLaunchdPlist({ nodePath: nodePath(), cliPath, port, logPath: paths.log }), { mode: 0o600 }); - const uid = process.getuid(); - try { - execFileSync("/bin/launchctl", ["bootout", `gui/${uid}/${LAUNCHD_LABEL}`], { stdio: ["ignore", "ignore", "ignore"] }); - } catch { - // Not loaded yet. + let manualStopped = false; + let managerActivationAttempted = false; + const rollback = () => cleanupManagedAutostart(paths, { + kind, + file, + stopRouter: managerActivationAttempted, + deactivateManager: managerActivationAttempted, + managerExpected: managerActivationAttempted, + message: `Failed to roll back ${kind} autostart`, + }); + try { + if (wasRunning) { + await stopInstance(paths); + manualStopped = true; } - try { + if (kind === "launchd") { + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, buildLaunchdPlist({ nodePath: nodePath(), cliPath, port, logPath: paths.log }), { mode: 0o600 }); + const uid = process.getuid(); + bootoutLaunchd(uid); + managerActivationAttempted = true; execFileSync("/bin/launchctl", ["bootstrap", `gui/${uid}`, file], { stdio: ["ignore", "ignore", "pipe"] }); - } catch (error) { - if (wasRunning) return restoreManualRouter(paths, port, error); - throw error; - } - } else { - mkdirSync(dirname(file), { recursive: true }); - writeFileSync(file, buildSystemdUnit({ nodePath: nodePath(), cliPath, port, logPath: paths.log }), { mode: 0o600 }); - try { + } else { + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, buildSystemdUnit({ nodePath: nodePath(), cliPath, port, logPath: paths.log }), { mode: 0o600 }); execFileSync("systemctl", ["--user", "daemon-reload"], { stdio: ["ignore", "ignore", "pipe"] }); - execFileSync("systemctl", ["--user", "enable", "--now", SYSTEMD_UNIT], { stdio: ["ignore", "ignore", "pipe"] }); - } catch (error) { - const wrapped = new Error(`systemd user service unavailable: ${error instanceof Error ? error.message : String(error)}`, { cause: error }); - if (wasRunning && !(await health(port, routerToken))) return restoreManualRouter(paths, port, wrapped); - throw wrapped; + try { + managerActivationAttempted = true; + execFileSync("systemctl", ["--user", "enable", "--now", SYSTEMD_UNIT], { stdio: ["ignore", "ignore", "pipe"] }); + } catch (error) { + throw new Error(`systemd user service unavailable: ${error instanceof Error ? error.message : String(error)}`, { cause: error }); + } } - } - const ready = await waitForHealth(port, routerToken); - if (ready) { + const ready = await waitForHealth(port, routerToken); + if (!ready) throw new Error(`Autostart is installed but the router did not become ready; inspect ${paths.log}`); console.log(`DSCodex autostart enabled (${kind}); router running on ${HOST}:${port}`); console.log(`DeepSeek key: ${ready.deepseek_key ? "configured" : "missing (resolves from the stored key at runtime)"}`); - return; + } catch (error) { + if (manualStopped || (wasRunning && !(await health(port, routerToken)))) { + return restoreManualRouter(paths, port, error, rollback); + } + return throwAfterRollback(error, rollback); } - const error = new Error(`Autostart is installed but the router did not become ready; inspect ${paths.log}`); - if (wasRunning) return restoreManualRouter(paths, port, error); - throw error; } async function autostartDisable(paths, { quiet = false } = {}) { const kind = autostartKind(); const file = autostartFile(paths); const wasEnabled = autostartEnabled(paths); - let removed = false; - if (wasEnabled) await stopInstance(paths); - if (kind === "launchd") { - try { - execFileSync("/bin/launchctl", ["bootout", `gui/${process.getuid()}/${LAUNCHD_LABEL}`], { stdio: ["ignore", "ignore", "ignore"] }); - removed = true; - } catch { - // Not loaded. - } - } else if (kind === "schtasks") { - try { - execFileSync("schtasks", ["/end", "/tn", WINDOWS_TASK], { stdio: ["ignore", "ignore", "ignore"] }); - } catch { - // Task absent or already stopped. - } - try { - execFileSync("schtasks", ["/delete", "/tn", WINDOWS_TASK, "/f"], { stdio: ["ignore", "ignore", "ignore"] }); - removed = true; - } catch { - // Task absent. - } - } else { - try { - execFileSync("systemctl", ["--user", "disable", "--now", SYSTEMD_UNIT], { stdio: ["ignore", "ignore", "ignore"] }); - removed = true; - } catch { - // Unit absent or systemd unavailable. - } - try { - execFileSync("systemctl", ["--user", "daemon-reload"], { stdio: ["ignore", "ignore", "ignore"] }); - } catch { - // Best effort. - } - } - if (existsSync(file)) { - unlinkSync(file); - removed = true; - } + const removed = wasEnabled || existsSync(file); + await cleanupManagedAutostart(paths, { + kind, + file, + stopRouter: wasEnabled, + managerExpected: wasEnabled, + message: `DSCodex autostart cleanup encountered errors (${kind})`, + }); if (quiet) return; if (removed) { console.log(`DSCodex autostart disabled (${kind}); the managed router was stopped`); @@ -1007,6 +1192,33 @@ The router re-execs itself with Node's --use-env-proxy (Node >= 24.5); loopback api.deepseek.com stay outside the proxy while GPT passthrough and vision use it.`); } +async function uninstallAll(paths) { + const failures = []; + const operations = [ + () => autostartDisable(paths, { quiet: true }), + () => stopInstance(paths), + () => uninstall({ paths }), + () => deactivateBridge(paths), + ]; + for (const operation of operations) { + try { + await operation(); + } catch (error) { + failures.push(error); + } + } + if (failures.length) { + const details = failures.map((error) => ( + error instanceof Error ? error.message : String(error) + )).join("; "); + throw new AggregateError( + failures, + `Uninstall completed all possible cleanup steps but encountered errors: ${details}`, + ); + } + console.log("Removed DSCodex-owned config, catalog, selection state, autostart entry, and app-server bridge"); +} + async function main() { const [command = "help", ...args] = process.argv.slice(2); const port = parsePort(args); @@ -1045,13 +1257,7 @@ async function main() { case "status": await status(port); break; case "doctor": await doctor(port); break; case "stop": await stop(); break; - case "uninstall": - await autostartDisable(paths, { quiet: true }); - await stop(); - uninstall({ paths }); - deactivateBridge(paths); - console.log("Removed DSCodex-owned config, catalog, selection state, autostart entry, and app-server bridge"); - break; + case "uninstall": await uninstallAll(paths); break; case "--version": case "version": console.log(VERSION); break; case "help": diff --git a/src/supervisor.mjs b/src/supervisor.mjs index 3cda688..587fe59 100644 --- a/src/supervisor.mjs +++ b/src/supervisor.mjs @@ -1,8 +1,480 @@ -import { closeSync, mkdirSync, openSync, writeSync } from "node:fs"; +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + unlinkSync, + writeFileSync, + writeSync, +} from "node:fs"; import { spawn } from "node:child_process"; -import { dirname } from "node:path"; +import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; +import { dirname, join } from "node:path"; export const SUPERVISOR_RESTART_DELAY_MS = 2_000; +const SUPERVISOR_STATE_FILE = "supervisor.pid"; +const SUPERVISOR_POLL_MS = 25; +const SUPERVISOR_LIVENESS_TIMEOUT_MS = 750; +const SUPERVISOR_CONTROL_SUFFIX = ".control"; +const SUPERVISOR_OWNER_FILE = "owner.json"; +const SUPERVISOR_CHALLENGE_PREFIX = "challenge-"; +const SUPERVISOR_CLAIM_ATTEMPTS = 32; + +function tokenValue(value) { + return typeof value === "string" && /^[A-Za-z0-9_-]{43}$/.test(value); +} + +function supervisorIdentity(value) { + return Number.isInteger(value?.pid) && value.pid > 0 + && typeof value.instanceId === "string" + && new RegExp(`^${value.pid}-\\d+-[0-9a-f]{16}$`).test(value.instanceId) + && tokenValue(value.stopToken); +} + +export function supervisorStatePath(logPath) { + if (!logPath) throw new Error("Supervisor log path is required"); + return join(dirname(logPath), SUPERVISOR_STATE_FILE); +} + +export function supervisorControlPath(logPath) { + return `${supervisorStatePath(logPath)}${SUPERVISOR_CONTROL_SUFFIX}`; +} + +function supervisorOwnerPath(logPath) { + return join(supervisorControlPath(logPath), SUPERVISOR_OWNER_FILE); +} + +function supervisorRequestPath(logPath, instanceId) { + return `${supervisorStatePath(logPath)}.stop-${instanceId}`; +} + +function atomicWrite(path, value) { + const temporary = `${path}.tmp-${process.pid}-${randomBytes(8).toString("hex")}`; + try { + writeFileSync(temporary, `${JSON.stringify(value)}\n`, { mode: 0o600 }); + renameSync(temporary, path); + } finally { + try { unlinkSync(temporary); } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + } +} + +function readJson(path) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + if (error?.code === "ENOENT" || error instanceof SyntaxError) return null; + throw error; + } +} + +function sameSupervisorIdentity(left, right) { + return supervisorIdentity(left) + && supervisorIdentity(right) + && left.pid === right.pid + && left.instanceId === right.instanceId + && left.stopToken === right.stopToken; +} + +function readSupervisorOwner(logPath) { + const owner = readJson(supervisorOwnerPath(logPath)); + return supervisorIdentity(owner) ? owner : null; +} + +function ownsSupervisorControl(logPath, state) { + return sameSupervisorIdentity(readSupervisorOwner(logPath), state); +} + +function ensureSupervisorStatePublished(logPath, state) { + if (sameSupervisorIdentity(readJson(supervisorStatePath(logPath)), state)) return true; + if (!ownsSupervisorControl(logPath, state)) return false; + atomicWrite(supervisorStatePath(logPath), state); + if (ownsSupervisorControl(logPath, state)) return true; + + // We were fenced during publication. Remove only the identity just written; + // the replacement owner's pump will republish its own generation if needed. + removeSupervisorState(logPath, state); + return false; +} + +function validateControlTimings(livenessTimeoutMs, pollIntervalMs) { + if (!Number.isInteger(livenessTimeoutMs) || livenessTimeoutMs < 1) { + throw new Error(`Invalid supervisor liveness timeout: ${livenessTimeoutMs}`); + } + if (!Number.isInteger(pollIntervalMs) || pollIntervalMs < 1) { + throw new Error(`Invalid supervisor poll interval: ${pollIntervalMs}`); + } +} + +function wait(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +function challengePath(logPath, nonce) { + return join(supervisorControlPath(logPath), `${SUPERVISOR_CHALLENGE_PREFIX}${nonce}.json`); +} + +function responsePath(logPath, nonce) { + return `${challengePath(logPath, nonce)}.response`; +} + +function livenessProof(state, nonce, issuedAt) { + return createHmac("sha256", Buffer.from(state.stopToken, "base64url")) + .update(`dscodex-supervisor-live\0${state.instanceId}\0${nonce}\0${issuedAt}`) + .digest("base64url"); +} + +function proofMatches(actual, expected) { + if (typeof actual !== "string" || typeof expected !== "string") return false; + const left = Buffer.from(actual); + const right = Buffer.from(expected); + return left.length === right.length && timingSafeEqual(left, right); +} + +export function readSupervisorState(logPath) { + const path = supervisorStatePath(logPath); + if (!existsSync(path)) return null; + let state; + try { + state = JSON.parse(readFileSync(path, "utf8")); + } catch { + throw new Error(`Invalid DSCodex supervisor state at ${path}`); + } + if (!supervisorIdentity(state)) { + throw new Error(`Untrusted DSCodex supervisor state at ${path}; refusing to control an unverified process`); + } + return state; +} + +// The request is authenticated with a per-instance token and names the exact +// supervisor instance. It never sends a signal to a possibly recycled PID. +export function requestSupervisorStop(logPath, state) { + if (!supervisorIdentity(state)) { + throw new Error("Refusing to stop a supervisor without a trusted instance identity"); + } + const path = supervisorRequestPath(logPath, state.instanceId); + atomicWrite(path, { instanceId: state.instanceId, stopToken: state.stopToken }); + return path; +} + +function stopRequested(logPath, state) { + const path = supervisorRequestPath(logPath, state.instanceId); + if (!existsSync(path)) return false; + try { + const request = JSON.parse(readFileSync(path, "utf8")); + return request?.instanceId === state.instanceId && request?.stopToken === state.stopToken; + } catch { + return false; + } +} + +function removeMatchingFile(path, expected) { + const before = readJson(path); + if (!matchesExpected(before, expected)) return; + const claimed = `${path}.remove-${process.pid}-${randomBytes(8).toString("hex")}`; + try { + renameSync(path, claimed); + } catch (error) { + if (error?.code === "ENOENT") return; + if (error?.code === "EPERM" || error?.code === "EACCES") return; + throw error; + } + let current = null; + let raw = null; + try { + raw = readFileSync(claimed, "utf8"); + current = JSON.parse(raw); + } catch { + // Restore invalid or replacement state below. + } + if (matchesExpected(current, expected)) { + try { + unlinkSync(claimed); + } catch (error) { + if (error?.code !== "EPERM" && error?.code !== "EACCES") throw error; + try { writeFileSync(claimed, "", { mode: 0o600 }); } catch {} + } + return; + } + if (raw !== null) { + try { + // Reserving the pathname with `wx` never overwrites state published by a + // replacement and works on filesystems which do not support hard links. + writeFileSync(path, raw, { flag: "wx", mode: 0o600 }); + } catch (error) { + if (error?.code !== "EEXIST") throw error; + } + } + try { + unlinkSync(claimed); + } catch (error) { + if (error?.code !== "EPERM" && error?.code !== "EACCES") throw error; + // The uniquely named displaced copy is harmless if Windows still has it open. + } +} + +function matchesExpected(current, expected) { + return current?.instanceId === expected.instanceId + && (expected.pid === undefined || current?.pid === expected.pid) + && (expected.stopToken === undefined || current?.stopToken === expected.stopToken); +} + +export function removeSupervisorState(logPath, expected) { + if (!supervisorIdentity(expected)) { + throw new Error("Refusing to remove supervisor state without a trusted instance identity"); + } + removeMatchingFile(supervisorStatePath(logPath), expected); +} + +function respondToSupervisorChallenges(logPath, state) { + if (!ownsSupervisorControl(logPath, state)) return false; + let names; + try { + names = readdirSync(supervisorControlPath(logPath)); + } catch (error) { + if (error?.code === "ENOENT") return false; + throw error; + } + for (const name of names) { + const match = /^challenge-([0-9a-f]{32})\.json$/.exec(name); + if (!match) continue; + const nonce = match[1]; + const request = readJson(challengePath(logPath, nonce)); + if (request?.instanceId !== state.instanceId + || request?.nonce !== nonce + || !Number.isSafeInteger(request?.issuedAt)) { + continue; + } + const proof = livenessProof(state, nonce, request.issuedAt); + const existing = readJson(responsePath(logPath, nonce)); + if (existing?.instanceId === state.instanceId + && existing?.nonce === nonce + && existing?.issuedAt === request.issuedAt + && proofMatches(existing?.proof, proof)) { + continue; + } + if (!ownsSupervisorControl(logPath, state)) return false; + atomicWrite(responsePath(logPath, nonce), { + instanceId: state.instanceId, + nonce, + issuedAt: request.issuedAt, + proof, + }); + } + return ownsSupervisorControl(logPath, state); +} + +async function challengeSupervisorOwner(logPath, target, { + livenessTimeoutMs, + pollIntervalMs, +}) { + const nonce = randomBytes(16).toString("hex"); + const issuedAt = Date.now(); + const requestPath = challengePath(logPath, nonce); + const replyPath = responsePath(logPath, nonce); + const expectedProof = livenessProof(target, nonce, issuedAt); + try { + atomicWrite(requestPath, { instanceId: target.instanceId, nonce, issuedAt }); + } catch (error) { + if (error?.code === "ENOENT") return "changed"; + throw error; + } + + const deadline = Date.now() + livenessTimeoutMs; + try { + while (true) { + const current = readSupervisorOwner(logPath); + if (!sameSupervisorIdentity(current, target)) return "changed"; + const response = readJson(replyPath); + if (response?.instanceId === target.instanceId + && response?.nonce === nonce + && response?.issuedAt === issuedAt + && proofMatches(response?.proof, expectedProof)) { + return "live"; + } + const remaining = deadline - Date.now(); + if (remaining <= 0) return "stale"; + await wait(Math.min(pollIntervalMs, remaining)); + } + } finally { + for (const path of [requestPath, replyPath]) { + try { + unlinkSync(path); + } catch (error) { + if (error?.code !== "ENOENT" && error?.code !== "EPERM" && error?.code !== "EACCES") { + throw error; + } + } + } + } +} + +export async function authenticateSupervisorOwner(logPath, state, { + livenessTimeoutMs = SUPERVISOR_LIVENESS_TIMEOUT_MS, + pollIntervalMs = SUPERVISOR_POLL_MS, +} = {}) { + if (!supervisorIdentity(state)) return false; + validateControlTimings(livenessTimeoutMs, pollIntervalMs); + const owner = readSupervisorOwner(logPath); + if (!sameSupervisorIdentity(owner, state)) return false; + return await challengeSupervisorOwner(logPath, owner, { + livenessTimeoutMs, + pollIntervalMs, + }) === "live"; +} + +// Alias kept intentionally terse for CLI callers which only need a boolean +// replacement for PID-based liveness checks. +export const verifySupervisorOwner = authenticateSupervisorOwner; + +function restoreDisplacedControl(logPath, quarantine) { + const path = supervisorControlPath(logPath); + try { + // Renaming a directory over an already populated replacement directory is + // rejected on supported platforms. Thus either this restores the displaced + // generation, or the generation currently at `path` remains the singleton. + renameSync(quarantine, path); + return true; + } catch (error) { + const replacementExists = existsSync(path); + if (replacementExists && ["EEXIST", "ENOTEMPTY", "EPERM", "EACCES"].includes(error?.code)) { + return false; + } + throw error; + } +} + +function quarantineSupervisorControl(logPath, state, expectedOwner) { + const path = supervisorControlPath(logPath); + const quarantine = `${path}.stale-${state.instanceId}-${randomBytes(8).toString("hex")}`; + try { + renameSync(path, quarantine); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } + const displaced = readJson(join(quarantine, SUPERVISOR_OWNER_FILE)); + const expectedMatches = expectedOwner + ? sameSupervisorIdentity(displaced, expectedOwner) + : !supervisorIdentity(displaced); + if (expectedMatches) return quarantine; + + // The owner changed between the pre-rename check and rename. Never delete + // that replacement generation; restore it when the canonical path is free. + // If a still newer generation already occupies the path, leaving this one in + // quarantine fences it without disturbing the current singleton. + restoreDisplacedControl(logPath, quarantine); + return null; +} + +function removeControlTree(path) { + try { + rmSync(path, { recursive: true, force: true }); + } catch (error) { + if (error?.code !== "ENOENT" && error?.code !== "EPERM" && error?.code !== "EACCES") { + throw error; + } + } +} + +function tryClaimSupervisorControl(logPath, state) { + const controlPath = supervisorControlPath(logPath); + try { + mkdirSync(controlPath, { mode: 0o700 }); + } catch (error) { + if (error?.code === "EEXIST") return "occupied"; + throw error; + } + + try { + writeFileSync(supervisorOwnerPath(logPath), `${JSON.stringify(state)}\n`, { + flag: "wx", + mode: 0o600, + }); + if (!ownsSupervisorControl(logPath, state)) return "retry"; + atomicWrite(supervisorStatePath(logPath), state); + if (!ownsSupervisorControl(logPath, state)) { + removeSupervisorState(logPath, state); + return "retry"; + } + return "acquired"; + } catch (error) { + // Leave a partially initialized generation in place. A later claimant will + // give it the normal liveness grace period and reclaim it safely; cleanup + // here would race with a replacement generation at the fixed path. + throw error; + } +} + +async function acquireSupervisorState(logPath, state, timings) { + for (let attempt = 0; attempt < SUPERVISOR_CLAIM_ATTEMPTS; attempt += 1) { + const claim = tryClaimSupervisorControl(logPath, state); + if (claim === "acquired") return true; + if (claim === "retry") continue; + + const observed = readSupervisorOwner(logPath); + if (observed) { + const result = await challengeSupervisorOwner(logPath, observed, timings); + if (result === "live") return false; + if (result === "changed") continue; + const latest = readSupervisorOwner(logPath); + if (!sameSupervisorIdentity(latest, observed)) continue; + } else { + // An owner may have crashed between mkdir and publishing owner.json. Give + // an initializing process the same grace period as an established owner. + await wait(timings.livenessTimeoutMs); + if (readSupervisorOwner(logPath)) continue; + } + + const quarantine = quarantineSupervisorControl(logPath, state, observed); + if (!quarantine) continue; + removeControlTree(quarantine); + } + throw new Error(`Could not claim DSCodex supervisor state at ${supervisorStatePath(logPath)}`); +} + +function startSupervisorControlPump( + logPath, + state, + pollIntervalMs, + livenessTimeoutMs, + onLost, +) { + let stopped = false; + let mismatchSince = null; + const noteUnconfirmed = () => { + if (mismatchSince === null) mismatchSince = Date.now(); + if (Date.now() - mismatchSince >= livenessTimeoutMs) onLost(); + }; + const service = () => { + if (stopped) return; + try { + if (respondToSupervisorChallenges(logPath, state) + && ensureSupervisorStatePublished(logPath, state)) { + mismatchSince = null; + return; + } + noteUnconfirmed(); + } catch { + // A single transient error is tolerated. A sustained inability to prove + // ownership must fence this instance, because a claimant can otherwise + // take over while this process keeps its old router child alive. + noteUnconfirmed(); + } + }; + service(); + const timer = setInterval(service, pollIntervalMs); + timer.unref?.(); + return () => { + stopped = true; + clearInterval(timer); + }; +} function appendSupervisorLog(fd, message) { try { @@ -25,8 +497,16 @@ function childExit(child) { }); } -function wait(milliseconds) { - return new Promise((resolve) => setTimeout(resolve, milliseconds)); +async function waitForRestart(milliseconds, shouldStop, pollIntervalMs = SUPERVISOR_POLL_MS) { + const deadline = Date.now() + milliseconds; + while (Date.now() < deadline) { + if (shouldStop()) return true; + await new Promise((resolve) => setTimeout( + resolve, + Math.min(pollIntervalMs, Math.max(1, deadline - Date.now())), + )); + } + return shouldStop(); } // Windows Task Scheduler launches this long-lived process through a hidden VBS @@ -41,43 +521,143 @@ export async function superviseRouter({ logPath, env = process.env, restartDelayMs = SUPERVISOR_RESTART_DELAY_MS, + livenessTimeoutMs = SUPERVISOR_LIVENESS_TIMEOUT_MS, + pollIntervalMs = SUPERVISOR_POLL_MS, }) { if (!nodePath || !cliPath || !logPath) throw new Error("Supervisor paths are required"); if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`Invalid port: ${port}`); if (!Number.isInteger(restartDelayMs) || restartDelayMs < 0) { throw new Error(`Invalid restart delay: ${restartDelayMs}`); } + validateControlTimings(livenessTimeoutMs, pollIntervalMs); mkdirSync(dirname(logPath), { recursive: true, mode: 0o700 }); const logFd = openSync(logPath, "a", 0o600); + const state = { + pid: process.pid, + instanceId: `${process.pid}-${Date.now()}-${randomBytes(8).toString("hex")}`, + stopToken: randomBytes(32).toString("base64url"), + }; const childEnv = { ...env }; // Every supervised launch is a fresh top-level serve. Let it resolve the // stored proxy and perform its own --use-env-proxy re-exec when required. delete childEnv.DSCODEX_PROXY_REEXEC; + childEnv.DSCODEX_SUPERVISOR_INSTANCE_ID = state.instanceId; + let ownsState = false; + let ownershipLost = false; + let resolveOwnershipLost; + const ownershipLostPromise = new Promise((resolve) => { + resolveOwnershipLost = resolve; + }); + const markOwnershipLost = () => { + if (ownershipLost) return; + ownershipLost = true; + resolveOwnershipLost(); + }; + let stopControlPump = () => {}; try { + ownsState = await acquireSupervisorState(logPath, state, { + livenessTimeoutMs, + pollIntervalMs, + }); + if (!ownsState) { + appendSupervisorLog(logFd, "another supervisor already owns this router; exiting"); + return; + } + stopControlPump = startSupervisorControlPump( + logPath, + state, + pollIntervalMs, + livenessTimeoutMs, + markOwnershipLost, + ); appendSupervisorLog(logFd, `started (node ${process.version}, port ${port})`); while (true) { + if (ownershipLost) { + appendSupervisorLog(logFd, "supervisor ownership was replaced; exiting"); + return; + } + if (!ownsSupervisorControl(logPath, state)) { + // A competing stale-owner cleanup can move then restore this directory. + // Do not spawn while it is absent, and fence only after the control pump + // has observed a continuous mismatch for the full liveness timeout. + await Promise.race([ownershipLostPromise, wait(pollIntervalMs)]); + continue; + } + if (stopRequested(logPath, state)) { + appendSupervisorLog(logFd, "authenticated stop requested; staying stopped"); + return; + } const child = spawn(nodePath, [cliPath, "serve", "--port", String(port)], { env: childEnv, stdio: ["ignore", logFd, logFd], windowsHide: true, }); - const result = await childExit(child); + const exitPromise = childExit(child); + const outcome = await Promise.race([ + exitPromise.then((result) => ({ kind: "exit", result })), + ownershipLostPromise.then(() => ({ kind: "lost" })), + ]); + if (outcome.kind === "lost") { + appendSupervisorLog(logFd, "supervisor ownership was replaced; stopping owned router child"); + // This is the ChildProcess handle returned by our own spawn, never a PID + // recovered from disk, so a recycled or unverified PID is not signalled. + try { child.kill(); } catch {} + const exited = await Promise.race([ + exitPromise.then(() => true), + wait(1_000).then(() => false), + ]); + if (!exited && child.exitCode === null && child.signalCode === null) { + try { child.kill("SIGKILL"); } catch {} + await Promise.race([exitPromise, wait(1_000)]); + } + return; + } + const { result } = outcome; if (!result.error && result.code === 0 && !result.signal) { appendSupervisorLog(logFd, "router exited cleanly; staying stopped"); return; } + if (stopRequested(logPath, state)) { + appendSupervisorLog(logFd, "authenticated stop requested after router exit; staying stopped"); + return; + } + const detail = result.error ? `spawn failed: ${result.error.message}` : result.signal ? `router exited on ${result.signal}` : `router exited with code ${result.code}`; appendSupervisorLog(logFd, `${detail}; restarting in ${restartDelayMs}ms`); - await wait(restartDelayMs); + if (await waitForRestart( + restartDelayMs, + () => ownershipLost || stopRequested(logPath, state), + pollIntervalMs, + )) { + if (ownershipLost) { + appendSupervisorLog(logFd, "supervisor ownership was replaced during restart delay; exiting"); + return; + } + appendSupervisorLog(logFd, "authenticated stop requested during restart delay; staying stopped"); + return; + } } } finally { - closeSync(logFd); + stopControlPump(); + try { + if (ownsState && ownsSupervisorControl(logPath, state)) { + removeMatchingFile(supervisorRequestPath(logPath, state.instanceId), { + instanceId: state.instanceId, + stopToken: state.stopToken, + }); + } + if (ownsState && ownsSupervisorControl(logPath, state)) { + removeSupervisorState(logPath, state); + } + } finally { + closeSync(logFd); + } } } diff --git a/test/autostart.test.mjs b/test/autostart.test.mjs index 5eba397..794fd67 100644 --- a/test/autostart.test.mjs +++ b/test/autostart.test.mjs @@ -13,11 +13,13 @@ import { buildSystemdUnit, buildWindowsRegisterScript, buildWindowsVbs, + cleanupAutostart, encodeWindowsVbs, } from "../src/autostart.mjs"; import { buildInstalledConfig } from "../src/config.mjs"; import { pathsFor } from "../src/constants.mjs"; import { ensureRouterToken } from "../src/keys.mjs"; +import { supervisorStatePath } from "../src/supervisor.mjs"; const CLI = fileURLToPath(new URL("../src/cli.mjs", import.meta.url)); const SUPERVISOR_FIXTURE_SOURCE = [ @@ -94,6 +96,64 @@ function runCli(args, env) { }); } +async function waitForText(path, pattern, message) { + for (let attempt = 0; attempt < 200; attempt += 1) { + if (existsSync(path) && pattern.test(readFileSync(path, "utf8"))) return; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error(`Timed out waiting for ${message}`); +} + +function fakeWindowsTaskHook(temp) { + const hook = join(temp, "fake-schtasks.cjs"); + const taskLog = join(temp, "task-operations.jsonl"); + writeFileSync(hook, [ + 'const fs = require("node:fs");', + 'const childProcess = require("node:child_process");', + 'const { syncBuiltinESMExports } = require("node:module");', + 'const original = childProcess.execFileSync;', + 'childProcess.execFileSync = function(file, args, options) {', + ' if (String(file).toLowerCase() === "schtasks") {', + ' fs.appendFileSync(process.env.DSCODEX_TEST_TASK_LOG, `${JSON.stringify(args)}\\n`);', + ' const action = String(args?.[0]).toLowerCase();', + ' if (process.env.DSCODEX_TEST_FAKE_TASK === "1" && action === "/query"', + ' && !fs.existsSync(process.env.DSCODEX_TEST_TASK_STATE)) {', + ' throw new Error("simulated missing task");', + ' }', + ' if (String(args?.[0]).toLowerCase() === "/run" && process.env.DSCODEX_TEST_TASK_CLI) {', + ' const child = childProcess.spawn(process.execPath, [', + ' process.env.DSCODEX_TEST_TASK_CLI, "serve", "--port", process.env.DSCODEX_TEST_TASK_PORT,', + ' ], { env: process.env, detached: true, stdio: "ignore", windowsHide: true });', + ' child.unref();', + ' }', + ' if (process.env.DSCODEX_TEST_FAKE_TASK === "1" && action === "/delete") {', + ' try { fs.unlinkSync(process.env.DSCODEX_TEST_TASK_STATE); } catch {}', + ' }', + ' return options?.encoding ? "" : Buffer.alloc(0);', + ' }', + ' if (process.env.DSCODEX_TEST_FAIL_POWERSHELL === "1"', + ' && String(file).toLowerCase().endsWith("powershell.exe")) {', + ' throw new Error("simulated registration failure");', + ' }', + ' if (process.env.DSCODEX_TEST_FAKE_TASK === "1"', + ' && String(file).toLowerCase().endsWith("powershell.exe")) {', + ' const command = String(args?.at(-1) ?? "");', + ' if (command.includes("Register-ScheduledTask")) {', + ' fs.writeFileSync(process.env.DSCODEX_TEST_TASK_STATE, "registered");', + ' }', + ' const output = command.includes("Get-ScheduledTask")', + ' ? JSON.stringify({ state: "Ready", lastTaskResult: 0 })', + ' : "";', + ' return options?.encoding ? output : Buffer.from(output);', + ' }', + ' return original.apply(this, arguments);', + '};', + 'syncBuiltinESMExports();', + '', + ].join("\n")); + return { hook, taskLog }; +} + test("launchd plist embeds absolute paths and restarts only on failure", () => { const plist = buildLaunchdPlist({ nodePath: "/opt/homebrew/bin/node", @@ -153,6 +213,33 @@ test("windows vbs waits for the hidden Node supervisor and quotes paths safely", assert.ok(!vbs.includes("cmd /c")); }); +test("autostart cleanup runs every step and aggregates failures", async () => { + const calls = []; + let thrown; + try { + await cleanupAutostart({ + stopRouter: () => { + calls.push("stop"); + throw new Error("invalid pid state"); + }, + deactivateManager: () => { + calls.push("deactivate"); + throw new Error("manager delete failed"); + }, + removeArtifact: () => calls.push("remove"), + reloadManager: () => calls.push("reload"), + message: "cleanup failed", + }); + } catch (error) { + thrown = error; + } + assert.ok(thrown instanceof AggregateError); + assert.deepEqual(calls, ["stop", "deactivate", "remove", "reload"]); + assert.equal(thrown.errors.length, 2); + assert.match(thrown.message, /invalid pid state/); + assert.match(thrown.message, /manager delete failed/); +}); + test("windows task registration is user-scoped and restarts router crashes", () => { const script = buildWindowsRegisterScript({ taskName: "DSCodex", @@ -289,6 +376,219 @@ test("failed Windows autostart registration leaves a healthy manual router runni } }); +test("failed Windows task replacement preserves the existing task and VBS", { + skip: process.platform !== "win32" ? "Windows manager integration test" : false, +}, async () => { + const codexHome = mkdtempSync(join(tmpdir(), "dscodex-autostart-existing-task-")); + const paths = prepareRouterHome(codexHome, 10110); + const vbsPath = join(paths.stateDir, "autostart-run.vbs"); + const previousVbs = Buffer.from("existing task shim", "utf8"); + const { hook, taskLog } = fakeWindowsTaskHook(codexHome); + writeFileSync(vbsPath, previousVbs); + try { + const result = await runCli(["autostart", "enable"], { + ...routerEnv(codexHome), + NODE_OPTIONS: `--require=${hook}`, + DSCODEX_TEST_TASK_LOG: taskLog, + DSCODEX_TEST_FAIL_POWERSHELL: "1", + }); + assert.notEqual(result.code, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stderr, /simulated registration failure/); + assert.deepEqual(readFileSync(vbsPath), previousVbs); + const operations = readFileSync(taskLog, "utf8"); + assert.doesNotMatch(operations, /"\/end"|"\/delete"/i); + } finally { + rmSync(codexHome, { recursive: true, force: true }); + } +}); + +test("Windows rollback stops a newly started task router when no manual router existed", { + timeout: 20_000, + skip: process.platform !== "win32" ? "Windows manager integration test" : false, +}, async () => { + const codexHome = mkdtempSync(join(tmpdir(), "dscodex-autostart-new-task-rollback-")); + const port = 20_000 + Math.floor(Math.random() * 20_000); + const paths = prepareRouterHome(codexHome, port); + const { hook, taskLog } = fakeWindowsTaskHook(codexHome); + const env = { + ...routerEnv(codexHome), + NODE_OPTIONS: `--require=${hook}`, + DSCODEX_TEST_TASK_LOG: taskLog, + DSCODEX_TEST_TASK_STATE: join(codexHome, "fake-task.state"), + DSCODEX_TEST_FAKE_TASK: "1", + DSCODEX_TEST_TASK_CLI: CLI, + DSCODEX_TEST_TASK_PORT: String(port), + }; + try { + const result = await runCli(["autostart", "enable", "--port", String(port)], env); + assert.notEqual(result.code, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stderr, /without a live scheduled supervisor/); + await waitForPortToClose(port); + assert.equal(existsSync(paths.pid), false); + assert.equal(existsSync(join(paths.stateDir, "autostart-run.vbs")), false); + const operations = readFileSync(taskLog, "utf8"); + assert.match(operations, /"\/run"/i); + assert.match(operations, /"\/end"/i); + assert.match(operations, /"\/delete"/i); + } finally { + try { await runCli(["stop", "--port", String(port)], env); } catch {} + rmSync(codexHome, { recursive: true, force: true }); + } +}); + +test("autostart disable removes the Windows manager and artifact after a pid-state failure", { + skip: process.platform !== "win32" ? "Windows manager integration test" : false, +}, async () => { + const codexHome = mkdtempSync(join(tmpdir(), "dscodex-autostart-disable-corrupt-")); + const paths = prepareRouterHome(codexHome, 10110); + const vbsPath = join(paths.stateDir, "autostart-run.vbs"); + const { hook, taskLog } = fakeWindowsTaskHook(codexHome); + writeFileSync(vbsPath, "fixture"); + writeFileSync(paths.pid, "not trusted json\n"); + try { + const result = await runCli(["autostart", "disable"], { + ...routerEnv(codexHome), + NODE_OPTIONS: `--require=${hook}`, + DSCODEX_TEST_TASK_LOG: taskLog, + }); + assert.notEqual(result.code, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stderr, /Invalid DSCodex pid state/); + assert.equal(existsSync(vbsPath), false); + const operations = readFileSync(taskLog, "utf8"); + assert.match(operations, /"\/end"/i); + assert.match(operations, /"\/delete"/i); + } finally { + rmSync(codexHome, { recursive: true, force: true }); + } +}); + +test("uninstall continues config cleanup after autostart and pid-state failures", { + skip: process.platform !== "win32" ? "Windows manager integration test" : false, +}, async () => { + const codexHome = mkdtempSync(join(tmpdir(), "dscodex-uninstall-corrupt-")); + const paths = prepareRouterHome(codexHome, 10110); + const vbsPath = join(paths.stateDir, "autostart-run.vbs"); + const { hook, taskLog } = fakeWindowsTaskHook(codexHome); + writeFileSync(vbsPath, "fixture"); + writeFileSync(paths.pid, "not trusted json\n"); + writeFileSync(paths.catalog, '{"models":[]}\n'); + writeFileSync(paths.selectionState, "{}\n"); + try { + const result = await runCli(["uninstall"], { + ...routerEnv(codexHome), + NODE_OPTIONS: `--require=${hook}`, + DSCODEX_TEST_TASK_LOG: taskLog, + }); + assert.notEqual(result.code, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stderr, /Uninstall completed all possible cleanup steps/); + assert.equal(existsSync(vbsPath), false); + assert.equal(existsSync(paths.catalog), false); + assert.equal(existsSync(paths.keyFile), false); + assert.equal(existsSync(paths.selectionState), false); + assert.doesNotMatch(readFileSync(paths.config, "utf8"), /DSCodex managed/); + const operations = readFileSync(taskLog, "utf8"); + assert.match(operations, /"\/delete"/i); + } finally { + rmSync(codexHome, { recursive: true, force: true }); + } +}); + +test("stop treats a reused live supervisor PID without an authenticated owner as stale", { + timeout: 12_000, +}, async () => { + const codexHome = mkdtempSync(join(tmpdir(), "dscodex-supervisor-reused-pid-stop-")); + const paths = prepareRouterHome(codexHome, 10110); + const stale = { + pid: process.pid, + instanceId: `${process.pid}-${Date.now()}-0123456789abcdef`, + stopToken: "A".repeat(43), + }; + writeFileSync(supervisorStatePath(paths.log), `${JSON.stringify(stale)}\n`); + try { + const stopped = await runCli(["stop"], routerEnv(codexHome)); + assert.equal(stopped.code, 0, `${stopped.stdout}\n${stopped.stderr}`); + assert.match(stopped.stdout, /Stopped DSCodex/); + assert.equal(existsSync(supervisorStatePath(paths.log)), false); + } finally { + rmSync(codexHome, { recursive: true, force: true }); + } +}); + +test("stop keeps a supervised router down when invoked during crash backoff", { + timeout: 15_000, +}, async () => { + const codexHome = mkdtempSync(join(tmpdir(), "dscodex-supervised-stop-")); + const blocker = createServer((_request, response) => response.end()); + blocker.listen(0, "127.0.0.1"); + await once(blocker, "listening"); + const port = blocker.address().port; + const paths = prepareRouterHome(codexHome, port); + const env = routerEnv(codexHome); + const supervisor = spawn(process.execPath, [CLI, "supervise", "--port", String(port)], { + env, + stdio: "ignore", + }); + const supervisorExited = once(supervisor, "exit"); + try { + await waitForFile(supervisorStatePath(paths.log), "supervisor pid state"); + await waitForText(paths.log, /restarting in 2000ms/, "supervisor crash backoff"); + + const stopped = await runCli(["stop", "--port", String(port)], env); + assert.equal(stopped.code, 0, `${stopped.stdout}\n${stopped.stderr}`); + assert.match(stopped.stdout, /Stopped DSCodex/); + const [code, signal] = await supervisorExited; + assert.equal(code, 0); + assert.equal(signal, null); + + await new Promise((resolve) => setTimeout(resolve, 2_100)); + assert.equal(existsSync(supervisorStatePath(paths.log)), false); + assert.equal(existsSync(paths.pid), false); + } finally { + supervisor.kill("SIGKILL"); + blocker.closeAllConnections?.(); + await new Promise((resolve) => blocker.close(resolve)); + rmSync(codexHome, { recursive: true, force: true }); + } +}); + +test("stop cleanly shuts down a live supervised router and its supervisor", { + timeout: 15_000, +}, async () => { + const codexHome = mkdtempSync(join(tmpdir(), "dscodex-supervised-live-stop-")); + const reservation = createServer(); + reservation.listen(0, "127.0.0.1"); + await once(reservation, "listening"); + const port = reservation.address().port; + await new Promise((resolve) => reservation.close(resolve)); + + const paths = prepareRouterHome(codexHome, port); + const env = routerEnv(codexHome); + const supervisor = spawn(process.execPath, [CLI, "supervise", "--port", String(port)], { + env, + stdio: "ignore", + }); + const supervisorExited = once(supervisor, "exit"); + try { + await waitForFile(supervisorStatePath(paths.log), "supervisor pid state"); + await waitForFile(paths.pid, "router pid state"); + + const stopped = await runCli(["stop", "--port", String(port)], env); + assert.equal(stopped.code, 0, `${stopped.stdout}\n${stopped.stderr}`); + assert.match(stopped.stdout, /Stopped DSCodex/); + + const [code, signal] = await supervisorExited; + assert.equal(code, 0); + assert.equal(signal, null); + await waitForPortToClose(port); + assert.equal(existsSync(supervisorStatePath(paths.log)), false); + assert.equal(existsSync(paths.pid), false); + } finally { + supervisor.kill("SIGKILL"); + try { await runCli(["stop", "--port", String(port)], env); } catch {} + rmSync(codexHome, { recursive: true, force: true }); + } +}); + test("serve owns its pid file across start and graceful shutdown", { timeout: 20_000 }, async () => { const codexHome = mkdtempSync(join(tmpdir(), "dscodex-serve-")); const port = 20_000 + Math.floor(Math.random() * 20_000); diff --git a/test/config.test.mjs b/test/config.test.mjs index 79073de..ce9107e 100644 --- a/test/config.test.mjs +++ b/test/config.test.mjs @@ -3,7 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; -import { buildCatalog } from "../src/catalog.mjs"; +import { buildCatalog, isCatalogReady } from "../src/catalog.mjs"; import { buildInstalledConfig, ensureManagedRouterBinding, @@ -69,6 +69,49 @@ test("catalog adds distinct whale-labelled V4 Flash and Pro entries", () => { } }); +test("catalog readiness accepts a generated catalog with both DeepSeek models", () => { + assert.equal(isCatalogReady(buildCatalog({ models: [TEMPLATE] })), true); +}); + +test("catalog readiness rejects native-only and partially merged catalogs", () => { + const catalog = buildCatalog({ models: [TEMPLATE] }); + const nativeOnly = { + models: catalog.models.filter((model) => !model.slug.startsWith("deepseek/")), + }; + const missingPro = { + models: catalog.models.filter((model) => model.slug !== "deepseek/deepseek-v4-pro"), + }; + + assert.equal(isCatalogReady(nativeOnly), false); + assert.equal(isCatalogReady(missingPro), false); +}); + +test("catalog readiness rejects entries missing current required fields", () => { + for (const field of [ + "slug", + "base_instructions", + "prefer_websockets", + "supports_reasoning_summaries", + ]) { + const catalog = buildCatalog({ models: [TEMPLATE] }); + const native = catalog.models.find((model) => model.slug === TEMPLATE.slug); + delete native[field]; + assert.equal(isCatalogReady(catalog), false, `missing ${field} should fail readiness`); + } +}); + +test("catalog readiness rejects malformed field types and duplicate slugs", () => { + const wrongType = buildCatalog({ models: [TEMPLATE] }); + wrongType.models[0].prefer_websockets = "false"; + assert.equal(isCatalogReady(wrongType), false); + + const duplicate = buildCatalog({ models: [TEMPLATE] }); + duplicate.models.push({ ...duplicate.models[0] }); + assert.equal(isCatalogReady(duplicate), false); + assert.equal(isCatalogReady({ models: [] }), false); + assert.equal(isCatalogReady(null), false); +}); + test("config injection is root-correct, reversible, and preserves user config", () => { const original = 'personality = "pragmatic"\n\n[features]\nmulti_agent = true\n\n[desktop]\ntheme = "light"\n'; const installed = buildInstalledConfig(original, { port: 10110, catalogPath: "/tmp/models.json", routerToken: ROUTER_TOKEN }); diff --git a/test/supervisor-recovery.test.mjs b/test/supervisor-recovery.test.mjs new file mode 100644 index 0000000..2c5773b --- /dev/null +++ b/test/supervisor-recovery.test.mjs @@ -0,0 +1,304 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmdirSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { createRequire, syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import test from "node:test"; +import { + authenticateSupervisorOwner, + readSupervisorState, + removeSupervisorState, + requestSupervisorStop, + superviseRouter, + supervisorStatePath, +} from "../src/supervisor.mjs"; + +const SUPERVISOR_MODULE = fileURLToPath(new URL("../src/supervisor.mjs", import.meta.url)); +const mutableFs = createRequire(import.meta.url)("node:fs"); + +const FIXTURE_SOURCE = [ + 'import { readFileSync, writeFileSync } from "node:fs";', + 'const attemptsPath = process.env.DSCODEX_TEST_SUPERVISOR_STATE;', + 'let attempts = 0;', + 'try { attempts = Number(readFileSync(attemptsPath, "utf8")) || 0; } catch {}', + 'attempts += 1;', + 'writeFileSync(attemptsPath, String(attempts));', + 'const failures = Number(process.env.DSCODEX_TEST_SUPERVISOR_FAILURES ?? 0);', + 'process.exit(attempts <= failures ? 23 : 0);', + '', +].join("\n"); + +async function waitFor(check, message, timeoutMs = 5_000) { + const deadline = Date.now() + timeoutMs; + let lastError; + while (Date.now() < deadline) { + try { + if (check()) return; + lastError = undefined; + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`Timed out waiting for ${message}`, { cause: lastError }); +} + +function safeSupervisorState(logPath) { + try { + return readSupervisorState(logPath); + } catch { + return null; + } +} + +function supervisorOptions({ fixturePath, attemptsPath, logPath, failures, restartDelayMs }) { + return { + nodePath: process.execPath, + cliPath: fixturePath, + port: 10110, + logPath, + restartDelayMs, + env: { + ...process.env, + DSCODEX_TEST_SUPERVISOR_STATE: attemptsPath, + DSCODEX_TEST_SUPERVISOR_FAILURES: String(failures), + DSCODEX_PROXY_REEXEC: "1", + }, + }; +} + +function staleIdentity(pid = process.pid) { + return { + pid, + instanceId: `${pid}-${Math.max(1, Date.now() - 60_000)}-0123456789abcdef`, + stopToken: "A".repeat(43), + }; +} + +function removeTestTree(path) { + if (!existsSync(path)) return; + for (const entry of readdirSync(path, { withFileTypes: true })) { + const child = join(path, entry.name); + if (entry.isDirectory() && !entry.isSymbolicLink()) removeTestTree(child); + else unlinkSync(child); + } + rmdirSync(path); +} + +async function terminate(child) { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, "exit"); + child.kill("SIGKILL"); + await Promise.race([ + exited, + new Promise((_, reject) => setTimeout( + () => reject(new Error(`Timed out terminating test helper PID ${child.pid}`)), + 3_000, + )), + ]); +} + +async function settleAfterStop(logPath, promises) { + const state = safeSupervisorState(logPath); + if (state) { + try { requestSupervisorStop(logPath, state); } catch {} + } + await Promise.race([ + Promise.allSettled(promises.filter(Boolean)), + new Promise((resolve) => setTimeout(resolve, 2_000)), + ]); +} + +test("a reused live PID without an owner response is stale and cannot suppress startup", { + timeout: 10_000, +}, async () => { + const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-reused-pid-")); + const fixturePath = join(temp, "router-fixture.mjs"); + const attemptsPath = join(temp, "attempts.txt"); + const logPath = join(temp, "server.log"); + try { + writeFileSync(fixturePath, FIXTURE_SOURCE); + const stale = staleIdentity(); + writeFileSync(supervisorStatePath(logPath), `${JSON.stringify(stale)}\n`); + const oldRequestPath = requestSupervisorStop(logPath, stale); + assert.equal(await authenticateSupervisorOwner(logPath, stale, { + livenessTimeoutMs: 50, + pollIntervalMs: 5, + }), false); + + await superviseRouter(supervisorOptions({ + fixturePath, + attemptsPath, + logPath, + failures: 0, + restartDelayMs: 10, + })); + + assert.equal(readFileSync(attemptsPath, "utf8"), "1"); + assert.equal(readSupervisorState(logPath), null); + assert.doesNotMatch(readFileSync(logPath, "utf8"), /another supervisor already owns/); + if (existsSync(oldRequestPath)) { + const oldRequest = JSON.parse(readFileSync(oldRequestPath, "utf8")); + assert.equal(oldRequest.instanceId, stale.instanceId); + assert.equal(oldRequest.stopToken, stale.stopToken); + } + } finally { + removeTestTree(temp); + } +}); + +for (const hardLinkError of ["EPERM", "ENOTSUP"]) { + test(`supervisor remains mutually exclusive and replacement-safe when linkSync throws ${hardLinkError}`, { + timeout: 15_000, + }, async () => { + const temp = mkdtempSync(join(tmpdir(), `dscodex-supervisor-no-link-${hardLinkError.toLowerCase()}-`)); + const fixturePath = join(temp, "router-fixture.mjs"); + const attemptsPath = join(temp, "attempts.txt"); + const logPath = join(temp, "server.log"); + const originalLinkSync = mutableFs.linkSync; + const promises = []; + try { + writeFileSync(fixturePath, FIXTURE_SOURCE); + mutableFs.linkSync = () => { + const error = new Error(`hard links unavailable in test (${hardLinkError})`); + error.code = hardLinkError; + throw error; + }; + syncBuiltinESMExports(); + + const options = supervisorOptions({ + fixturePath, + attemptsPath, + logPath, + failures: 100, + restartDelayMs: 10_000, + }); + const ownerPromise = superviseRouter(options); + promises.push(ownerPromise); + const contenderPromise = superviseRouter(options); + promises.push(contenderPromise); + + await contenderPromise; + await waitFor( + () => safeSupervisorState(logPath) !== null + && existsSync(attemptsPath) + && readFileSync(attemptsPath, "utf8") === "1", + "one supervisor to launch exactly one child", + ); + const firstOwner = readSupervisorState(logPath); + assert.equal(await authenticateSupervisorOwner(logPath, firstOwner, { + livenessTimeoutMs: 1_000, + pollIntervalMs: 5, + }), true); + const samePidImpostor = staleIdentity(firstOwner.pid); + assert.equal(await authenticateSupervisorOwner(logPath, samePidImpostor, { + livenessTimeoutMs: 50, + pollIntervalMs: 5, + }), false); + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.equal(readFileSync(attemptsPath, "utf8"), "1"); + assert.deepEqual(readSupervisorState(logPath), firstOwner); + + requestSupervisorStop(logPath, firstOwner); + await ownerPromise; + assert.equal(readSupervisorState(logPath), null); + + const replacementPromise = superviseRouter(options); + promises.push(replacementPromise); + await waitFor( + () => safeSupervisorState(logPath)?.instanceId !== undefined + && safeSupervisorState(logPath)?.instanceId !== firstOwner.instanceId, + "replacement supervisor identity", + ); + const replacement = readSupervisorState(logPath); + + // Model a delayed finally block from the old owner after a replacement + // has published. Cleanup is allowed to remove only its exact generation. + removeSupervisorState(logPath, firstOwner); + assert.deepEqual(readSupervisorState(logPath), replacement); + + requestSupervisorStop(logPath, replacement); + await replacementPromise; + assert.equal(readSupervisorState(logPath), null); + } finally { + mutableFs.linkSync = originalLinkSync; + syncBuiltinESMExports(); + await settleAfterStop(logPath, promises); + removeTestTree(temp); + } + }); +} + +test("a crashed supervisor's state and authenticated request cannot poison its replacement", { + timeout: 15_000, +}, async () => { + const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-crash-recovery-")); + const fixturePath = join(temp, "router-fixture.mjs"); + const helperPath = join(temp, "supervisor-helper.mjs"); + const attemptsPath = join(temp, "attempts.txt"); + const logPath = join(temp, "server.log"); + let helper; + try { + writeFileSync(fixturePath, FIXTURE_SOURCE); + const helperOptions = supervisorOptions({ + fixturePath, + attemptsPath, + logPath, + failures: 1, + restartDelayMs: 30_000, + }); + writeFileSync(helperPath, [ + `import { superviseRouter } from ${JSON.stringify(pathToFileURL(SUPERVISOR_MODULE).href)};`, + `await superviseRouter(${JSON.stringify(helperOptions)});`, + "", + ].join("\n")); + + helper = spawn(process.execPath, [helperPath], { + env: process.env, + stdio: "ignore", + windowsHide: true, + }); + await waitFor( + () => safeSupervisorState(logPath)?.pid === helper.pid + && existsSync(logPath) + && /restarting in 30000ms/.test(readFileSync(logPath, "utf8")), + "the helper supervisor to own state and enter restart backoff", + ); + const crashedOwner = readSupervisorState(logPath); + assert.equal(readFileSync(attemptsPath, "utf8"), "1"); + + await terminate(helper); + helper = undefined; + const staleRequestPath = requestSupervisorStop(logPath, crashedOwner); + + await superviseRouter(supervisorOptions({ + fixturePath, + attemptsPath, + logPath, + failures: 1, + restartDelayMs: 10, + })); + + assert.equal(readFileSync(attemptsPath, "utf8"), "2"); + assert.equal(readSupervisorState(logPath), null); + if (existsSync(staleRequestPath)) { + const staleRequest = JSON.parse(readFileSync(staleRequestPath, "utf8")); + assert.equal(staleRequest.instanceId, crashedOwner.instanceId); + assert.equal(staleRequest.stopToken, crashedOwner.stopToken); + } + } finally { + await terminate(helper); + removeTestTree(temp); + } +}); diff --git a/test/supervisor.test.mjs b/test/supervisor.test.mjs index e788e3e..14bd816 100644 --- a/test/supervisor.test.mjs +++ b/test/supervisor.test.mjs @@ -1,9 +1,28 @@ import assert from "node:assert/strict"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import fs, { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { randomBytes } from "node:crypto"; +import { syncBuiltinESMExports } from "node:module"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; -import { superviseRouter } from "../src/supervisor.mjs"; +import { + authenticateSupervisorOwner, + readSupervisorState, + removeSupervisorState, + requestSupervisorStop, + superviseRouter, + supervisorControlPath, + supervisorStatePath, +} from "../src/supervisor.mjs"; const FIXTURE_SOURCE = [ 'import { readFileSync, writeFileSync } from "node:fs";', @@ -18,6 +37,36 @@ const FIXTURE_SOURCE = [ '', ].join("\n"); +const LONG_RUNNING_FIXTURE_SOURCE = [ + 'import { writeFileSync } from "node:fs";', + 'writeFileSync(process.env.DSCODEX_TEST_SUPERVISOR_STATE, "running");', + 'setInterval(() => {}, 1_000);', + '', +].join("\n"); + +const DELAYED_EXIT_FIXTURE_SOURCE = [ + 'import { writeFileSync } from "node:fs";', + 'writeFileSync(process.env.DSCODEX_TEST_SUPERVISOR_STATE, "running");', + 'setTimeout(() => process.exit(0), 400);', + '', +].join("\n"); + +function fakeSupervisorState(pid = process.pid) { + return { + pid, + instanceId: `${pid}-${Date.now()}-${randomBytes(8).toString("hex")}`, + stopToken: randomBytes(32).toString("base64url"), + }; +} + +async function waitFor(check, message) { + for (let attempt = 0; attempt < 200; attempt += 1) { + if (check()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`Timed out waiting for ${message}`); +} + test("supervisor logs stderr, restarts nonzero exits, and stops on exit zero", async () => { const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-")); const fixturePath = join(temp, "supervisor-child.mjs"); @@ -48,3 +97,317 @@ test("supervisor logs stderr, restarts nonzero exits, and stops on exit zero", a rmSync(temp, { recursive: true, force: true }); } }); + +test("authenticated stop cancels a supervisor while it is waiting to restart", async () => { + const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-stop-")); + const fixturePath = join(temp, "supervisor-child.mjs"); + const statePath = join(temp, "attempts.txt"); + const logPath = join(temp, "server.log"); + try { + writeFileSync(fixturePath, FIXTURE_SOURCE); + const supervised = superviseRouter({ + nodePath: process.execPath, + cliPath: fixturePath, + port: 10110, + logPath, + restartDelayMs: 500, + env: { + ...process.env, + DSCODEX_TEST_SUPERVISOR_STATE: statePath, + DSCODEX_TEST_SUPERVISOR_FAILURES: "100", + DSCODEX_PROXY_REEXEC: "1", + }, + }); + + await waitFor( + () => existsSync(logPath) && /restarting in 500ms/.test(readFileSync(logPath, "utf8")), + "supervisor restart delay", + ); + const state = readSupervisorState(logPath); + assert.ok(state); + const requestPath = requestSupervisorStop(logPath, state); + await supervised; + await new Promise((resolve) => setTimeout(resolve, 550)); + + assert.equal(readFileSync(statePath, "utf8"), "1"); + assert.equal(existsSync(supervisorStatePath(logPath)), false); + assert.equal(existsSync(requestPath), false); + assert.equal(readdirSync(temp).some((name) => name.startsWith("supervisor.pid.stop-")), false); + assert.match(readFileSync(logPath, "utf8"), /authenticated stop requested during restart delay/); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("a concurrent supervisor cannot replace the live supervisor identity", async () => { + const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-owner-")); + const fixturePath = join(temp, "supervisor-child.mjs"); + const statePath = join(temp, "attempts.txt"); + const logPath = join(temp, "server.log"); + const options = { + nodePath: process.execPath, + cliPath: fixturePath, + port: 10110, + logPath, + restartDelayMs: 500, + env: { + ...process.env, + DSCODEX_TEST_SUPERVISOR_STATE: statePath, + DSCODEX_TEST_SUPERVISOR_FAILURES: "100", + DSCODEX_PROXY_REEXEC: "1", + }, + }; + try { + writeFileSync(fixturePath, FIXTURE_SOURCE); + const first = superviseRouter(options); + await waitFor(() => readSupervisorState(logPath) !== null, "first supervisor state"); + const owner = readSupervisorState(logPath); + + await superviseRouter(options); + assert.deepEqual(readSupervisorState(logPath), owner); + + requestSupervisorStop(logPath, owner); + await first; + assert.match(readFileSync(logPath, "utf8"), /another supervisor already owns this router/); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("a stale authenticated state is reclaimed even when its PID belongs to a live process", async () => { + const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-recycled-pid-")); + const fixturePath = join(temp, "supervisor-child.mjs"); + const attemptsPath = join(temp, "attempts.txt"); + const logPath = join(temp, "server.log"); + const stale = fakeSupervisorState(process.pid); + try { + writeFileSync(fixturePath, FIXTURE_SOURCE); + writeFileSync(supervisorStatePath(logPath), `${JSON.stringify(stale)}\n`); + mkdirSync(supervisorControlPath(logPath)); + writeFileSync( + join(supervisorControlPath(logPath), "owner.json"), + `${JSON.stringify(stale)}\n`, + ); + const staleStopPath = requestSupervisorStop(logPath, stale); + + assert.equal(await authenticateSupervisorOwner(logPath, stale, { + livenessTimeoutMs: 30, + pollIntervalMs: 5, + }), false); + + const options = { + nodePath: process.execPath, + cliPath: fixturePath, + port: 10110, + logPath, + restartDelayMs: 10, + livenessTimeoutMs: 30, + pollIntervalMs: 5, + env: { + ...process.env, + DSCODEX_TEST_SUPERVISOR_STATE: attemptsPath, + DSCODEX_TEST_SUPERVISOR_FAILURES: "0", + }, + }; + await superviseRouter(options); + + assert.equal(readFileSync(attemptsPath, "utf8"), "1"); + assert.equal(existsSync(staleStopPath), true); + assert.equal(existsSync(supervisorStatePath(logPath)), false); + assert.equal(existsSync(supervisorControlPath(logPath)), true); + assert.doesNotMatch(readFileSync(logPath, "utf8"), /another supervisor already owns/); + + // Normal cleanup deliberately leaves a stale generation instead of racing + // to rename a possible replacement. The next launch authenticates, safely + // reclaims it, and starts normally. + await superviseRouter(options); + assert.equal(readFileSync(attemptsPath, "utf8"), "2"); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("supervisor ownership does not require filesystem hard-link support", async () => { + const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-no-hardlinks-")); + const fixturePath = join(temp, "supervisor-child.mjs"); + const attemptsPath = join(temp, "attempts.txt"); + const logPath = join(temp, "server.log"); + const originalLinkSync = fs.linkSync; + let first = null; + let owner = null; + try { + fs.linkSync = () => { + const error = new Error("hard links are unavailable on this filesystem"); + error.code = "ENOTSUP"; + throw error; + }; + syncBuiltinESMExports(); + writeFileSync(fixturePath, FIXTURE_SOURCE); + const options = { + nodePath: process.execPath, + cliPath: fixturePath, + port: 10110, + logPath, + restartDelayMs: 500, + livenessTimeoutMs: 150, + pollIntervalMs: 5, + env: { + ...process.env, + DSCODEX_TEST_SUPERVISOR_STATE: attemptsPath, + DSCODEX_TEST_SUPERVISOR_FAILURES: "100", + }, + }; + + first = superviseRouter(options); + await waitFor( + () => existsSync(logPath) && /restarting in 500ms/.test(readFileSync(logPath, "utf8")), + "supervisor without hard links", + ); + owner = readSupervisorState(logPath); + assert.equal(await authenticateSupervisorOwner(logPath, owner, { + livenessTimeoutMs: 150, + pollIntervalMs: 5, + }), true); + + await superviseRouter(options); + assert.deepEqual(readSupervisorState(logPath), owner); + requestSupervisorStop(logPath, owner); + await first; + first = null; + + assert.equal(readFileSync(attemptsPath, "utf8"), "1"); + assert.equal(existsSync(supervisorControlPath(logPath)), true); + } finally { + if (first && owner) { + try { requestSupervisorStop(logPath, owner); } catch {} + await first.catch(() => {}); + } + fs.linkSync = originalLinkSync; + syncBuiltinESMExports(); + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("a fenced supervisor cannot remove replacement state during cleanup", async () => { + const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-fenced-cleanup-")); + const fixturePath = join(temp, "supervisor-child.mjs"); + const childStatePath = join(temp, "child-state.txt"); + const logPath = join(temp, "server.log"); + const displacedControlPath = `${supervisorControlPath(logPath)}.displaced-for-test`; + try { + writeFileSync(fixturePath, LONG_RUNNING_FIXTURE_SOURCE); + const supervised = superviseRouter({ + nodePath: process.execPath, + cliPath: fixturePath, + port: 10110, + logPath, + livenessTimeoutMs: 100, + pollIntervalMs: 5, + env: { + ...process.env, + DSCODEX_TEST_SUPERVISOR_STATE: childStatePath, + }, + }); + await waitFor(() => existsSync(childStatePath), "long-running router child"); + const original = readSupervisorState(logPath); + const replacement = fakeSupervisorState(process.pid); + + renameSync(supervisorControlPath(logPath), displacedControlPath); + mkdirSync(supervisorControlPath(logPath)); + writeFileSync( + join(supervisorControlPath(logPath), "owner.json"), + `${JSON.stringify(replacement)}\n`, + ); + writeFileSync(supervisorStatePath(logPath), `${JSON.stringify(replacement)}\n`); + + removeSupervisorState(logPath, original); + assert.deepEqual(readSupervisorState(logPath), replacement); + await supervised; + + assert.deepEqual(readSupervisorState(logPath), replacement); + assert.deepEqual( + JSON.parse(readFileSync(join(supervisorControlPath(logPath), "owner.json"), "utf8")), + replacement, + ); + assert.match(readFileSync(logPath, "utf8"), /ownership was replaced/); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("a temporary control-directory displacement does not fence a restored owner", async () => { + const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-control-restore-")); + const fixturePath = join(temp, "supervisor-child.mjs"); + const childStatePath = join(temp, "child-state.txt"); + const logPath = join(temp, "server.log"); + const displacedControlPath = `${supervisorControlPath(logPath)}.temporary-displacement`; + try { + writeFileSync(fixturePath, DELAYED_EXIT_FIXTURE_SOURCE); + const supervised = superviseRouter({ + nodePath: process.execPath, + cliPath: fixturePath, + port: 10110, + logPath, + livenessTimeoutMs: 100, + pollIntervalMs: 5, + env: { + ...process.env, + DSCODEX_TEST_SUPERVISOR_STATE: childStatePath, + }, + }); + await waitFor(() => existsSync(childStatePath), "delayed router child"); + + renameSync(supervisorControlPath(logPath), displacedControlPath); + await new Promise((resolve) => setTimeout(resolve, 20)); + renameSync(displacedControlPath, supervisorControlPath(logPath)); + await supervised; + + const log = readFileSync(logPath, "utf8"); + assert.match(log, /router exited cleanly; staying stopped/); + assert.doesNotMatch(log, /ownership was replaced/); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test("a sustained control I/O failure fences the owned router child", async () => { + const temp = mkdtempSync(join(tmpdir(), "dscodex-supervisor-control-io-failure-")); + const fixturePath = join(temp, "supervisor-child.mjs"); + const childStatePath = join(temp, "child-state.txt"); + const logPath = join(temp, "server.log"); + const controlPath = supervisorControlPath(logPath); + const originalReaddirSync = fs.readdirSync; + try { + writeFileSync(fixturePath, DELAYED_EXIT_FIXTURE_SOURCE); + const supervised = superviseRouter({ + nodePath: process.execPath, + cliPath: fixturePath, + port: 10110, + logPath, + livenessTimeoutMs: 50, + pollIntervalMs: 5, + env: { + ...process.env, + DSCODEX_TEST_SUPERVISOR_STATE: childStatePath, + }, + }); + await waitFor(() => existsSync(childStatePath), "router before control I/O failure"); + + fs.readdirSync = (path, ...args) => { + if (String(path) === controlPath) { + const error = new Error("simulated sustained control I/O failure"); + error.code = "EACCES"; + throw error; + } + return originalReaddirSync(path, ...args); + }; + syncBuiltinESMExports(); + await supervised; + + assert.match(readFileSync(logPath, "utf8"), /ownership was replaced; stopping owned router child/); + } finally { + fs.readdirSync = originalReaddirSync; + syncBuiltinESMExports(); + rmSync(temp, { recursive: true, force: true }); + } +});