diff --git a/bin/__deep-probe-mock.cjs b/bin/__deep-probe-mock.cjs new file mode 100755 index 0000000000..f07761b351 --- /dev/null +++ b/bin/__deep-probe-mock.cjs @@ -0,0 +1,31 @@ +#!/usr/bin/env node +'use strict'; +// Test fixture — a fake provider CLI for deep-probe / stall integration tests. +// Behavior is chosen by MOCK_MODE; MOCK_DELAY_MS controls the slow/hang timing. +// probe_ok (default) — print PROBE_OK, exit 0 +// quota — print a 429/RESOURCE_EXHAUSTED line to stderr, exit 1 +// auth — print a 401 unauthorized line to stderr, exit 1 +// slow_ok — print a short preamble, wait MOCK_DELAY_MS, then PROBE_OK, exit 0 +// hang — print a short preamble, then never output/exit (until killed) +const mode = process.env.MOCK_MODE || 'probe_ok'; +const delay = Number(process.env.MOCK_DELAY_MS || 0); + +function main() { + if (mode === 'quota') { + process.stderr.write('RESOURCE_EXHAUSTED (code 429): Individual quota reached. Resets in 32h54m.\n'); + process.exit(1); + } else if (mode === 'auth') { + process.stderr.write('HTTP 401 unauthorized: invalid api key\n'); + process.exit(1); + } else if (mode === 'hang') { + process.stdout.write('PREAMBLE'.repeat(25)); // ~200-byte preamble, then silence forever + setInterval(() => {}, 1 << 30); + } else if (mode === 'slow_ok') { + process.stdout.write('PREAMBLE'.repeat(25)); // small preamble (<500B), like the real slow slots + setTimeout(() => { process.stdout.write('\nPROBE_OK\n'); process.exit(0); }, delay); + } else { + process.stdout.write('PROBE_OK\n'); + process.exit(0); + } +} +main(); diff --git a/bin/call-quorum-slot-quota-reset.test.cjs b/bin/call-quorum-slot-quota-reset.test.cjs new file mode 100644 index 0000000000..76a8979502 --- /dev/null +++ b/bin/call-quorum-slot-quota-reset.test.cjs @@ -0,0 +1,45 @@ +'use strict'; + +// P4 — parse a provider's quota-reset window from its error text so the cooldown +// matches the real quota (rolling ~33h) instead of the fixed 30-min failure TTL. +// Motivated by antigravity-1 (Google 429 "Resets in 32h54m49s") being re-probed and +// re-failed every 30 min all session. + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { parseQuotaResetMs } = require('./call-quorum-slot.cjs'); + +const H = 3600000, M = 60000, S = 1000; + +describe('parseQuotaResetMs — quota reset-window parser', () => { + it('parses the compact Google form "Resets in 32h54m49s"', () => { + assert.equal(parseQuotaResetMs('RESOURCE_EXHAUSTED (code 429): Resets in 32h54m49s.'), + 32 * H + 54 * M + 49 * S); + }); + + it('parses "resets in 1h30m" and "resets in 45m"', () => { + assert.equal(parseQuotaResetMs('quota reached, resets in 1h30m'), 1 * H + 30 * M); + assert.equal(parseQuotaResetMs('resets in 45m'), 45 * M); + }); + + it('parses verbose "try again in 30 minutes" / "in 2 hours" / "in 45 seconds"', () => { + assert.equal(parseQuotaResetMs('rate limited, try again in 30 minutes'), 30 * M); + assert.equal(parseQuotaResetMs('quota exceeded; available in 2 hours'), 2 * H); + assert.equal(parseQuotaResetMs('retry in 45 seconds'), 45 * S); + }); + + it('returns null when no reset window is present', () => { + assert.equal(parseQuotaResetMs('429 Too Many Requests'), null); + assert.equal(parseQuotaResetMs('some unrelated error'), null); + assert.equal(parseQuotaResetMs(''), null); + assert.equal(parseQuotaResetMs(null), null); + }); + + it('clamps absurd windows to 48h (defends against a malformed "9999h")', () => { + assert.equal(parseQuotaResetMs('resets in 9999h'), 48 * H); + }); + + it('does not misfire on a bare number without a unit', () => { + assert.equal(parseQuotaResetMs('error 500 on attempt 3'), null); + }); +}); diff --git a/bin/call-quorum-slot.cjs b/bin/call-quorum-slot.cjs index 5394e11152..760230889d 100644 --- a/bin/call-quorum-slot.cjs +++ b/bin/call-quorum-slot.cjs @@ -223,6 +223,37 @@ function classifyErrorType(msg) { return 'UNKNOWN'; } +// ─── P4 — parse a provider's quota-reset window from its error text ────────── +// Quota errors often carry the reset time ("Resets in 32h54m49s", "resets in 30 +// minutes", "try again in 3600 seconds"). The default 30-min failure TTL is far +// shorter than a rolling daily quota (~33h), so a quota-dead slot was re-probed and +// re-failed every 30 min. Parsing the real reset lets the cooldown match it exactly. +// Returns milliseconds until reset (clamped to [0, 48h]) or null if not present. +function parseQuotaResetMs(msg) { + if (!msg) return null; + const s = String(msg); + const MAX = 48 * 3600 * 1000; + const clamp = (ms) => (ms > 0 ? Math.min(ms, MAX) : null); + // Compact form: 32h54m49s / 1h30m / 45m / 90s, optionally after "resets in". + let m = s.match(/resets?\s+in\s+((?:\d+\s*[hms]\s*){1,3})/i) || s.match(/\b((?:\d+\s*[hms]\s*){2,3})\b/i); + if (m) { + let ms = 0, mm; const re = /(\d+)\s*([hms])/gi; + while ((mm = re.exec(m[1]))) { + const n = Number(mm[1]); const u = mm[2].toLowerCase(); + ms += n * (u === 'h' ? 3600000 : u === 'm' ? 60000 : 1000); + } + const c = clamp(ms); if (c) return c; + } + // Verbose: "resets in 30 minutes" / "try again in 2 hours" / "in 45 seconds". + m = s.match(/(?:resets?|try again|retry|available)\s+(?:in\s+)?(\d+)\s*(hours?|minutes?|seconds?|hrs?|mins?|secs?)\b/i); + if (m) { + const n = Number(m[1]); const u = m[2].toLowerCase(); + const mult = u.startsWith('h') ? 3600000 : u.startsWith('m') ? 60000 : 1000; + return clamp(n * mult); + } + return null; +} + function writeFailureLog(slotName, errorMsg, stderrText) { try { const pp = require('./planning-paths.cjs'); @@ -234,22 +265,36 @@ function writeFailureLog(slotName, errorMsg, stderrText) { const rawPattern = (stderrText && stderrText.length > 0) ? stderrText : errorMsg; const pattern = rawPattern.replace(/\x1b\[[0-9;]*m/g, '').slice(0, 200); + // P4: for QUOTA failures, parse the provider's reset window so the cooldown matches + // the real quota (rolling ~33h) instead of the 30-min TTL. + const quotaResetMs = error_type === 'QUOTA' ? parseQuotaResetMs(rawPattern) : null; + const cooldownUntil = quotaResetMs ? new Date(Date.now() + quotaResetMs).toISOString() : null; + // Lock-free read-modify-write under an exclusive writer lock so parallel slots // never lose records (and a torn read never wipes the whole log). atomicUpdateJson(logPath, (current) => { let records = Array.isArray(current) ? current : []; - // Garbage-collect stale records (older than 60 minutes) to prevent unbounded growth - const gcCutoff = Date.now() - 60 * 60 * 1000; - records = records.filter(r => new Date(r.last_seen).getTime() > gcCutoff); + // Garbage-collect stale records (older than 60 minutes) to prevent unbounded growth. + // EXCEPT: keep records whose cooldown_until is still in the future (a quota-dead slot + // whose reset is hours away must survive GC, or P4's long cooldown is lost after 60 min). + const now = Date.now(); + const gcCutoff = now - 60 * 60 * 1000; + records = records.filter(r => + new Date(r.last_seen).getTime() > gcCutoff || + (r.cooldown_until && new Date(r.cooldown_until).getTime() > now)); // Update or insert record const existing = records.find(r => r.slot === slotName && r.error_type === error_type); if (existing) { existing.count++; existing.last_seen = new Date().toISOString(); + // Keep the latest known reset (a fresh quota message may push it out). + if (cooldownUntil) existing.cooldown_until = cooldownUntil; } else { - records.push({ slot: slotName, error_type, pattern, count: 1, last_seen: new Date().toISOString() }); + const rec = { slot: slotName, error_type, pattern, count: 1, last_seen: new Date().toISOString() }; + if (cooldownUntil) rec.cooldown_until = cooldownUntil; + records.push(rec); } return records; }, []); @@ -560,6 +605,12 @@ function runSubprocess(provider, prompt, idleTimeoutMs, hardTimeoutMs, allowedTo const MAX_BUF = 10 * 1024 * 1024; let l1Truncated = false; let l1OriginalSize = 0; + // P2: clamp the initial idle/hard timers to the max-safe bound. Node fires setTimeout + // delays above 2^31-1 ms IMMEDIATELY (silently disabling the timer), so an oversized + // idle_timeout_ms/hard_timeout_ms would leave a slot with no timeout at all. Mirrors + // stallTimeoutFor()'s clamp, which previously guarded only the stall window. + idleTimeoutMs = Math.min(idleTimeoutMs, TIMEOUT_MAX); + hardTimeoutMs = Math.min(hardTimeoutMs, TIMEOUT_MAX); // Kill entire process group, then destroy streams to force 'close' even if // grandchildren keep the pipes open (the common case with ccr/opencode). @@ -599,21 +650,25 @@ function runSubprocess(provider, prompt, idleTimeoutMs, hardTimeoutMs, allowedTo let rateLimitHits = 0; const RATE_LIMIT_THRESHOLD = 2; // kill after 2 consecutive rate-limit messages let totalBytesReceived = 0; - // Tighter idle timeout when the CLI has produced < STALL_BYTE_THRESHOLD bytes - // (header-only → probably hung). Per-slot override via providers.json - // `stall_timeout_ms` (see stallTimeoutFor): slow models (e.g. GLM-5.2[1m], - // MiniMax-M3) legitimately emit a small preamble then pause >30s mid-generation - // — they are slow, not stalled — so they need a longer threshold to avoid being - // false-killed. + // P2 — STALL detection = "no FIRST byte", NOT "model is thinking". + // Prior bug: the idle window was tightened to STALL_TIMEOUT_MS whenever total output + // was < 500 bytes, so a slow model that emitted a short preamble (e.g. 202-byte CLI + // framing) then paused >30s mid-generation was false-killed as STALL. A stall timeout + // is meant for "the process stopped writing", not "the model is still generating". + // Fix: a dedicated first-byte timer catches the genuine dead-on-arrival hang (zero bytes + // by STALL_TIMEOUT_MS); once ANY byte arrives it is cleared and the normal per-chunk + // idle window governs — so slow thinkers stream fine. Per-slot `stall_timeout_ms` + // (stallTimeoutFor) still tunes the first-byte budget for slow-to-start providers. const STALL_TIMEOUT_MS = stallTimeoutFor(provider); - const STALL_BYTE_THRESHOLD = 500; // below this = "just a header, probably stalled" + let firstByteTimer = setTimeout(() => { + if (totalBytesReceived === 0 && !timedOut) { timedOut = true; timeoutType = 'STALL'; killGroup(); } + }, STALL_TIMEOUT_MS); child.stdout.on('data', d => { totalBytesReceived += d.length; + clearTimeout(firstByteTimer); // first byte seen → past the dead-on-arrival window clearTimeout(idleTimer); - // Adaptive idle: use tighter 30s timeout when CLI has barely produced output (header-only stall) - const effectiveIdle = totalBytesReceived < STALL_BYTE_THRESHOLD ? STALL_TIMEOUT_MS : idleTimeoutMs; - idleTimer = setTimeout(() => { timedOut = true; timeoutType = totalBytesReceived < STALL_BYTE_THRESHOLD ? 'STALL' : 'IDLE'; killGroup(); }, effectiveIdle); + idleTimer = setTimeout(() => { timedOut = true; timeoutType = 'IDLE'; killGroup(); }, idleTimeoutMs); const chunk = d.toString(); l1OriginalSize += chunk.length; if (stdout.length < MAX_BUF) { @@ -639,9 +694,9 @@ function runSubprocess(provider, prompt, idleTimeoutMs, hardTimeoutMs, allowedTo }); child.stderr.on('data', d => { totalBytesReceived += d.length; + clearTimeout(firstByteTimer); // first byte seen (stderr counts) → past dead-on-arrival clearTimeout(idleTimer); - const effectiveIdle = totalBytesReceived < STALL_BYTE_THRESHOLD ? STALL_TIMEOUT_MS : idleTimeoutMs; - idleTimer = setTimeout(() => { timedOut = true; timeoutType = totalBytesReceived < STALL_BYTE_THRESHOLD ? 'STALL' : 'IDLE'; killGroup(); }, effectiveIdle); + idleTimer = setTimeout(() => { timedOut = true; timeoutType = 'IDLE'; killGroup(); }, idleTimeoutMs); const chunk = d.toString().slice(0, 4096); stderr += chunk; // Check stderr for rate-limit patterns too (gemini logs to stderr) @@ -659,6 +714,7 @@ function runSubprocess(provider, prompt, idleTimeoutMs, hardTimeoutMs, allowedTo child.on('close', (code) => { clearTimeout(idleTimer); clearTimeout(hardTimer); + clearTimeout(firstByteTimer); if (timedOut) { // Content-based reclassification: when STALL fires, inspect the partial // stdout/stderr for known error patterns before using the generic label. @@ -1106,4 +1162,4 @@ if (require.main === module) { } // ─── Test exports (SHELL-ESCAPE-01, TRUNC-01, INFRA-367) ─────────────────────── -module.exports = { buildSpawnArgs, stallTimeoutFor, recordTelemetry, findProjectRoot, writeFailureLog, atomicUpdateJson, VERDICTS, VERDICT_LINE_RE, parseVerdictLine }; +module.exports = { buildSpawnArgs, stallTimeoutFor, recordTelemetry, findProjectRoot, writeFailureLog, atomicUpdateJson, parseQuotaResetMs, VERDICTS, VERDICT_LINE_RE, parseVerdictLine }; diff --git a/bin/quorum-preflight-deep-integration.test.cjs b/bin/quorum-preflight-deep-integration.test.cjs new file mode 100644 index 0000000000..605bcbca54 --- /dev/null +++ b/bin/quorum-preflight-deep-integration.test.cjs @@ -0,0 +1,85 @@ +'use strict'; + +// P1 — LIVE deep-probe integration harness. Spawns a real (mock) provider CLI through +// deepProbeSlot and asserts the end-to-end classification. This is the harness that was +// deferred when P1's policy landed; it exercises the actual subprocess path, not just the +// pure classifier. The reviewers' "quota-dead slot blocked before review" case is the +// quota test below (deep probe downgrades → the P3 gate would then block). + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const path = require('path'); +const { deepProbeSlot, computeQuorumGate } = require('./quorum-preflight.cjs'); + +const MOCK = path.join(__dirname, '__deep-probe-mock.cjs'); +try { fs.chmodSync(MOCK, 0o755); } catch (_) {} + +function mockProvider(mode, extra = {}) { + return { + name: `mock-${mode}`, + type: 'subprocess', + resolvedCli: MOCK, // resolveSpawnTarget prefers resolvedCli (absolute) + args_template: ['{prompt}'], // honored verbatim by resolveArgsTemplate + deep_probe: { prompt: 'respond with: PROBE_OK', expect: 'PROBE_OK', timeout_ms: 4000 }, + env: { MOCK_MODE: mode, ...extra }, + }; +} + +describe('deepProbeSlot — live spawn classification (integration)', () => { + it('healthy slot (prints PROBE_OK) → ok, OK', async () => { + const r = await deepProbeSlot(mockProvider('probe_ok'), 10000); + assert.equal(r.ok, true); + assert.equal(r.classification, 'OK'); + }); + + it('quota-dead slot (429/RESOURCE_EXHAUSTED) → DOWNGRADED (the L1/L2-missed case)', async () => { + const r = await deepProbeSlot(mockProvider('quota'), 10000); + assert.equal(r.ok, false); + assert.equal(r.classification, 'QUOTA'); + }); + + it('auth-dead slot (401) → DOWNGRADED', async () => { + const r = await deepProbeSlot(mockProvider('auth'), 10000); + assert.equal(r.ok, false); + assert.equal(r.classification, 'AUTH'); + }); + + it('slow-but-healthy slot (preamble, ~1.5s pause, then PROBE_OK) → ok, NOT killed', async () => { + // The P2 shape at the deep-probe layer: a short preamble then a pause. Must pass. + const r = await deepProbeSlot(mockProvider('slow_ok', { MOCK_DELAY_MS: '1500' }), 10000); + assert.equal(r.ok, true); + assert.equal(r.classification, 'OK'); + }); + + it('hung slot (preamble then silence) → timeout is INCONCLUSIVE, never downgraded (P2-safe)', async () => { + // A tight per-probe timeout fires; a timeout must NOT downgrade (would re-create the + // slow-slot false-kill). ok stays true. + const prov = mockProvider('hang'); + prov.deep_probe = { prompt: 'x', expect: 'PROBE_OK', timeout_ms: 800 }; + const r = await deepProbeSlot(prov, 10000); + assert.equal(r.ok, true); + assert.equal(r.classification, 'INCONCLUSIVE'); + }); + + it('unspawnable slot (no target) → SKIP, never false-kills', async () => { + const r = await deepProbeSlot({ name: 'x', type: 'subprocess', args_template: ['{prompt}'] }, 10000); + assert.equal(r.ok, true); + assert.equal(r.classification, 'SKIP'); + }); + + it('E2E (reviewers\' key test): a quota-dead slot is downgraded → gate BLOCKS before review', async () => { + // 3 slots pass L1/L2, max_quorum_size=3, but one is quota-dead. The deep probe catches + // it (L1/L2 could not), dropping available to 2/3 → computeQuorumGate blocks. Without the + // deep gate this quorum would have dispatched a real review on a dead slot. + const slots = ['probe_ok', 'probe_ok', 'quota']; + const results = await Promise.all(slots.map((m) => + deepProbeSlot(mockProvider(m), 10000).then(r => ({ m, r })))); + let available = slots.length; + for (const { r } of results) if (r.ok === false) available -= 1; + assert.equal(available, 2, 'the quota-dead slot must be downgraded by the deep probe'); + const gate = computeQuorumGate(available, 3, false); + assert.equal(gate.blocked, true, 'reduced (2/3) panel must block before any review runs'); + assert.equal(gate.waiver_required, true); + }); +}); diff --git a/bin/quorum-preflight-deep.test.cjs b/bin/quorum-preflight-deep.test.cjs new file mode 100644 index 0000000000..4cc3dc7de8 --- /dev/null +++ b/bin/quorum-preflight-deep.test.cjs @@ -0,0 +1,76 @@ +'use strict'; + +// P1 — deep inference probe policy (pure logic). The spawn is deferred to a +// harness-backed follow-up; these guard the safety-critical policy: +// - downgrade a slot ONLY on a fast explicit auth/quota signal (what L1/L2 miss); +// - a TIMEOUT is inconclusive and never downgrades (so a slow-but-healthy slot — +// the P2 failure mode — is not re-killed by the deep probe); +// - run the probe when --deep or when the panel is degraded, budget permitting. + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { shouldRunDeepProbe, classifyDeepProbeResult, computeQuorumGate } = require('./quorum-preflight.cjs'); + +describe('shouldRunDeepProbe — when to run the deep gate', () => { + it('runs when --deep is set', () => { + assert.equal(shouldRunDeepProbe({ deep: true, degraded: false, budgetMs: null }), true); + }); + it('auto-runs on a degraded panel WITH an explicit sufficient budget (real dispatch path)', () => { + assert.equal(shouldRunDeepProbe({ deep: false, degraded: true, budgetMs: 60000, minBudgetMs: 45000 }), true); + }); + it('does NOT auto-run on degraded with no budget — the cheap --all --probe path stays fast', () => { + // Regression guard: auto-running live deep probes in the budget-less liveness path + // blew its <8s budget (spawnSync ETIMEDOUT). Auto-enable requires an explicit budget. + assert.equal(shouldRunDeepProbe({ deep: false, degraded: true, budgetMs: null }), false); + }); + it('does NOT run on a healthy panel with no --deep (happy path untouched)', () => { + assert.equal(shouldRunDeepProbe({ deep: false, degraded: false, budgetMs: null }), false); + }); + it('is skipped when the time budget cannot cover a probe', () => { + assert.equal(shouldRunDeepProbe({ deep: true, degraded: true, budgetMs: 1000, minBudgetMs: 45000 }), false); + assert.equal(shouldRunDeepProbe({ deep: true, degraded: true, budgetMs: 60000, minBudgetMs: 45000 }), true); + }); + + it('F1 (known gap, regression bar): a FULL panel is NOT deep-probed → a quota-dead slot slips', () => { + // Documents the accepted limitation so a future change that closes it trips this test. + // Full panel (3/3): degraded=false, so even with a sufficient budget and no --deep the + // deep probe does NOT run — a quota-dead slot is not pre-screened and the gate reports met. + assert.equal(shouldRunDeepProbe({ deep: false, degraded: false, budgetMs: 60000 }), false); + const gate = computeQuorumGate(3, 3, false); + assert.equal(gate.quorum_met, true); + assert.equal(gate.blocked, false); + // --deep DOES force a full-panel probe (the escape hatch for known-suspect slots). + assert.equal(shouldRunDeepProbe({ deep: true, degraded: false, budgetMs: 60000 }), true); + }); +}); + +describe('classifyDeepProbeResult — downgrade only on fast auth/quota', () => { + it('downgrades on an explicit auth failure (401/403/unauthorized)', () => { + assert.equal(classifyDeepProbeResult('HTTP 401 unauthorized').ok, false); + assert.equal(classifyDeepProbeResult('Error: invalid api key').classification, 'AUTH'); + }); + it('downgrades on an explicit quota/rate-limit signal (429/quota/resource exhausted)', () => { + assert.equal(classifyDeepProbeResult('RESOURCE_EXHAUSTED (code 429): quota reached').ok, false); + assert.equal(classifyDeepProbeResult('429 Too Many Requests').classification, 'QUOTA'); + }); + it('does NOT downgrade on a timeout — inconclusive, protecting slow-but-healthy slots (P2)', () => { + const r = classifyDeepProbeResult('', { timedOut: true }); + assert.equal(r.ok, true); + assert.equal(r.classification, 'INCONCLUSIVE'); + }); + it('passes when the expected token is present', () => { + const r = classifyDeepProbeResult('...\nPROBE_OK\n', { expect: 'PROBE_OK' }); + assert.equal(r.ok, true); + assert.equal(r.classification, 'OK'); + }); + it('assumes alive on ambiguous non-error output (never false-kills on ambiguity)', () => { + const r = classifyDeepProbeResult('some normal model chatter', { expect: 'PROBE_OK' }); + assert.equal(r.ok, true); + assert.equal(r.classification, 'INCONCLUSIVE'); + }); + it('the antigravity-1 shape: fast 429 with reset → downgraded (the L1/L2-missed case)', () => { + const r = classifyDeepProbeResult('RESOURCE_EXHAUSTED (code 429): Resets in 32h54m49s'); + assert.equal(r.ok, false); + assert.equal(r.classification, 'QUOTA'); + }); +}); diff --git a/bin/quorum-preflight-gate.test.cjs b/bin/quorum-preflight-gate.test.cjs new file mode 100644 index 0000000000..597a0ff059 --- /dev/null +++ b/bin/quorum-preflight-gate.test.cjs @@ -0,0 +1,68 @@ +'use strict'; + +// P3 — authoritative degraded-panel gate. Previously the "available < max_quorum_size → +// BLOCK unless --force-quorum" decision lived in quorum.md prose (LLM-interpreted), so a +// panel that collapsed to 1/7 could run 6 silent reduced rounds. computeQuorumGate makes +// it a deterministic, testable machine field. + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { computeQuorumGate } = require('./quorum-preflight.cjs'); + +describe('computeQuorumGate — degraded-panel gate', () => { + it('quorum met (available >= max): not blocked, not degraded', () => { + const g = computeQuorumGate(5, 3, false); + assert.equal(g.quorum_met, true); + assert.equal(g.blocked, false); + assert.equal(g.degraded, false); + assert.equal(g.available_count, 5); + assert.ok(!g.waiver_required); + }); + + it('exactly at threshold (available === max): met, not blocked', () => { + const g = computeQuorumGate(3, 3, false); + assert.equal(g.quorum_met, true); + assert.equal(g.blocked, false); + assert.equal(g.degraded, false); + }); + + it('degraded (1 <= available < max) without waiver: BLOCKED + waiver_required + degraded', () => { + const g = computeQuorumGate(1, 3, false); + assert.equal(g.quorum_met, false); + assert.equal(g.blocked, true); + assert.equal(g.waiver_required, true); + assert.equal(g.degraded, true); + assert.match(g.gate_reason, /only 1\/3/); + }); + + it('panel down (0 available) without waiver: BLOCKED, degraded=false (nothing to run)', () => { + const g = computeQuorumGate(0, 3, false); + assert.equal(g.blocked, true); + assert.equal(g.waiver_required, true); + assert.equal(g.degraded, false); // 0 slots is "panel down", not "degraded" + assert.match(g.gate_reason, /panel down/); + }); + + it('--force-quorum waives a degraded panel: not blocked, waiver_used recorded', () => { + const g = computeQuorumGate(1, 3, true); + assert.equal(g.blocked, false); + assert.equal(g.waiver_used, true); + assert.ok(!g.waiver_required); + assert.match(g.gate_reason, /WAIVED via --force-quorum/); + }); + + it('--force-quorum on a met quorum is a no-op (no spurious waiver_used)', () => { + const g = computeQuorumGate(4, 3, true); + assert.equal(g.blocked, false); + assert.ok(!g.waiver_used); // waiver only recorded when it actually overrode a block + }); + + it('the production failure shape: 1 of 7 with default max 3 blocks unless forced', () => { + // This is exactly the session that motivated P3 — codex-1 only, panel collapsed. + const blocked = computeQuorumGate(1, 3, false); + assert.equal(blocked.blocked, true, '1/7-style panel must block by default'); + const forced = computeQuorumGate(1, 3, true); + assert.equal(forced.blocked, false, '--force-quorum must allow explicit reduced-quorum runs'); + assert.equal(forced.waiver_used, true); + }); +}); diff --git a/bin/quorum-preflight-validate.test.cjs b/bin/quorum-preflight-validate.test.cjs new file mode 100644 index 0000000000..2b4fd3aed2 --- /dev/null +++ b/bin/quorum-preflight-validate.test.cjs @@ -0,0 +1,80 @@ +'use strict'; + +// P5 — providers.json schema validator. There was no gate on the config, so an +// unspawnable subprocess slot (no cli/mainTool) or an http slot missing baseUrl/apiKeyEnv +// only surfaced as a spawn crash at dispatch. This catches drift up front. + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { validateProviders } = require('./quorum-preflight.cjs'); + +describe('validateProviders — providers.json schema gate', () => { + it('accepts a well-formed subprocess slot with a deep_probe', () => { + const r = validateProviders([ + { name: 'codex-1', type: 'subprocess', cli: '/usr/bin/codex', deep_probe: { prompt: 'x', expect: 'y' } }, + ]); + assert.equal(r.ok, true); + assert.equal(r.errors.length, 0); + assert.equal(r.warnings.length, 0); + }); + + it('errors when a subprocess slot has no spawn target (cli AND mainTool absent)', () => { + const r = validateProviders([{ name: 'broken-1', type: 'subprocess' }]); + assert.equal(r.ok, false); + assert.match(r.errors.join(), /broken-1: subprocess.*no spawn target/); + }); + + it('accepts a subprocess slot with only mainTool (cli optional)', () => { + const r = validateProviders([{ name: 'ok-1', type: 'subprocess', mainTool: 'codex', deep_probe: {} }]); + assert.equal(r.ok, true); + }); + + it('errors when an http slot is missing baseUrl / apiKeyEnv', () => { + const r = validateProviders([{ name: 'api-1', type: 'http', deep_probe: {} }]); + assert.equal(r.ok, false); + assert.match(r.errors.join(), /api-1: http slot missing baseUrl/); + assert.match(r.errors.join(), /api-1: http slot missing apiKeyEnv/); + }); + + it('warns (not errors) when an inference slot lacks a deep_probe — the antigravity-1 shape', () => { + // antigravity-1's live entry has cli but no deep_probe → it failed opaquely as + // BINARY_MISSING instead of being inference-health-gated. + const r = validateProviders([{ name: 'antigravity-1', type: 'subprocess', cli: '/x/agy' }]); + assert.equal(r.ok, true); // spawnable, so not an error + assert.match(r.warnings.join(), /antigravity-1: no deep_probe/); + }); + + it('errors on a missing type and on duplicate names', () => { + const r = validateProviders([ + { name: 'x' }, + { name: 'dupe', type: 'subprocess', cli: '/a', deep_probe: {} }, + { name: 'dupe', type: 'subprocess', cli: '/b', deep_probe: {} }, + ]); + assert.equal(r.ok, false); + assert.match(r.errors.join(), /x: missing type/); + assert.match(r.errors.join(), /duplicate provider name: dupe/); + }); + + it('tolerates a non-array / empty input (fail-open)', () => { + assert.equal(validateProviders(null).ok, true); + assert.equal(validateProviders([]).ok, true); + }); + + it('F2: SKIPS intentionally-disabled (active:false) slots — no ERROR flood for off slots', () => { + // An inactive http slot legitimately omits baseUrl/apiKeyEnv until turned on. Validating + // it would emit ERRORs on every preflight (stderr flood). It must be skipped entirely. + const r = validateProviders([ + { name: 'api-1', type: 'http', active: false }, // off + incomplete → skipped + { name: 'codex-1', type: 'subprocess', cli: '/x', deep_probe: {} }, + ]); + assert.equal(r.ok, true, 'inactive incomplete slot must not error'); + assert.equal(r.errors.length, 0); + assert.ok(!r.warnings.join().includes('api-1'), 'inactive slot must not warn either'); + }); + + it('F2: an ACTIVE incomplete http slot still errors (skip is scoped to active:false only)', () => { + const r = validateProviders([{ name: 'api-2', type: 'http' }]); // active undefined → validated + assert.equal(r.ok, false); + assert.match(r.errors.join(), /api-2: http slot missing baseUrl/); + }); +}); diff --git a/bin/quorum-preflight.cjs b/bin/quorum-preflight.cjs index 17b5f7aeb0..4de865bb6a 100644 --- a/bin/quorum-preflight.cjs +++ b/bin/quorum-preflight.cjs @@ -32,11 +32,22 @@ const { resolveCli } = require('./resolve-cli.cjs'); const https = require('https'); const http = require('http'); const { resolveSpawnTarget } = require('./resolve-cli.cjs'); +const { resolveArgsTemplate } = require('./provider-arg-templates.cjs'); const { loadProviders } = require('./resolve-providers.cjs'); // Probe is ON by default for --all; --no-probe to skip, --probe still accepted for compat const NO_PROBE = process.argv.includes('--no-probe'); const PROBE = !NO_PROBE; +// P3 — when the panel is degraded (fewer available slots than max_quorum_size), preflight +// emits an authoritative `blocked`/`waiver_required` gate. --force-quorum records an explicit +// waiver so the machine field reflects the override deterministically (invariant: no downstream +// dispatch on a blocked panel without this flag). +const FORCE_QUORUM = process.argv.includes('--force-quorum'); +// P1 — deep inference probe. Opt-in (--deep) and auto-enabled when the panel is degraded +// (P3). L1 (--version) and L2 (/models) can't tell a quota/auth-dead slot from a healthy +// one (L2 even treats 401/403 as reachable), so a dead slot passes preflight and only fails +// during the real review. The deep probe runs the provider's deep_probe prompt to catch it. +const DEEP_PROBE = process.argv.includes('--deep'); // ─── Time-budget parsing ──────────────────────────────────────────────────── // --budget-ms threads a soft deadline that --all degrades within: when the @@ -335,16 +346,29 @@ function probeInferenceHistory(ttlMinutes = 30) { if (!fs.existsSync(logPath)) return {}; const records = JSON.parse(fs.readFileSync(logPath, 'utf8')); if (!Array.isArray(records)) return {}; - const cutoff = Date.now() - ttlMinutes * 60 * 1000; + const now = Date.now(); + const cutoff = now - ttlMinutes * 60 * 1000; const result = {}; for (const r of records) { - if (new Date(r.last_seen).getTime() > cutoff) { + // P4 — a QUOTA failure with a parsed reset window (cooldown_until) keeps the slot + // unavailable until the ACTUAL reset (rolling ~33h), not just the 30-min TTL — so a + // quota-dead slot is not re-probed and re-failed every 30 min. Otherwise fall back + // to the recency TTL. The `retired` flag marks a slot held out by a long cooldown + // (transparency for the roster); a single non-quota blip never sets it. + const cooldownUntil = r.cooldown_until ? new Date(r.cooldown_until).getTime() : 0; + const coolingDown = cooldownUntil > now; + const recent = new Date(r.last_seen).getTime() > cutoff; + if (recent || coolingDown) { + const mins = coolingDown ? Math.round((cooldownUntil - now) / 60000) : 0; result[r.slot] = { ok: false, - reason: `${r.error_type}: ${(r.pattern || '').slice(0, 100)}`, + reason: coolingDown + ? `${r.error_type}: cooling down ~${mins}min (until ${r.cooldown_until})` + : `${r.error_type}: ${(r.pattern || '').slice(0, 100)}`, error_type: r.error_type, count: r.count, last_seen: r.last_seen, + ...(coolingDown ? { cooldown_until: r.cooldown_until, retired: r.error_type === 'QUOTA' } : {}), }; } } @@ -352,6 +376,54 @@ function probeInferenceHistory(ttlMinutes = 30) { } catch (_) { return {}; } // fail-open } +// ─── P1 — live deep inference probe (spawn) ────────────────────────────────── +// Runs the provider's deep_probe prompt through the real CLI and classifies the +// result via classifyDeepProbeResult. Regression-safe by construction: a spawn/args +// error or a timeout resolves to ok:true (SKIP/INCONCLUSIVE) — the deep probe can +// ONLY downgrade a slot on a fast explicit auth/quota signal, never on slowness or a +// harness problem, so it cannot re-create the P2 slow-slot false-kill or knock out a +// slot L1/L2 already vetted. +function deepProbeSlot(provider, budgetMs) { + return new Promise((resolve) => { + const probe = provider.deep_probe || { prompt: 'respond with: PROBE_OK', expect: 'PROBE_OK', timeout_ms: 45000 }; + const target = resolveSpawnTarget(provider) || provider.cli; + const tmpl = resolveArgsTemplate(provider); + if (!target || !Array.isArray(tmpl)) { + // F3 (known gap): http slots have no CLI → resolveSpawnTarget returns '' → SKIP here. + // http backups are L1/L2-verified only (L2 pings /models); P1's CLI deep_probe cannot + // inference-verify them. Closing that hole needs a real HTTP chat-completion probe — + // a separate deep_probe contract, out of scope for this layer. + resolve({ ok: true, classification: 'SKIP', reason: 'no spawn target / args_template' }); + return; + } + const args = tmpl.map(a => (a === '{prompt}' ? probe.prompt : a)); + // N1: budgetMs is applied PER SLOT (probes fan out in parallel, so wall-clock ≈ the + // slowest single probe, not the sum). Callers sizing an aggregate deadline should + // account for N parallel spawns of up to this cap each. + const timeoutMs = Math.min(probe.timeout_ms || 45000, budgetMs == null ? 2147483647 : budgetMs); + let out = '', done = false, timedOut = false, child; + const finish = (r) => { + if (done) return; done = true; + // N2 — capture child locally + guard: finish() is only wired after a successful + // spawn, but guard the kill so a not-yet-assigned/exited child can never throw a + // ReferenceError out of the probe (which would reject the Promise.all fan-out). + const c = child; + if (c && c.pid) { try { process.kill(-c.pid, 'SIGKILL'); } catch (_) { try { c.kill('SIGKILL'); } catch (_) {} } } + resolve(r); + }; + try { + child = spawn(target, args, { env: { ...process.env, ...(provider.env || {}) }, stdio: ['pipe', 'pipe', 'pipe'], detached: true }); + } catch (e) { resolve({ ok: true, classification: 'SKIP', reason: 'spawn error: ' + e.message }); return; } + const timer = setTimeout(() => { timedOut = true; finish(classifyDeepProbeResult(out, { timedOut: true, expect: probe.expect })); }, timeoutMs); + // ccr slots read the prompt from stdin; others are non-interactive. + try { if (provider.type === 'ccr') { child.stdin.write(probe.prompt); child.stdin.end(); } else { child.stdin.end(); } } catch (_) {} + child.stdout.on('data', d => { if (out.length < 65536) out += d.toString(); }); + child.stderr.on('data', d => { if (out.length < 65536) out += d.toString(); }); + child.on('close', () => { clearTimeout(timer); if (!timedOut) finish(classifyDeepProbeResult(out, { timedOut: false, expect: probe.expect })); }); + child.on('error', (e) => { clearTimeout(timer); finish({ ok: true, classification: 'SKIP', reason: 'spawn error: ' + e.message }); }); + }); +} + // ─── Two-layer parallel health probe ──────────────────────────────────────── async function probeHealth(providers) { const cache = loadCache(); @@ -544,6 +616,11 @@ async function main() { console.log('OK'); } else if (mode === '--all') { const providers = findProviders(); + // P5 — surface providers.json config problems (drift, unspawnable slots, missing + // deep_probe) as non-fatal diagnostics rather than letting them crash at fan-out. + const _val = validateProviders(providers); + for (const e of _val.errors) process.stderr.write(`[preflight] config ERROR: ${e}\n`); + for (const w of _val.warnings) process.stderr.write(`[preflight] config warn: ${w}\n`); const active = cfg.quorum_active || []; const team = buildTeam(providers, active); const maxSize = cfg.max_quorum_size ?? 3; @@ -632,6 +709,47 @@ async function main() { if (output.backup_slots.length > 0) { process.stderr.write(`[preflight] Tiered ordering: ${output.primary_slots.length} primary (CLI) + ${output.backup_slots.length} backup (HTTP API)\n`); } + + // ─── P1 — deep inference gate. L1/L2/L3 can pass a quota/auth-dead slot (L2 even + // treats 401/403 as reachable). Run the real deep_probe when --deep OR when the + // prelim panel is already degraded, budget permitting, and downgrade any slot that + // returns a FAST auth/quota signal (timeouts/spawn errors never downgrade — see + // deepProbeSlot/classifyDeepProbeResult). This runs off the happy path by default. + // F1 (known limitation): a quota/auth-dead slot in a FULL panel (available >= maxSize, + // so degraded=false) with no --deep is NOT deep-probed — it dispatches and fails at + // the real review, and L3 failure-history catches it the NEXT round (self-heals, one + // wasted round). Deep-probing every slot every run is expensive, so this is accepted; + // pass --deep (or run when the panel is already degraded) to pre-empt it. NOTE this + // also means --force-quorum on a full panel won't pre-screen a known-suspect slot. + const prelimGate = computeQuorumGate(output.available_slots.length, maxSize, FORCE_QUORUM); + if (shouldRunDeepProbe({ deep: DEEP_PROBE, degraded: prelimGate.degraded, budgetMs: BUDGET_MS })) { + const results = await Promise.all( + output.available_slots.map(async (name) => ({ name, r: await deepProbeSlot(providerByName.get(name) || {}, BUDGET_MS) })) + ); + const downgraded = results.filter(({ r }) => r && r.ok === false); + if (downgraded.length > 0) { + const deadNames = new Set(downgraded.map(d => d.name)); + for (const { name, r } of downgraded) { + if (output.health[name]) { output.health[name].healthy = false; output.health[name].layer4 = r; } + output.unavailable_slots.push({ name, reason: `layer4: ${r.reason}` }); + } + output.available_slots = output.available_slots.filter(n => !deadNames.has(n)); + output.primary_slots = output.available_slots.filter(s => typeMap.get(s) !== 'http'); + output.backup_slots = output.available_slots.filter(s => typeMap.get(s) === 'http').concat(dedupedOut); + process.stderr.write(`[preflight] deep-probe downgraded ${downgraded.length} slot(s): ${downgraded.map(d => d.name).join(', ')}\n`); + } + output.deep_probe_ran = true; + } + + // ─── P3 — authoritative degraded-panel gate (see computeQuorumGate). Machine + // field so the block/waiver decision is deterministic + testable, not re-derived + // by LLM-interpreted markdown. `available_count` = distinct healthy slots eligible + // to vote (deduped primaries + HTTP backups; demoted duplicates excluded). Computed + // AFTER the deep-probe pass so the gate reflects the post-deep roster. + Object.assign(output, computeQuorumGate(output.available_slots.length, maxSize, FORCE_QUORUM)); + if (output.blocked) { + process.stderr.write(`[preflight] ${output.gate_reason}\n`); + } } console.log(JSON.stringify(output)); @@ -642,8 +760,113 @@ async function main() { } } +// ─── P3 — authoritative degraded-panel gate (pure, testable) ───────────────── +// Was LLM-interpreted prose in quorum.md (availableCount < max_quorum_size → BLOCK +// unless --force-quorum). Now a machine field so the decision is deterministic. +// Invariant (quorum review 2026-07-01): when `blocked` is true, no downstream path may +// dispatch a quorum without a recorded --force-quorum waiver. +function computeQuorumGate(availableCount, maxSize, forceQuorum) { + const quorumMet = availableCount >= maxSize; + const gate = { + available_count: availableCount, + quorum_met: quorumMet, + // `degraded` = some slots present but fewer than required. P1's deep-probe gate auto-enables on this. + degraded: availableCount >= 1 && !quorumMet, + }; + if (quorumMet) { + gate.blocked = false; + gate.gate_reason = `quorum met: ${availableCount}/${maxSize} slots available`; + } else if (forceQuorum) { + gate.blocked = false; + gate.waiver_used = true; + gate.gate_reason = `reduced quorum WAIVED via --force-quorum: ${availableCount}/${maxSize} available`; + } else { + gate.blocked = true; + gate.waiver_required = true; + gate.gate_reason = availableCount === 0 + ? `BLOCKED: 0/${maxSize} slots available (panel down) — pass --force-quorum only with explicit user awareness` + : `BLOCKED: only ${availableCount}/${maxSize} slots available — pass --force-quorum to proceed on a reduced quorum`; + } + return gate; +} + +// ─── P5 — providers.json schema validator (pure, testable) ─────────────────── +// There was no schema gate on providers.json: a subprocess slot with no resolvable +// spawn target, or an http slot missing baseUrl/apiKeyEnv, only surfaced as a spawn +// crash at dispatch (spawn(null) → whole quorum offline). A slot with no deep_probe +// can't be inference-health-gated (P1). This validator surfaces those as errors / +// warnings so config drift is caught up front instead of at fan-out time. +function validateProviders(providers) { + const errors = [], warnings = []; + const list = Array.isArray(providers) ? providers : []; + const seen = new Set(); + for (const p of list) { + if (!p || typeof p !== 'object' || !p.name) { errors.push('provider entry missing/!object name'); continue; } + // F2 — skip intentionally-disabled slots: an `active:false` provider (e.g. the api-* + // HTTP slots) may legitimately omit baseUrl/apiKeyEnv until it's turned on — validating + // it would flood stderr with ERRORs on every preflight. Only gate slots that can run. + if (p.active === false) continue; + if (seen.has(p.name)) errors.push(`duplicate provider name: ${p.name}`); + seen.add(p.name); + const type = p.type; + if (type === 'subprocess' || type === 'ccr') { + if (!p.cli && !p.mainTool) errors.push(`${p.name}: subprocess/ccr slot has no spawn target (cli or mainTool)`); + } else if (type === 'http') { + if (!p.baseUrl) errors.push(`${p.name}: http slot missing baseUrl`); + if (!p.apiKeyEnv) errors.push(`${p.name}: http slot missing apiKeyEnv`); + } else if (!type) { + errors.push(`${p.name}: missing type`); + } + // Inference slots without a deep_probe can't be health-gated by P1's deep layer. + if ((type === 'subprocess' || type === 'ccr' || type === 'http') && !p.deep_probe) { + warnings.push(`${p.name}: no deep_probe — cannot be inference-health-gated`); + } + } + return { ok: errors.length === 0, errors, warnings }; +} + +// ─── P1 — deep inference probe: decision + result classification (pure) ────── +// The spawn itself is intentionally NOT yet wired into the main() hot path — an +// untested subprocess probe on the path that gates EVERY quorum is the one change +// that could take the whole system down, so it lands with a mock-CLI integration +// harness in a follow-up. These two pure helpers encode the safe policy and are unit-tested. + +// Decide whether to run the (expensive, spawn-per-slot) deep probe. +// - Explicit --deep: run (subject to budget when one is set). +// - Degraded panel WITHOUT --deep: auto-enable ONLY when an explicit budget is set and +// covers a probe. The default cheap `--all --probe` liveness path passes no budget +// (budgetMs == null), so it must NOT pay deep-probe latency there — the real quorum +// dispatch passes --budget-ms, so it still auto-verifies a reduced panel with inference. +function shouldRunDeepProbe({ deep, degraded, budgetMs, minBudgetMs = 45000 }) { + if (deep) return budgetMs == null ? true : budgetMs >= minBudgetMs; + if (!degraded) return false; + return budgetMs != null && budgetMs >= minBudgetMs; +} + +// Classify a deep-probe result. CRITICAL: downgrade a slot ONLY on a FAST, explicit +// auth/quota signal — the exact class L1/L2 miss. A timeout is treated as INCONCLUSIVE +// and does NOT downgrade, so a slow-but-healthy slot (the P2 failure mode) is never +// false-killed by the deep probe. Ambiguous non-error output is assumed alive. +function classifyDeepProbeResult(output, { timedOut = false, expect = null } = {}) { + if (timedOut) { + return { ok: true, classification: 'INCONCLUSIVE', reason: 'deep-probe timed out (slow, not downgraded)' }; + } + const text = String(output || ''); + if (/\b(401|403)\b|unauthorized|forbidden|invalid.*api.?key/i.test(text)) { + return { ok: false, classification: 'AUTH', reason: 'deep-probe: authentication failure' }; + } + if (/\b(402|429)\b|quota|resource.?exhausted|too many requests|exhausted your capacity|rate.?limit/i.test(text)) { + return { ok: false, classification: 'QUOTA', reason: 'deep-probe: quota/rate-limit' }; + } + if (expect && text.includes(expect)) { + return { ok: true, classification: 'OK', reason: `deep-probe: matched "${expect}"` }; + } + // Non-empty, no error, no expect match → assume alive (never false-kill on ambiguity). + return { ok: true, classification: text.trim() ? 'INCONCLUSIVE' : 'EMPTY', reason: 'deep-probe: no error signal' }; +} + if (require.main === module) { main().catch(e => { console.error(e); process.exit(1); }); } -module.exports = { dedupBySlotIdentity, probeHealth, findProviders }; +module.exports = { dedupBySlotIdentity, probeHealth, findProviders, computeQuorumGate, validateProviders, shouldRunDeepProbe, classifyDeepProbeResult, deepProbeSlot };