From c2f2078be4064ec3c7b040d5af0d98484874f6cd Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Wed, 1 Jul 2026 14:16:24 +0100 Subject: [PATCH 1/7] harden(quorum) P2: stall timeout = no-first-byte, not "model thinking" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The <500-byte adaptive idle window false-killed slow-but-healthy slots: a model that emitted a short CLI preamble (e.g. 202-byte framing) then paused >30s while generating was killed as STALL, because total output stayed under the 500-byte threshold so the idle window was tightened to 30s mid-generation. Observed live on claude-z-ai / claude-minimax ("202 bytes then 30s silence -> killed"), which collapsed a 7-slot quorum to 1 across 6 rounds. Fix: STALL now means "produced no FIRST byte", not "still thinking". A dedicated first-byte timer fires only if zero bytes arrive by STALL_TIMEOUT_MS (genuine dead-on-arrival hang); once ANY byte arrives it is cleared and the normal per-chunk idle window governs, so slow streamers are not penalised. Also clamp the initial idle/hard timers to TIMEOUT_MAX (Node fires >2^31-1ms timers immediately, which would silently disable them) — previously only the stall window was clamped. Empirically validated: widening the stall window on the two affected slots was the only change that let them complete a heavy review prompt, confirming the diagnosis. All 40 existing call-quorum-slot timeout/latency/infra/retry tests pass. Follow-up: add a mock-CLI integration harness to time-test runSubprocess (also needed for the P1/P3 "quota-dead slot blocked before review" test). Co-Authored-By: Claude Opus 4.8 --- bin/call-quorum-slot.cjs | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/bin/call-quorum-slot.cjs b/bin/call-quorum-slot.cjs index 5394e11152..7d616c332a 100644 --- a/bin/call-quorum-slot.cjs +++ b/bin/call-quorum-slot.cjs @@ -560,6 +560,12 @@ function runSubprocess(provider, prompt, idleTimeoutMs, hardTimeoutMs, allowedTo const MAX_BUF = 10 * 1024 * 1024; let l1Truncated = false; let l1OriginalSize = 0; + // P2: clamp the initial idle/hard timers to the max-safe bound. Node fires setTimeout + // delays above 2^31-1 ms IMMEDIATELY (silently disabling the timer), so an oversized + // idle_timeout_ms/hard_timeout_ms would leave a slot with no timeout at all. Mirrors + // stallTimeoutFor()'s clamp, which previously guarded only the stall window. + idleTimeoutMs = Math.min(idleTimeoutMs, TIMEOUT_MAX); + hardTimeoutMs = Math.min(hardTimeoutMs, TIMEOUT_MAX); // Kill entire process group, then destroy streams to force 'close' even if // grandchildren keep the pipes open (the common case with ccr/opencode). @@ -599,21 +605,25 @@ function runSubprocess(provider, prompt, idleTimeoutMs, hardTimeoutMs, allowedTo let rateLimitHits = 0; const RATE_LIMIT_THRESHOLD = 2; // kill after 2 consecutive rate-limit messages let totalBytesReceived = 0; - // Tighter idle timeout when the CLI has produced < STALL_BYTE_THRESHOLD bytes - // (header-only → probably hung). Per-slot override via providers.json - // `stall_timeout_ms` (see stallTimeoutFor): slow models (e.g. GLM-5.2[1m], - // MiniMax-M3) legitimately emit a small preamble then pause >30s mid-generation - // — they are slow, not stalled — so they need a longer threshold to avoid being - // false-killed. + // P2 — STALL detection = "no FIRST byte", NOT "model is thinking". + // Prior bug: the idle window was tightened to STALL_TIMEOUT_MS whenever total output + // was < 500 bytes, so a slow model that emitted a short preamble (e.g. 202-byte CLI + // framing) then paused >30s mid-generation was false-killed as STALL. A stall timeout + // is meant for "the process stopped writing", not "the model is still generating". + // Fix: a dedicated first-byte timer catches the genuine dead-on-arrival hang (zero bytes + // by STALL_TIMEOUT_MS); once ANY byte arrives it is cleared and the normal per-chunk + // idle window governs — so slow thinkers stream fine. Per-slot `stall_timeout_ms` + // (stallTimeoutFor) still tunes the first-byte budget for slow-to-start providers. const STALL_TIMEOUT_MS = stallTimeoutFor(provider); - const STALL_BYTE_THRESHOLD = 500; // below this = "just a header, probably stalled" + let firstByteTimer = setTimeout(() => { + if (totalBytesReceived === 0 && !timedOut) { timedOut = true; timeoutType = 'STALL'; killGroup(); } + }, STALL_TIMEOUT_MS); child.stdout.on('data', d => { totalBytesReceived += d.length; + clearTimeout(firstByteTimer); // first byte seen → past the dead-on-arrival window clearTimeout(idleTimer); - // Adaptive idle: use tighter 30s timeout when CLI has barely produced output (header-only stall) - const effectiveIdle = totalBytesReceived < STALL_BYTE_THRESHOLD ? STALL_TIMEOUT_MS : idleTimeoutMs; - idleTimer = setTimeout(() => { timedOut = true; timeoutType = totalBytesReceived < STALL_BYTE_THRESHOLD ? 'STALL' : 'IDLE'; killGroup(); }, effectiveIdle); + idleTimer = setTimeout(() => { timedOut = true; timeoutType = 'IDLE'; killGroup(); }, idleTimeoutMs); const chunk = d.toString(); l1OriginalSize += chunk.length; if (stdout.length < MAX_BUF) { @@ -639,9 +649,9 @@ function runSubprocess(provider, prompt, idleTimeoutMs, hardTimeoutMs, allowedTo }); child.stderr.on('data', d => { totalBytesReceived += d.length; + clearTimeout(firstByteTimer); // first byte seen (stderr counts) → past dead-on-arrival clearTimeout(idleTimer); - const effectiveIdle = totalBytesReceived < STALL_BYTE_THRESHOLD ? STALL_TIMEOUT_MS : idleTimeoutMs; - idleTimer = setTimeout(() => { timedOut = true; timeoutType = totalBytesReceived < STALL_BYTE_THRESHOLD ? 'STALL' : 'IDLE'; killGroup(); }, effectiveIdle); + idleTimer = setTimeout(() => { timedOut = true; timeoutType = 'IDLE'; killGroup(); }, idleTimeoutMs); const chunk = d.toString().slice(0, 4096); stderr += chunk; // Check stderr for rate-limit patterns too (gemini logs to stderr) @@ -659,6 +669,7 @@ function runSubprocess(provider, prompt, idleTimeoutMs, hardTimeoutMs, allowedTo child.on('close', (code) => { clearTimeout(idleTimer); clearTimeout(hardTimer); + clearTimeout(firstByteTimer); if (timedOut) { // Content-based reclassification: when STALL fires, inspect the partial // stdout/stderr for known error patterns before using the generic label. From 17b7d85cc42a492362ce419f89393629869d1149 Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Wed, 1 Jul 2026 15:06:44 +0100 Subject: [PATCH 2/7] harden(quorum) P3: degraded-panel gate as authoritative machine field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "available < max_quorum_size → BLOCK unless --force-quorum" decision lived only in quorum.md prose (LLM-interpreted), so a panel that collapsed to 1/7 could run silent reduced rounds with no deterministic gate. This session did exactly that — 6 rounds at 1/7. Add computeQuorumGate(availableCount, maxSize, forceQuorum) — a pure, exported, tested function — and emit its result on the preflight --all/--probe output: { available_count, quorum_met, degraded, blocked, waiver_required|waiver_used, gate_reason }. --force-quorum records an explicit waiver so the override is machine-visible. Invariant: when `blocked` is true, no downstream path may dispatch without a recorded waiver. `degraded` (1 <= available < max) is the signal P1's deep-probe gate will auto-enable on. 7 new unit tests (incl. the 1-of-7 production shape); existing preflight roster/probe/dedup suites still green. Co-Authored-By: Claude Opus 4.8 --- bin/quorum-preflight-gate.test.cjs | 68 ++++++++++++++++++++++++++++++ bin/quorum-preflight.cjs | 46 +++++++++++++++++++- 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 bin/quorum-preflight-gate.test.cjs diff --git a/bin/quorum-preflight-gate.test.cjs b/bin/quorum-preflight-gate.test.cjs new file mode 100644 index 0000000000..597a0ff059 --- /dev/null +++ b/bin/quorum-preflight-gate.test.cjs @@ -0,0 +1,68 @@ +'use strict'; + +// P3 — authoritative degraded-panel gate. Previously the "available < max_quorum_size → +// BLOCK unless --force-quorum" decision lived in quorum.md prose (LLM-interpreted), so a +// panel that collapsed to 1/7 could run 6 silent reduced rounds. computeQuorumGate makes +// it a deterministic, testable machine field. + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { computeQuorumGate } = require('./quorum-preflight.cjs'); + +describe('computeQuorumGate — degraded-panel gate', () => { + it('quorum met (available >= max): not blocked, not degraded', () => { + const g = computeQuorumGate(5, 3, false); + assert.equal(g.quorum_met, true); + assert.equal(g.blocked, false); + assert.equal(g.degraded, false); + assert.equal(g.available_count, 5); + assert.ok(!g.waiver_required); + }); + + it('exactly at threshold (available === max): met, not blocked', () => { + const g = computeQuorumGate(3, 3, false); + assert.equal(g.quorum_met, true); + assert.equal(g.blocked, false); + assert.equal(g.degraded, false); + }); + + it('degraded (1 <= available < max) without waiver: BLOCKED + waiver_required + degraded', () => { + const g = computeQuorumGate(1, 3, false); + assert.equal(g.quorum_met, false); + assert.equal(g.blocked, true); + assert.equal(g.waiver_required, true); + assert.equal(g.degraded, true); + assert.match(g.gate_reason, /only 1\/3/); + }); + + it('panel down (0 available) without waiver: BLOCKED, degraded=false (nothing to run)', () => { + const g = computeQuorumGate(0, 3, false); + assert.equal(g.blocked, true); + assert.equal(g.waiver_required, true); + assert.equal(g.degraded, false); // 0 slots is "panel down", not "degraded" + assert.match(g.gate_reason, /panel down/); + }); + + it('--force-quorum waives a degraded panel: not blocked, waiver_used recorded', () => { + const g = computeQuorumGate(1, 3, true); + assert.equal(g.blocked, false); + assert.equal(g.waiver_used, true); + assert.ok(!g.waiver_required); + assert.match(g.gate_reason, /WAIVED via --force-quorum/); + }); + + it('--force-quorum on a met quorum is a no-op (no spurious waiver_used)', () => { + const g = computeQuorumGate(4, 3, true); + assert.equal(g.blocked, false); + assert.ok(!g.waiver_used); // waiver only recorded when it actually overrode a block + }); + + it('the production failure shape: 1 of 7 with default max 3 blocks unless forced', () => { + // This is exactly the session that motivated P3 — codex-1 only, panel collapsed. + const blocked = computeQuorumGate(1, 3, false); + assert.equal(blocked.blocked, true, '1/7-style panel must block by default'); + const forced = computeQuorumGate(1, 3, true); + assert.equal(forced.blocked, false, '--force-quorum must allow explicit reduced-quorum runs'); + assert.equal(forced.waiver_used, true); + }); +}); diff --git a/bin/quorum-preflight.cjs b/bin/quorum-preflight.cjs index 17b5f7aeb0..75d6e00519 100644 --- a/bin/quorum-preflight.cjs +++ b/bin/quorum-preflight.cjs @@ -37,6 +37,11 @@ const { loadProviders } = require('./resolve-providers.cjs'); // Probe is ON by default for --all; --no-probe to skip, --probe still accepted for compat const NO_PROBE = process.argv.includes('--no-probe'); const PROBE = !NO_PROBE; +// P3 — when the panel is degraded (fewer available slots than max_quorum_size), preflight +// emits an authoritative `blocked`/`waiver_required` gate. --force-quorum records an explicit +// waiver so the machine field reflects the override deterministically (invariant: no downstream +// dispatch on a blocked panel without this flag). +const FORCE_QUORUM = process.argv.includes('--force-quorum'); // ─── Time-budget parsing ──────────────────────────────────────────────────── // --budget-ms threads a soft deadline that --all degrades within: when the @@ -632,6 +637,15 @@ async function main() { if (output.backup_slots.length > 0) { process.stderr.write(`[preflight] Tiered ordering: ${output.primary_slots.length} primary (CLI) + ${output.backup_slots.length} backup (HTTP API)\n`); } + + // ─── P3 — authoritative degraded-panel gate (see computeQuorumGate). Machine + // field so the block/waiver decision is deterministic + testable, not re-derived + // by LLM-interpreted markdown. `available_count` = distinct healthy slots eligible + // to vote (deduped primaries + HTTP backups; demoted duplicates excluded). + Object.assign(output, computeQuorumGate(output.available_slots.length, maxSize, FORCE_QUORUM)); + if (output.blocked) { + process.stderr.write(`[preflight] ${output.gate_reason}\n`); + } } console.log(JSON.stringify(output)); @@ -642,8 +656,38 @@ async function main() { } } +// ─── P3 — authoritative degraded-panel gate (pure, testable) ───────────────── +// Was LLM-interpreted prose in quorum.md (availableCount < max_quorum_size → BLOCK +// unless --force-quorum). Now a machine field so the decision is deterministic. +// Invariant (quorum review 2026-07-01): when `blocked` is true, no downstream path may +// dispatch a quorum without a recorded --force-quorum waiver. +function computeQuorumGate(availableCount, maxSize, forceQuorum) { + const quorumMet = availableCount >= maxSize; + const gate = { + available_count: availableCount, + quorum_met: quorumMet, + // `degraded` = some slots present but fewer than required. P1's deep-probe gate auto-enables on this. + degraded: availableCount >= 1 && !quorumMet, + }; + if (quorumMet) { + gate.blocked = false; + gate.gate_reason = `quorum met: ${availableCount}/${maxSize} slots available`; + } else if (forceQuorum) { + gate.blocked = false; + gate.waiver_used = true; + gate.gate_reason = `reduced quorum WAIVED via --force-quorum: ${availableCount}/${maxSize} available`; + } else { + gate.blocked = true; + gate.waiver_required = true; + gate.gate_reason = availableCount === 0 + ? `BLOCKED: 0/${maxSize} slots available (panel down) — pass --force-quorum only with explicit user awareness` + : `BLOCKED: only ${availableCount}/${maxSize} slots available — pass --force-quorum to proceed on a reduced quorum`; + } + return gate; +} + if (require.main === module) { main().catch(e => { console.error(e); process.exit(1); }); } -module.exports = { dedupBySlotIdentity, probeHealth, findProviders }; +module.exports = { dedupBySlotIdentity, probeHealth, findProviders, computeQuorumGate }; From 44e12c2a85139ec5503c2ca415a3bf29340d5762 Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Wed, 1 Jul 2026 15:09:31 +0100 Subject: [PATCH 3/7] harden(quorum) P4: quota-reset-aware cooldown (not the fixed 30-min TTL) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A quota-dead slot (e.g. antigravity-1: Google 429 "Resets in 32h54m49s") was re-probed and re-failed every 30 minutes because the failure TTL is far shorter than a rolling daily quota. Now the reset window is parsed from the error text and used as the cooldown, so the slot is held out until it actually recovers. - parseQuotaResetMs(msg): parses "Resets in 32h54m49s", "resets in 1h30m", "try again in 30 minutes", "in 2 hours", etc. → ms (clamped [0, 48h]); null if absent. - writeFailureLog: for QUOTA failures, stamp cooldown_until = now + parsed window; GC now preserves records whose cooldown_until is still in the future (else the long cooldown would be lost after the 60-min GC). - probeInferenceHistory: a record with a future cooldown_until keeps the slot unavailable until the real reset (surfaced as "cooling down ~Nmin"), with a `retired` flag for QUOTA holds. Non-quota blips never set cooldown_until, so a one-off failure can't retire a slot (addresses the reviewer's retirement-criterion concern). 6 new parser unit tests; preflight-probe / call-quorum infra/retry/stall / P3-gate suites all still green. Co-Authored-By: Claude Opus 4.8 --- bin/call-quorum-slot-quota-reset.test.cjs | 45 +++++++++++++++++++ bin/call-quorum-slot.cjs | 55 ++++++++++++++++++++--- bin/quorum-preflight.cjs | 19 ++++++-- 3 files changed, 111 insertions(+), 8 deletions(-) create mode 100644 bin/call-quorum-slot-quota-reset.test.cjs diff --git a/bin/call-quorum-slot-quota-reset.test.cjs b/bin/call-quorum-slot-quota-reset.test.cjs new file mode 100644 index 0000000000..76a8979502 --- /dev/null +++ b/bin/call-quorum-slot-quota-reset.test.cjs @@ -0,0 +1,45 @@ +'use strict'; + +// P4 — parse a provider's quota-reset window from its error text so the cooldown +// matches the real quota (rolling ~33h) instead of the fixed 30-min failure TTL. +// Motivated by antigravity-1 (Google 429 "Resets in 32h54m49s") being re-probed and +// re-failed every 30 min all session. + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { parseQuotaResetMs } = require('./call-quorum-slot.cjs'); + +const H = 3600000, M = 60000, S = 1000; + +describe('parseQuotaResetMs — quota reset-window parser', () => { + it('parses the compact Google form "Resets in 32h54m49s"', () => { + assert.equal(parseQuotaResetMs('RESOURCE_EXHAUSTED (code 429): Resets in 32h54m49s.'), + 32 * H + 54 * M + 49 * S); + }); + + it('parses "resets in 1h30m" and "resets in 45m"', () => { + assert.equal(parseQuotaResetMs('quota reached, resets in 1h30m'), 1 * H + 30 * M); + assert.equal(parseQuotaResetMs('resets in 45m'), 45 * M); + }); + + it('parses verbose "try again in 30 minutes" / "in 2 hours" / "in 45 seconds"', () => { + assert.equal(parseQuotaResetMs('rate limited, try again in 30 minutes'), 30 * M); + assert.equal(parseQuotaResetMs('quota exceeded; available in 2 hours'), 2 * H); + assert.equal(parseQuotaResetMs('retry in 45 seconds'), 45 * S); + }); + + it('returns null when no reset window is present', () => { + assert.equal(parseQuotaResetMs('429 Too Many Requests'), null); + assert.equal(parseQuotaResetMs('some unrelated error'), null); + assert.equal(parseQuotaResetMs(''), null); + assert.equal(parseQuotaResetMs(null), null); + }); + + it('clamps absurd windows to 48h (defends against a malformed "9999h")', () => { + assert.equal(parseQuotaResetMs('resets in 9999h'), 48 * H); + }); + + it('does not misfire on a bare number without a unit', () => { + assert.equal(parseQuotaResetMs('error 500 on attempt 3'), null); + }); +}); diff --git a/bin/call-quorum-slot.cjs b/bin/call-quorum-slot.cjs index 7d616c332a..760230889d 100644 --- a/bin/call-quorum-slot.cjs +++ b/bin/call-quorum-slot.cjs @@ -223,6 +223,37 @@ function classifyErrorType(msg) { return 'UNKNOWN'; } +// ─── P4 — parse a provider's quota-reset window from its error text ────────── +// Quota errors often carry the reset time ("Resets in 32h54m49s", "resets in 30 +// minutes", "try again in 3600 seconds"). The default 30-min failure TTL is far +// shorter than a rolling daily quota (~33h), so a quota-dead slot was re-probed and +// re-failed every 30 min. Parsing the real reset lets the cooldown match it exactly. +// Returns milliseconds until reset (clamped to [0, 48h]) or null if not present. +function parseQuotaResetMs(msg) { + if (!msg) return null; + const s = String(msg); + const MAX = 48 * 3600 * 1000; + const clamp = (ms) => (ms > 0 ? Math.min(ms, MAX) : null); + // Compact form: 32h54m49s / 1h30m / 45m / 90s, optionally after "resets in". + let m = s.match(/resets?\s+in\s+((?:\d+\s*[hms]\s*){1,3})/i) || s.match(/\b((?:\d+\s*[hms]\s*){2,3})\b/i); + if (m) { + let ms = 0, mm; const re = /(\d+)\s*([hms])/gi; + while ((mm = re.exec(m[1]))) { + const n = Number(mm[1]); const u = mm[2].toLowerCase(); + ms += n * (u === 'h' ? 3600000 : u === 'm' ? 60000 : 1000); + } + const c = clamp(ms); if (c) return c; + } + // Verbose: "resets in 30 minutes" / "try again in 2 hours" / "in 45 seconds". + m = s.match(/(?:resets?|try again|retry|available)\s+(?:in\s+)?(\d+)\s*(hours?|minutes?|seconds?|hrs?|mins?|secs?)\b/i); + if (m) { + const n = Number(m[1]); const u = m[2].toLowerCase(); + const mult = u.startsWith('h') ? 3600000 : u.startsWith('m') ? 60000 : 1000; + return clamp(n * mult); + } + return null; +} + function writeFailureLog(slotName, errorMsg, stderrText) { try { const pp = require('./planning-paths.cjs'); @@ -234,22 +265,36 @@ function writeFailureLog(slotName, errorMsg, stderrText) { const rawPattern = (stderrText && stderrText.length > 0) ? stderrText : errorMsg; const pattern = rawPattern.replace(/\x1b\[[0-9;]*m/g, '').slice(0, 200); + // P4: for QUOTA failures, parse the provider's reset window so the cooldown matches + // the real quota (rolling ~33h) instead of the 30-min TTL. + const quotaResetMs = error_type === 'QUOTA' ? parseQuotaResetMs(rawPattern) : null; + const cooldownUntil = quotaResetMs ? new Date(Date.now() + quotaResetMs).toISOString() : null; + // Lock-free read-modify-write under an exclusive writer lock so parallel slots // never lose records (and a torn read never wipes the whole log). atomicUpdateJson(logPath, (current) => { let records = Array.isArray(current) ? current : []; - // Garbage-collect stale records (older than 60 minutes) to prevent unbounded growth - const gcCutoff = Date.now() - 60 * 60 * 1000; - records = records.filter(r => new Date(r.last_seen).getTime() > gcCutoff); + // Garbage-collect stale records (older than 60 minutes) to prevent unbounded growth. + // EXCEPT: keep records whose cooldown_until is still in the future (a quota-dead slot + // whose reset is hours away must survive GC, or P4's long cooldown is lost after 60 min). + const now = Date.now(); + const gcCutoff = now - 60 * 60 * 1000; + records = records.filter(r => + new Date(r.last_seen).getTime() > gcCutoff || + (r.cooldown_until && new Date(r.cooldown_until).getTime() > now)); // Update or insert record const existing = records.find(r => r.slot === slotName && r.error_type === error_type); if (existing) { existing.count++; existing.last_seen = new Date().toISOString(); + // Keep the latest known reset (a fresh quota message may push it out). + if (cooldownUntil) existing.cooldown_until = cooldownUntil; } else { - records.push({ slot: slotName, error_type, pattern, count: 1, last_seen: new Date().toISOString() }); + const rec = { slot: slotName, error_type, pattern, count: 1, last_seen: new Date().toISOString() }; + if (cooldownUntil) rec.cooldown_until = cooldownUntil; + records.push(rec); } return records; }, []); @@ -1117,4 +1162,4 @@ if (require.main === module) { } // ─── Test exports (SHELL-ESCAPE-01, TRUNC-01, INFRA-367) ─────────────────────── -module.exports = { buildSpawnArgs, stallTimeoutFor, recordTelemetry, findProjectRoot, writeFailureLog, atomicUpdateJson, VERDICTS, VERDICT_LINE_RE, parseVerdictLine }; +module.exports = { buildSpawnArgs, stallTimeoutFor, recordTelemetry, findProjectRoot, writeFailureLog, atomicUpdateJson, parseQuotaResetMs, VERDICTS, VERDICT_LINE_RE, parseVerdictLine }; diff --git a/bin/quorum-preflight.cjs b/bin/quorum-preflight.cjs index 75d6e00519..5b49633b2d 100644 --- a/bin/quorum-preflight.cjs +++ b/bin/quorum-preflight.cjs @@ -340,16 +340,29 @@ function probeInferenceHistory(ttlMinutes = 30) { if (!fs.existsSync(logPath)) return {}; const records = JSON.parse(fs.readFileSync(logPath, 'utf8')); if (!Array.isArray(records)) return {}; - const cutoff = Date.now() - ttlMinutes * 60 * 1000; + const now = Date.now(); + const cutoff = now - ttlMinutes * 60 * 1000; const result = {}; for (const r of records) { - if (new Date(r.last_seen).getTime() > cutoff) { + // P4 — a QUOTA failure with a parsed reset window (cooldown_until) keeps the slot + // unavailable until the ACTUAL reset (rolling ~33h), not just the 30-min TTL — so a + // quota-dead slot is not re-probed and re-failed every 30 min. Otherwise fall back + // to the recency TTL. The `retired` flag marks a slot held out by a long cooldown + // (transparency for the roster); a single non-quota blip never sets it. + const cooldownUntil = r.cooldown_until ? new Date(r.cooldown_until).getTime() : 0; + const coolingDown = cooldownUntil > now; + const recent = new Date(r.last_seen).getTime() > cutoff; + if (recent || coolingDown) { + const mins = coolingDown ? Math.round((cooldownUntil - now) / 60000) : 0; result[r.slot] = { ok: false, - reason: `${r.error_type}: ${(r.pattern || '').slice(0, 100)}`, + reason: coolingDown + ? `${r.error_type}: cooling down ~${mins}min (until ${r.cooldown_until})` + : `${r.error_type}: ${(r.pattern || '').slice(0, 100)}`, error_type: r.error_type, count: r.count, last_seen: r.last_seen, + ...(coolingDown ? { cooldown_until: r.cooldown_until, retired: r.error_type === 'QUOTA' } : {}), }; } } From edc4ef547df3a9c012e3b3ebe19d38f8d8196b3f Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Wed, 1 Jul 2026 15:11:54 +0100 Subject: [PATCH 4/7] harden(quorum) P5: providers.json schema validator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was no schema gate on providers.json: a subprocess slot with no resolvable spawn target, or an http slot missing baseUrl/apiKeyEnv, surfaced only as a spawn crash at dispatch (spawn(null) took the whole quorum offline). A slot with no deep_probe (e.g. antigravity-1) failed opaquely instead of being inference-gated. Add validateProviders(providers) — pure, exported, tested — returning { ok, errors, warnings }: errors for unspawnable subprocess slots, http slots missing baseUrl/apiKeyEnv, missing type, and duplicate names; warnings for inference slots lacking a deep_probe. Wired as non-fatal diagnostics on the preflight --all path so config drift is caught up front. 7 new unit tests (incl. the antigravity-1 no-deep_probe shape); preflight gate/probe/roster/dedup suites still green. NOTE (operational, not code): reconciling the drifted providers.json copies (repo scaffold vs ~/.claude/nf-bin vs .claude/nf/bin) and dropping the quota-dead antigravity-1 from nf.json quorum_active are user-config actions left to the operator; P4's cooldown already holds antigravity-1 out until its quota resets. Co-Authored-By: Claude Opus 4.8 --- bin/quorum-preflight-validate.test.cjs | 62 ++++++++++++++++++++++++++ bin/quorum-preflight.cjs | 38 +++++++++++++++- 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 bin/quorum-preflight-validate.test.cjs diff --git a/bin/quorum-preflight-validate.test.cjs b/bin/quorum-preflight-validate.test.cjs new file mode 100644 index 0000000000..749d697bfb --- /dev/null +++ b/bin/quorum-preflight-validate.test.cjs @@ -0,0 +1,62 @@ +'use strict'; + +// P5 — providers.json schema validator. There was no gate on the config, so an +// unspawnable subprocess slot (no cli/mainTool) or an http slot missing baseUrl/apiKeyEnv +// only surfaced as a spawn crash at dispatch. This catches drift up front. + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { validateProviders } = require('./quorum-preflight.cjs'); + +describe('validateProviders — providers.json schema gate', () => { + it('accepts a well-formed subprocess slot with a deep_probe', () => { + const r = validateProviders([ + { name: 'codex-1', type: 'subprocess', cli: '/usr/bin/codex', deep_probe: { prompt: 'x', expect: 'y' } }, + ]); + assert.equal(r.ok, true); + assert.equal(r.errors.length, 0); + assert.equal(r.warnings.length, 0); + }); + + it('errors when a subprocess slot has no spawn target (cli AND mainTool absent)', () => { + const r = validateProviders([{ name: 'broken-1', type: 'subprocess' }]); + assert.equal(r.ok, false); + assert.match(r.errors.join(), /broken-1: subprocess.*no spawn target/); + }); + + it('accepts a subprocess slot with only mainTool (cli optional)', () => { + const r = validateProviders([{ name: 'ok-1', type: 'subprocess', mainTool: 'codex', deep_probe: {} }]); + assert.equal(r.ok, true); + }); + + it('errors when an http slot is missing baseUrl / apiKeyEnv', () => { + const r = validateProviders([{ name: 'api-1', type: 'http', deep_probe: {} }]); + assert.equal(r.ok, false); + assert.match(r.errors.join(), /api-1: http slot missing baseUrl/); + assert.match(r.errors.join(), /api-1: http slot missing apiKeyEnv/); + }); + + it('warns (not errors) when an inference slot lacks a deep_probe — the antigravity-1 shape', () => { + // antigravity-1's live entry has cli but no deep_probe → it failed opaquely as + // BINARY_MISSING instead of being inference-health-gated. + const r = validateProviders([{ name: 'antigravity-1', type: 'subprocess', cli: '/x/agy' }]); + assert.equal(r.ok, true); // spawnable, so not an error + assert.match(r.warnings.join(), /antigravity-1: no deep_probe/); + }); + + it('errors on a missing type and on duplicate names', () => { + const r = validateProviders([ + { name: 'x' }, + { name: 'dupe', type: 'subprocess', cli: '/a', deep_probe: {} }, + { name: 'dupe', type: 'subprocess', cli: '/b', deep_probe: {} }, + ]); + assert.equal(r.ok, false); + assert.match(r.errors.join(), /x: missing type/); + assert.match(r.errors.join(), /duplicate provider name: dupe/); + }); + + it('tolerates a non-array / empty input (fail-open)', () => { + assert.equal(validateProviders(null).ok, true); + assert.equal(validateProviders([]).ok, true); + }); +}); diff --git a/bin/quorum-preflight.cjs b/bin/quorum-preflight.cjs index 5b49633b2d..a58d232820 100644 --- a/bin/quorum-preflight.cjs +++ b/bin/quorum-preflight.cjs @@ -562,6 +562,11 @@ async function main() { console.log('OK'); } else if (mode === '--all') { const providers = findProviders(); + // P5 — surface providers.json config problems (drift, unspawnable slots, missing + // deep_probe) as non-fatal diagnostics rather than letting them crash at fan-out. + const _val = validateProviders(providers); + for (const e of _val.errors) process.stderr.write(`[preflight] config ERROR: ${e}\n`); + for (const w of _val.warnings) process.stderr.write(`[preflight] config warn: ${w}\n`); const active = cfg.quorum_active || []; const team = buildTeam(providers, active); const maxSize = cfg.max_quorum_size ?? 3; @@ -699,8 +704,39 @@ function computeQuorumGate(availableCount, maxSize, forceQuorum) { return gate; } +// ─── P5 — providers.json schema validator (pure, testable) ─────────────────── +// There was no schema gate on providers.json: a subprocess slot with no resolvable +// spawn target, or an http slot missing baseUrl/apiKeyEnv, only surfaced as a spawn +// crash at dispatch (spawn(null) → whole quorum offline). A slot with no deep_probe +// can't be inference-health-gated (P1). This validator surfaces those as errors / +// warnings so config drift is caught up front instead of at fan-out time. +function validateProviders(providers) { + const errors = [], warnings = []; + const list = Array.isArray(providers) ? providers : []; + const seen = new Set(); + for (const p of list) { + if (!p || typeof p !== 'object' || !p.name) { errors.push('provider entry missing/!object name'); continue; } + if (seen.has(p.name)) errors.push(`duplicate provider name: ${p.name}`); + seen.add(p.name); + const type = p.type; + if (type === 'subprocess' || type === 'ccr') { + if (!p.cli && !p.mainTool) errors.push(`${p.name}: subprocess/ccr slot has no spawn target (cli or mainTool)`); + } else if (type === 'http') { + if (!p.baseUrl) errors.push(`${p.name}: http slot missing baseUrl`); + if (!p.apiKeyEnv) errors.push(`${p.name}: http slot missing apiKeyEnv`); + } else if (!type) { + errors.push(`${p.name}: missing type`); + } + // Inference slots without a deep_probe can't be health-gated by P1's deep layer. + if ((type === 'subprocess' || type === 'ccr' || type === 'http') && !p.deep_probe) { + warnings.push(`${p.name}: no deep_probe — cannot be inference-health-gated`); + } + } + return { ok: errors.length === 0, errors, warnings }; +} + if (require.main === module) { main().catch(e => { console.error(e); process.exit(1); }); } -module.exports = { dedupBySlotIdentity, probeHealth, findProviders, computeQuorumGate }; +module.exports = { dedupBySlotIdentity, probeHealth, findProviders, computeQuorumGate, validateProviders }; From 1f90ca801061a094f855ae84ec40c4ce344ece7a Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Wed, 1 Jul 2026 15:14:38 +0100 Subject: [PATCH 5/7] harden(quorum) P1: deep-probe gate policy (spawn wiring deferred to harness) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit L1 (--version) and L2 (/models) can't distinguish a quota/auth-dead slot from a healthy one — L2 even treats HTTP 401/403 as reachable — so a dead slot passes preflight and only fails during the real review (this session's failure). P1 adds a deep inference probe to catch it. Landed here (pure, tested, regression-safe): - --deep flag; shouldRunDeepProbe() auto-enables the gate when the panel is degraded (P3's `degraded`), budget permitting — happy path untouched (default off). - classifyDeepProbeResult(): downgrades a slot ONLY on a FAST explicit auth/quota signal (the class L1/L2 miss). A TIMEOUT is INCONCLUSIVE and never downgrades, so a slow-but-healthy slot (the P2 failure mode) cannot be re-killed by the deep probe; ambiguous non-error output is assumed alive (never false-kill on ambiguity). Deliberately NOT wired: the live subprocess spawn is not inserted into the main() hot path yet. An untested spawn on the path that gates EVERY quorum is the single change that could take the whole system offline, so it lands with a mock-CLI integration harness in a focused follow-up (the same harness that will time-test P2's runSubprocess and the reviewers' "quota-dead slot blocked before review" e2e case). 10 new policy unit tests; full preflight suite (gate/validate/probe/roster/dedup) green. Co-Authored-By: Claude Opus 4.8 --- bin/quorum-preflight-deep.test.cjs | 59 ++++++++++++++++++++++++++++++ bin/quorum-preflight.cjs | 44 +++++++++++++++++++++- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 bin/quorum-preflight-deep.test.cjs diff --git a/bin/quorum-preflight-deep.test.cjs b/bin/quorum-preflight-deep.test.cjs new file mode 100644 index 0000000000..7033942640 --- /dev/null +++ b/bin/quorum-preflight-deep.test.cjs @@ -0,0 +1,59 @@ +'use strict'; + +// P1 — deep inference probe policy (pure logic). The spawn is deferred to a +// harness-backed follow-up; these guard the safety-critical policy: +// - downgrade a slot ONLY on a fast explicit auth/quota signal (what L1/L2 miss); +// - a TIMEOUT is inconclusive and never downgrades (so a slow-but-healthy slot — +// the P2 failure mode — is not re-killed by the deep probe); +// - run the probe when --deep or when the panel is degraded, budget permitting. + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { shouldRunDeepProbe, classifyDeepProbeResult } = require('./quorum-preflight.cjs'); + +describe('shouldRunDeepProbe — when to run the deep gate', () => { + it('runs when --deep is set', () => { + assert.equal(shouldRunDeepProbe({ deep: true, degraded: false, budgetMs: null }), true); + }); + it('auto-runs when the panel is degraded even without --deep', () => { + assert.equal(shouldRunDeepProbe({ deep: false, degraded: true, budgetMs: null }), true); + }); + it('does NOT run on a healthy panel with no --deep (happy path untouched)', () => { + assert.equal(shouldRunDeepProbe({ deep: false, degraded: false, budgetMs: null }), false); + }); + it('is skipped when the time budget cannot cover a probe', () => { + assert.equal(shouldRunDeepProbe({ deep: true, degraded: true, budgetMs: 1000, minBudgetMs: 45000 }), false); + assert.equal(shouldRunDeepProbe({ deep: true, degraded: true, budgetMs: 60000, minBudgetMs: 45000 }), true); + }); +}); + +describe('classifyDeepProbeResult — downgrade only on fast auth/quota', () => { + it('downgrades on an explicit auth failure (401/403/unauthorized)', () => { + assert.equal(classifyDeepProbeResult('HTTP 401 unauthorized').ok, false); + assert.equal(classifyDeepProbeResult('Error: invalid api key').classification, 'AUTH'); + }); + it('downgrades on an explicit quota/rate-limit signal (429/quota/resource exhausted)', () => { + assert.equal(classifyDeepProbeResult('RESOURCE_EXHAUSTED (code 429): quota reached').ok, false); + assert.equal(classifyDeepProbeResult('429 Too Many Requests').classification, 'QUOTA'); + }); + it('does NOT downgrade on a timeout — inconclusive, protecting slow-but-healthy slots (P2)', () => { + const r = classifyDeepProbeResult('', { timedOut: true }); + assert.equal(r.ok, true); + assert.equal(r.classification, 'INCONCLUSIVE'); + }); + it('passes when the expected token is present', () => { + const r = classifyDeepProbeResult('...\nPROBE_OK\n', { expect: 'PROBE_OK' }); + assert.equal(r.ok, true); + assert.equal(r.classification, 'OK'); + }); + it('assumes alive on ambiguous non-error output (never false-kills on ambiguity)', () => { + const r = classifyDeepProbeResult('some normal model chatter', { expect: 'PROBE_OK' }); + assert.equal(r.ok, true); + assert.equal(r.classification, 'INCONCLUSIVE'); + }); + it('the antigravity-1 shape: fast 429 with reset → downgraded (the L1/L2-missed case)', () => { + const r = classifyDeepProbeResult('RESOURCE_EXHAUSTED (code 429): Resets in 32h54m49s'); + assert.equal(r.ok, false); + assert.equal(r.classification, 'QUOTA'); + }); +}); diff --git a/bin/quorum-preflight.cjs b/bin/quorum-preflight.cjs index a58d232820..3afa6f4c12 100644 --- a/bin/quorum-preflight.cjs +++ b/bin/quorum-preflight.cjs @@ -42,6 +42,11 @@ const PROBE = !NO_PROBE; // waiver so the machine field reflects the override deterministically (invariant: no downstream // dispatch on a blocked panel without this flag). const FORCE_QUORUM = process.argv.includes('--force-quorum'); +// P1 — deep inference probe. Opt-in (--deep) and auto-enabled when the panel is degraded +// (P3). L1 (--version) and L2 (/models) can't tell a quota/auth-dead slot from a healthy +// one (L2 even treats 401/403 as reachable), so a dead slot passes preflight and only fails +// during the real review. The deep probe runs the provider's deep_probe prompt to catch it. +const DEEP_PROBE = process.argv.includes('--deep'); // ─── Time-budget parsing ──────────────────────────────────────────────────── // --budget-ms threads a soft deadline that --all degrades within: when the @@ -735,8 +740,45 @@ function validateProviders(providers) { return { ok: errors.length === 0, errors, warnings }; } +// ─── P1 — deep inference probe: decision + result classification (pure) ────── +// The spawn itself is intentionally NOT yet wired into the main() hot path — an +// untested subprocess probe on the path that gates EVERY quorum is the one change +// that could take the whole system down, so it lands with a mock-CLI integration +// harness in a follow-up. These two pure helpers encode the safe policy and are unit-tested. + +// Run the deep probe when explicitly requested OR when the panel is degraded (so a +// reduced panel is verified with real inference before we trust it), but only if the +// remaining time budget covers at least one probe. +function shouldRunDeepProbe({ deep, degraded, budgetMs, minBudgetMs = 45000 }) { + if (!deep && !degraded) return false; + if (budgetMs == null) return true; // no budget cap → allowed + return budgetMs >= minBudgetMs; +} + +// Classify a deep-probe result. CRITICAL: downgrade a slot ONLY on a FAST, explicit +// auth/quota signal — the exact class L1/L2 miss. A timeout is treated as INCONCLUSIVE +// and does NOT downgrade, so a slow-but-healthy slot (the P2 failure mode) is never +// false-killed by the deep probe. Ambiguous non-error output is assumed alive. +function classifyDeepProbeResult(output, { timedOut = false, expect = null } = {}) { + if (timedOut) { + return { ok: true, classification: 'INCONCLUSIVE', reason: 'deep-probe timed out (slow, not downgraded)' }; + } + const text = String(output || ''); + if (/\b(401|403)\b|unauthorized|forbidden|invalid.*api.?key/i.test(text)) { + return { ok: false, classification: 'AUTH', reason: 'deep-probe: authentication failure' }; + } + if (/\b(402|429)\b|quota|resource.?exhausted|too many requests|exhausted your capacity|rate.?limit/i.test(text)) { + return { ok: false, classification: 'QUOTA', reason: 'deep-probe: quota/rate-limit' }; + } + if (expect && text.includes(expect)) { + return { ok: true, classification: 'OK', reason: `deep-probe: matched "${expect}"` }; + } + // Non-empty, no error, no expect match → assume alive (never false-kill on ambiguity). + return { ok: true, classification: text.trim() ? 'INCONCLUSIVE' : 'EMPTY', reason: 'deep-probe: no error signal' }; +} + if (require.main === module) { main().catch(e => { console.error(e); process.exit(1); }); } -module.exports = { dedupBySlotIdentity, probeHealth, findProviders, computeQuorumGate, validateProviders }; +module.exports = { dedupBySlotIdentity, probeHealth, findProviders, computeQuorumGate, validateProviders, shouldRunDeepProbe, classifyDeepProbeResult }; From 2bf5a0c7baaf5d0593d0e9e15a4cec92382c5fe3 Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Wed, 1 Jul 2026 15:28:35 +0100 Subject: [PATCH 6/7] harden(quorum) P1 (live): wire deep-probe spawn into preflight + mock-CLI harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes P1 — the deep inference gate is now wired into preflight and exercised by a real subprocess harness (the deferred piece from the prior P1 commit). - deepProbeSlot(provider, budgetMs): spawns the provider's deep_probe prompt through the real CLI and classifies via classifyDeepProbeResult. Regression-safe: spawn/args errors and TIMEOUTS resolve ok:true (SKIP/INCONCLUSIVE) — it can ONLY downgrade on a fast explicit auth/quota signal, never on slowness (can't recreate the P2 false-kill). - Wired into `--all --probe`: after L1/L2/L3, if shouldRunDeepProbe() it deep-probes the available slots, moves any downgraded (auth/quota-dead) to unavailable (layer4), then computes the P3 gate on the post-deep roster — so a quota-dead slot that L1/L2 passed now blocks BEFORE any review runs. - CRITICAL fix vs the naive wiring: auto-enable on `degraded` requires an EXPLICIT budget (--budget-ms) that covers a probe. The cheap budget-less `--all --probe` liveness path must never pay deep-probe latency (a naive auto-enable made it spawnSync-ETIMEDOUT and broke its <8s budget). Real quorum dispatch passes --budget-ms, so it still auto-verifies a reduced panel; the fast probe path stays fast. Harness: bin/__deep-probe-mock.cjs (fake CLI: probe_ok/quota/auth/slow_ok/hang) drives 6 live integration tests incl. the reviewers' key E2E — a quota-dead slot is downgraded by the deep probe → the gate blocks a 2/3 panel before review. Slow-preamble-then-pause passes; hang→timeout is inconclusive (P2-safe). 110 tests green across the full call-quorum-slot + quorum-preflight suites. Co-Authored-By: Claude Opus 4.8 --- bin/__deep-probe-mock.cjs | 31 +++++++ ...quorum-preflight-deep-integration.test.cjs | 85 +++++++++++++++++++ bin/quorum-preflight-deep.test.cjs | 9 +- bin/quorum-preflight.cjs | 83 ++++++++++++++++-- 4 files changed, 198 insertions(+), 10 deletions(-) create mode 100755 bin/__deep-probe-mock.cjs create mode 100644 bin/quorum-preflight-deep-integration.test.cjs diff --git a/bin/__deep-probe-mock.cjs b/bin/__deep-probe-mock.cjs new file mode 100755 index 0000000000..f07761b351 --- /dev/null +++ b/bin/__deep-probe-mock.cjs @@ -0,0 +1,31 @@ +#!/usr/bin/env node +'use strict'; +// Test fixture — a fake provider CLI for deep-probe / stall integration tests. +// Behavior is chosen by MOCK_MODE; MOCK_DELAY_MS controls the slow/hang timing. +// probe_ok (default) — print PROBE_OK, exit 0 +// quota — print a 429/RESOURCE_EXHAUSTED line to stderr, exit 1 +// auth — print a 401 unauthorized line to stderr, exit 1 +// slow_ok — print a short preamble, wait MOCK_DELAY_MS, then PROBE_OK, exit 0 +// hang — print a short preamble, then never output/exit (until killed) +const mode = process.env.MOCK_MODE || 'probe_ok'; +const delay = Number(process.env.MOCK_DELAY_MS || 0); + +function main() { + if (mode === 'quota') { + process.stderr.write('RESOURCE_EXHAUSTED (code 429): Individual quota reached. Resets in 32h54m.\n'); + process.exit(1); + } else if (mode === 'auth') { + process.stderr.write('HTTP 401 unauthorized: invalid api key\n'); + process.exit(1); + } else if (mode === 'hang') { + process.stdout.write('PREAMBLE'.repeat(25)); // ~200-byte preamble, then silence forever + setInterval(() => {}, 1 << 30); + } else if (mode === 'slow_ok') { + process.stdout.write('PREAMBLE'.repeat(25)); // small preamble (<500B), like the real slow slots + setTimeout(() => { process.stdout.write('\nPROBE_OK\n'); process.exit(0); }, delay); + } else { + process.stdout.write('PROBE_OK\n'); + process.exit(0); + } +} +main(); diff --git a/bin/quorum-preflight-deep-integration.test.cjs b/bin/quorum-preflight-deep-integration.test.cjs new file mode 100644 index 0000000000..605bcbca54 --- /dev/null +++ b/bin/quorum-preflight-deep-integration.test.cjs @@ -0,0 +1,85 @@ +'use strict'; + +// P1 — LIVE deep-probe integration harness. Spawns a real (mock) provider CLI through +// deepProbeSlot and asserts the end-to-end classification. This is the harness that was +// deferred when P1's policy landed; it exercises the actual subprocess path, not just the +// pure classifier. The reviewers' "quota-dead slot blocked before review" case is the +// quota test below (deep probe downgrades → the P3 gate would then block). + +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const path = require('path'); +const { deepProbeSlot, computeQuorumGate } = require('./quorum-preflight.cjs'); + +const MOCK = path.join(__dirname, '__deep-probe-mock.cjs'); +try { fs.chmodSync(MOCK, 0o755); } catch (_) {} + +function mockProvider(mode, extra = {}) { + return { + name: `mock-${mode}`, + type: 'subprocess', + resolvedCli: MOCK, // resolveSpawnTarget prefers resolvedCli (absolute) + args_template: ['{prompt}'], // honored verbatim by resolveArgsTemplate + deep_probe: { prompt: 'respond with: PROBE_OK', expect: 'PROBE_OK', timeout_ms: 4000 }, + env: { MOCK_MODE: mode, ...extra }, + }; +} + +describe('deepProbeSlot — live spawn classification (integration)', () => { + it('healthy slot (prints PROBE_OK) → ok, OK', async () => { + const r = await deepProbeSlot(mockProvider('probe_ok'), 10000); + assert.equal(r.ok, true); + assert.equal(r.classification, 'OK'); + }); + + it('quota-dead slot (429/RESOURCE_EXHAUSTED) → DOWNGRADED (the L1/L2-missed case)', async () => { + const r = await deepProbeSlot(mockProvider('quota'), 10000); + assert.equal(r.ok, false); + assert.equal(r.classification, 'QUOTA'); + }); + + it('auth-dead slot (401) → DOWNGRADED', async () => { + const r = await deepProbeSlot(mockProvider('auth'), 10000); + assert.equal(r.ok, false); + assert.equal(r.classification, 'AUTH'); + }); + + it('slow-but-healthy slot (preamble, ~1.5s pause, then PROBE_OK) → ok, NOT killed', async () => { + // The P2 shape at the deep-probe layer: a short preamble then a pause. Must pass. + const r = await deepProbeSlot(mockProvider('slow_ok', { MOCK_DELAY_MS: '1500' }), 10000); + assert.equal(r.ok, true); + assert.equal(r.classification, 'OK'); + }); + + it('hung slot (preamble then silence) → timeout is INCONCLUSIVE, never downgraded (P2-safe)', async () => { + // A tight per-probe timeout fires; a timeout must NOT downgrade (would re-create the + // slow-slot false-kill). ok stays true. + const prov = mockProvider('hang'); + prov.deep_probe = { prompt: 'x', expect: 'PROBE_OK', timeout_ms: 800 }; + const r = await deepProbeSlot(prov, 10000); + assert.equal(r.ok, true); + assert.equal(r.classification, 'INCONCLUSIVE'); + }); + + it('unspawnable slot (no target) → SKIP, never false-kills', async () => { + const r = await deepProbeSlot({ name: 'x', type: 'subprocess', args_template: ['{prompt}'] }, 10000); + assert.equal(r.ok, true); + assert.equal(r.classification, 'SKIP'); + }); + + it('E2E (reviewers\' key test): a quota-dead slot is downgraded → gate BLOCKS before review', async () => { + // 3 slots pass L1/L2, max_quorum_size=3, but one is quota-dead. The deep probe catches + // it (L1/L2 could not), dropping available to 2/3 → computeQuorumGate blocks. Without the + // deep gate this quorum would have dispatched a real review on a dead slot. + const slots = ['probe_ok', 'probe_ok', 'quota']; + const results = await Promise.all(slots.map((m) => + deepProbeSlot(mockProvider(m), 10000).then(r => ({ m, r })))); + let available = slots.length; + for (const { r } of results) if (r.ok === false) available -= 1; + assert.equal(available, 2, 'the quota-dead slot must be downgraded by the deep probe'); + const gate = computeQuorumGate(available, 3, false); + assert.equal(gate.blocked, true, 'reduced (2/3) panel must block before any review runs'); + assert.equal(gate.waiver_required, true); + }); +}); diff --git a/bin/quorum-preflight-deep.test.cjs b/bin/quorum-preflight-deep.test.cjs index 7033942640..62838d4abf 100644 --- a/bin/quorum-preflight-deep.test.cjs +++ b/bin/quorum-preflight-deep.test.cjs @@ -15,8 +15,13 @@ describe('shouldRunDeepProbe — when to run the deep gate', () => { it('runs when --deep is set', () => { assert.equal(shouldRunDeepProbe({ deep: true, degraded: false, budgetMs: null }), true); }); - it('auto-runs when the panel is degraded even without --deep', () => { - assert.equal(shouldRunDeepProbe({ deep: false, degraded: true, budgetMs: null }), true); + it('auto-runs on a degraded panel WITH an explicit sufficient budget (real dispatch path)', () => { + assert.equal(shouldRunDeepProbe({ deep: false, degraded: true, budgetMs: 60000, minBudgetMs: 45000 }), true); + }); + it('does NOT auto-run on degraded with no budget — the cheap --all --probe path stays fast', () => { + // Regression guard: auto-running live deep probes in the budget-less liveness path + // blew its <8s budget (spawnSync ETIMEDOUT). Auto-enable requires an explicit budget. + assert.equal(shouldRunDeepProbe({ deep: false, degraded: true, budgetMs: null }), false); }); it('does NOT run on a healthy panel with no --deep (happy path untouched)', () => { assert.equal(shouldRunDeepProbe({ deep: false, degraded: false, budgetMs: null }), false); diff --git a/bin/quorum-preflight.cjs b/bin/quorum-preflight.cjs index 3afa6f4c12..904d8b6044 100644 --- a/bin/quorum-preflight.cjs +++ b/bin/quorum-preflight.cjs @@ -32,6 +32,7 @@ const { resolveCli } = require('./resolve-cli.cjs'); const https = require('https'); const http = require('http'); const { resolveSpawnTarget } = require('./resolve-cli.cjs'); +const { resolveArgsTemplate } = require('./provider-arg-templates.cjs'); const { loadProviders } = require('./resolve-providers.cjs'); // Probe is ON by default for --all; --no-probe to skip, --probe still accepted for compat @@ -375,6 +376,43 @@ function probeInferenceHistory(ttlMinutes = 30) { } catch (_) { return {}; } // fail-open } +// ─── P1 — live deep inference probe (spawn) ────────────────────────────────── +// Runs the provider's deep_probe prompt through the real CLI and classifies the +// result via classifyDeepProbeResult. Regression-safe by construction: a spawn/args +// error or a timeout resolves to ok:true (SKIP/INCONCLUSIVE) — the deep probe can +// ONLY downgrade a slot on a fast explicit auth/quota signal, never on slowness or a +// harness problem, so it cannot re-create the P2 slow-slot false-kill or knock out a +// slot L1/L2 already vetted. +function deepProbeSlot(provider, budgetMs) { + return new Promise((resolve) => { + const probe = provider.deep_probe || { prompt: 'respond with: PROBE_OK', expect: 'PROBE_OK', timeout_ms: 45000 }; + const target = resolveSpawnTarget(provider) || provider.cli; + const tmpl = resolveArgsTemplate(provider); + if (!target || !Array.isArray(tmpl)) { + resolve({ ok: true, classification: 'SKIP', reason: 'no spawn target / args_template' }); + return; + } + const args = tmpl.map(a => (a === '{prompt}' ? probe.prompt : a)); + const timeoutMs = Math.min(probe.timeout_ms || 45000, budgetMs == null ? 2147483647 : budgetMs); + let out = '', done = false, timedOut = false, child; + const finish = (r) => { + if (done) return; done = true; + try { process.kill(-child.pid, 'SIGKILL'); } catch (_) { try { child.kill('SIGKILL'); } catch (_) {} } + resolve(r); + }; + try { + child = spawn(target, args, { env: { ...process.env, ...(provider.env || {}) }, stdio: ['pipe', 'pipe', 'pipe'], detached: true }); + } catch (e) { resolve({ ok: true, classification: 'SKIP', reason: 'spawn error: ' + e.message }); return; } + const timer = setTimeout(() => { timedOut = true; finish(classifyDeepProbeResult(out, { timedOut: true, expect: probe.expect })); }, timeoutMs); + // ccr slots read the prompt from stdin; others are non-interactive. + try { if (provider.type === 'ccr') { child.stdin.write(probe.prompt); child.stdin.end(); } else { child.stdin.end(); } } catch (_) {} + child.stdout.on('data', d => { if (out.length < 65536) out += d.toString(); }); + child.stderr.on('data', d => { if (out.length < 65536) out += d.toString(); }); + child.on('close', () => { clearTimeout(timer); if (!timedOut) finish(classifyDeepProbeResult(out, { timedOut: false, expect: probe.expect })); }); + child.on('error', (e) => { clearTimeout(timer); finish({ ok: true, classification: 'SKIP', reason: 'spawn error: ' + e.message }); }); + }); +} + // ─── Two-layer parallel health probe ──────────────────────────────────────── async function probeHealth(providers) { const cache = loadCache(); @@ -661,10 +699,36 @@ async function main() { process.stderr.write(`[preflight] Tiered ordering: ${output.primary_slots.length} primary (CLI) + ${output.backup_slots.length} backup (HTTP API)\n`); } + // ─── P1 — deep inference gate. L1/L2/L3 can pass a quota/auth-dead slot (L2 even + // treats 401/403 as reachable). Run the real deep_probe when --deep OR when the + // prelim panel is already degraded, budget permitting, and downgrade any slot that + // returns a FAST auth/quota signal (timeouts/spawn errors never downgrade — see + // deepProbeSlot/classifyDeepProbeResult). This runs off the happy path by default. + const prelimGate = computeQuorumGate(output.available_slots.length, maxSize, FORCE_QUORUM); + if (shouldRunDeepProbe({ deep: DEEP_PROBE, degraded: prelimGate.degraded, budgetMs: BUDGET_MS })) { + const results = await Promise.all( + output.available_slots.map(async (name) => ({ name, r: await deepProbeSlot(providerByName.get(name) || {}, BUDGET_MS) })) + ); + const downgraded = results.filter(({ r }) => r && r.ok === false); + if (downgraded.length > 0) { + const deadNames = new Set(downgraded.map(d => d.name)); + for (const { name, r } of downgraded) { + if (output.health[name]) { output.health[name].healthy = false; output.health[name].layer4 = r; } + output.unavailable_slots.push({ name, reason: `layer4: ${r.reason}` }); + } + output.available_slots = output.available_slots.filter(n => !deadNames.has(n)); + output.primary_slots = output.available_slots.filter(s => typeMap.get(s) !== 'http'); + output.backup_slots = output.available_slots.filter(s => typeMap.get(s) === 'http').concat(dedupedOut); + process.stderr.write(`[preflight] deep-probe downgraded ${downgraded.length} slot(s): ${downgraded.map(d => d.name).join(', ')}\n`); + } + output.deep_probe_ran = true; + } + // ─── P3 — authoritative degraded-panel gate (see computeQuorumGate). Machine // field so the block/waiver decision is deterministic + testable, not re-derived // by LLM-interpreted markdown. `available_count` = distinct healthy slots eligible - // to vote (deduped primaries + HTTP backups; demoted duplicates excluded). + // to vote (deduped primaries + HTTP backups; demoted duplicates excluded). Computed + // AFTER the deep-probe pass so the gate reflects the post-deep roster. Object.assign(output, computeQuorumGate(output.available_slots.length, maxSize, FORCE_QUORUM)); if (output.blocked) { process.stderr.write(`[preflight] ${output.gate_reason}\n`); @@ -746,13 +810,16 @@ function validateProviders(providers) { // that could take the whole system down, so it lands with a mock-CLI integration // harness in a follow-up. These two pure helpers encode the safe policy and are unit-tested. -// Run the deep probe when explicitly requested OR when the panel is degraded (so a -// reduced panel is verified with real inference before we trust it), but only if the -// remaining time budget covers at least one probe. +// Decide whether to run the (expensive, spawn-per-slot) deep probe. +// - Explicit --deep: run (subject to budget when one is set). +// - Degraded panel WITHOUT --deep: auto-enable ONLY when an explicit budget is set and +// covers a probe. The default cheap `--all --probe` liveness path passes no budget +// (budgetMs == null), so it must NOT pay deep-probe latency there — the real quorum +// dispatch passes --budget-ms, so it still auto-verifies a reduced panel with inference. function shouldRunDeepProbe({ deep, degraded, budgetMs, minBudgetMs = 45000 }) { - if (!deep && !degraded) return false; - if (budgetMs == null) return true; // no budget cap → allowed - return budgetMs >= minBudgetMs; + if (deep) return budgetMs == null ? true : budgetMs >= minBudgetMs; + if (!degraded) return false; + return budgetMs != null && budgetMs >= minBudgetMs; } // Classify a deep-probe result. CRITICAL: downgrade a slot ONLY on a FAST, explicit @@ -781,4 +848,4 @@ if (require.main === module) { main().catch(e => { console.error(e); process.exit(1); }); } -module.exports = { dedupBySlotIdentity, probeHealth, findProviders, computeQuorumGate, validateProviders, shouldRunDeepProbe, classifyDeepProbeResult }; +module.exports = { dedupBySlotIdentity, probeHealth, findProviders, computeQuorumGate, validateProviders, shouldRunDeepProbe, classifyDeepProbeResult, deepProbeSlot }; From ea22528791512f5705f230cbc391363877eff4db Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Wed, 1 Jul 2026 15:48:55 +0100 Subject: [PATCH 7/7] harden(quorum) assessment fixes: F2 skip-inactive, N2 guard, F1/F3/N1 docs + F1 test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-model adversarial assessment (claude-z-ai + claude-minimax) of the P1–P5 branch. Applying the agreed should-fixes; the rest are documented as accepted gaps. - F2 (should-fix): validateProviders now skips `active:false` slots — an intentionally disabled api-* HTTP slot legitimately omits baseUrl/apiKeyEnv and was emitting ERROR on every preflight (stderr flood). +2 tests (skip inactive; active-incomplete still errors). - N2 (harden): deepProbeSlot.finish() captures `child` locally and guards the kill, so a not-yet-assigned/exited child can never throw a ReferenceError out of the Promise.all fan-out. - F1 (documented + regression bar): a quota/auth-dead slot in a FULL panel (degraded=false, no --deep) is not deep-probed → dispatched, L3 catches it next round (self-heals). Added a test pinning this accepted behavior + the --deep escape hatch. Deep-probing every slot every run is too expensive to make the default. - F3 / N1 (documented): http slots bypass P1's CLI deep-probe (need an HTTP completion probe — out of scope); budgetMs is applied per-slot in the parallel fan-out. - F4 REFUTED, N5 REFUTED (validator requires BOTH cli+mainTool absent — mainTool-only is fine). Full preflight suite green (validate 9, deep 12, gate 7, deep-integration 7, probe 9). Co-Authored-By: Claude Opus 4.8 --- bin/quorum-preflight-deep.test.cjs | 14 +++++++++++++- bin/quorum-preflight-validate.test.cjs | 18 ++++++++++++++++++ bin/quorum-preflight.cjs | 23 ++++++++++++++++++++++- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/bin/quorum-preflight-deep.test.cjs b/bin/quorum-preflight-deep.test.cjs index 62838d4abf..4cc3dc7de8 100644 --- a/bin/quorum-preflight-deep.test.cjs +++ b/bin/quorum-preflight-deep.test.cjs @@ -9,7 +9,7 @@ const { describe, it } = require('node:test'); const assert = require('node:assert/strict'); -const { shouldRunDeepProbe, classifyDeepProbeResult } = require('./quorum-preflight.cjs'); +const { shouldRunDeepProbe, classifyDeepProbeResult, computeQuorumGate } = require('./quorum-preflight.cjs'); describe('shouldRunDeepProbe — when to run the deep gate', () => { it('runs when --deep is set', () => { @@ -30,6 +30,18 @@ describe('shouldRunDeepProbe — when to run the deep gate', () => { assert.equal(shouldRunDeepProbe({ deep: true, degraded: true, budgetMs: 1000, minBudgetMs: 45000 }), false); assert.equal(shouldRunDeepProbe({ deep: true, degraded: true, budgetMs: 60000, minBudgetMs: 45000 }), true); }); + + it('F1 (known gap, regression bar): a FULL panel is NOT deep-probed → a quota-dead slot slips', () => { + // Documents the accepted limitation so a future change that closes it trips this test. + // Full panel (3/3): degraded=false, so even with a sufficient budget and no --deep the + // deep probe does NOT run — a quota-dead slot is not pre-screened and the gate reports met. + assert.equal(shouldRunDeepProbe({ deep: false, degraded: false, budgetMs: 60000 }), false); + const gate = computeQuorumGate(3, 3, false); + assert.equal(gate.quorum_met, true); + assert.equal(gate.blocked, false); + // --deep DOES force a full-panel probe (the escape hatch for known-suspect slots). + assert.equal(shouldRunDeepProbe({ deep: true, degraded: false, budgetMs: 60000 }), true); + }); }); describe('classifyDeepProbeResult — downgrade only on fast auth/quota', () => { diff --git a/bin/quorum-preflight-validate.test.cjs b/bin/quorum-preflight-validate.test.cjs index 749d697bfb..2b4fd3aed2 100644 --- a/bin/quorum-preflight-validate.test.cjs +++ b/bin/quorum-preflight-validate.test.cjs @@ -59,4 +59,22 @@ describe('validateProviders — providers.json schema gate', () => { assert.equal(validateProviders(null).ok, true); assert.equal(validateProviders([]).ok, true); }); + + it('F2: SKIPS intentionally-disabled (active:false) slots — no ERROR flood for off slots', () => { + // An inactive http slot legitimately omits baseUrl/apiKeyEnv until turned on. Validating + // it would emit ERRORs on every preflight (stderr flood). It must be skipped entirely. + const r = validateProviders([ + { name: 'api-1', type: 'http', active: false }, // off + incomplete → skipped + { name: 'codex-1', type: 'subprocess', cli: '/x', deep_probe: {} }, + ]); + assert.equal(r.ok, true, 'inactive incomplete slot must not error'); + assert.equal(r.errors.length, 0); + assert.ok(!r.warnings.join().includes('api-1'), 'inactive slot must not warn either'); + }); + + it('F2: an ACTIVE incomplete http slot still errors (skip is scoped to active:false only)', () => { + const r = validateProviders([{ name: 'api-2', type: 'http' }]); // active undefined → validated + assert.equal(r.ok, false); + assert.match(r.errors.join(), /api-2: http slot missing baseUrl/); + }); }); diff --git a/bin/quorum-preflight.cjs b/bin/quorum-preflight.cjs index 904d8b6044..4de865bb6a 100644 --- a/bin/quorum-preflight.cjs +++ b/bin/quorum-preflight.cjs @@ -389,15 +389,26 @@ function deepProbeSlot(provider, budgetMs) { const target = resolveSpawnTarget(provider) || provider.cli; const tmpl = resolveArgsTemplate(provider); if (!target || !Array.isArray(tmpl)) { + // F3 (known gap): http slots have no CLI → resolveSpawnTarget returns '' → SKIP here. + // http backups are L1/L2-verified only (L2 pings /models); P1's CLI deep_probe cannot + // inference-verify them. Closing that hole needs a real HTTP chat-completion probe — + // a separate deep_probe contract, out of scope for this layer. resolve({ ok: true, classification: 'SKIP', reason: 'no spawn target / args_template' }); return; } const args = tmpl.map(a => (a === '{prompt}' ? probe.prompt : a)); + // N1: budgetMs is applied PER SLOT (probes fan out in parallel, so wall-clock ≈ the + // slowest single probe, not the sum). Callers sizing an aggregate deadline should + // account for N parallel spawns of up to this cap each. const timeoutMs = Math.min(probe.timeout_ms || 45000, budgetMs == null ? 2147483647 : budgetMs); let out = '', done = false, timedOut = false, child; const finish = (r) => { if (done) return; done = true; - try { process.kill(-child.pid, 'SIGKILL'); } catch (_) { try { child.kill('SIGKILL'); } catch (_) {} } + // N2 — capture child locally + guard: finish() is only wired after a successful + // spawn, but guard the kill so a not-yet-assigned/exited child can never throw a + // ReferenceError out of the probe (which would reject the Promise.all fan-out). + const c = child; + if (c && c.pid) { try { process.kill(-c.pid, 'SIGKILL'); } catch (_) { try { c.kill('SIGKILL'); } catch (_) {} } } resolve(r); }; try { @@ -704,6 +715,12 @@ async function main() { // prelim panel is already degraded, budget permitting, and downgrade any slot that // returns a FAST auth/quota signal (timeouts/spawn errors never downgrade — see // deepProbeSlot/classifyDeepProbeResult). This runs off the happy path by default. + // F1 (known limitation): a quota/auth-dead slot in a FULL panel (available >= maxSize, + // so degraded=false) with no --deep is NOT deep-probed — it dispatches and fails at + // the real review, and L3 failure-history catches it the NEXT round (self-heals, one + // wasted round). Deep-probing every slot every run is expensive, so this is accepted; + // pass --deep (or run when the panel is already degraded) to pre-empt it. NOTE this + // also means --force-quorum on a full panel won't pre-screen a known-suspect slot. const prelimGate = computeQuorumGate(output.available_slots.length, maxSize, FORCE_QUORUM); if (shouldRunDeepProbe({ deep: DEEP_PROBE, degraded: prelimGate.degraded, budgetMs: BUDGET_MS })) { const results = await Promise.all( @@ -785,6 +802,10 @@ function validateProviders(providers) { const seen = new Set(); for (const p of list) { if (!p || typeof p !== 'object' || !p.name) { errors.push('provider entry missing/!object name'); continue; } + // F2 — skip intentionally-disabled slots: an `active:false` provider (e.g. the api-* + // HTTP slots) may legitimately omit baseUrl/apiKeyEnv until it's turned on — validating + // it would flood stderr with ERRORs on every preflight. Only gate slots that can run. + if (p.active === false) continue; if (seen.has(p.name)) errors.push(`duplicate provider name: ${p.name}`); seen.add(p.name); const type = p.type;