diff --git a/src/bridge-pool.ts b/src/bridge-pool.ts index 9259084..9b3cf0d 100644 --- a/src/bridge-pool.ts +++ b/src/bridge-pool.ts @@ -19,6 +19,10 @@ export type ParkedBridge = { pendingTools: Map; /** SDK assistant messages whose usage was already reported to OpenCode. */ seenAssistantUsageIds: Set; + /** Model the user was last told is answering (init model, then fallback target). */ + announcedModel?: string | null; + /** Model OpenCode asked for on this turn. */ + requestedModel?: string | null; createdAt: number; /** Continues consuming the SDK stream after tools resolve. */ continueStream?: () => AsyncGenerator; diff --git a/src/proxy.ts b/src/proxy.ts index 8aae174..78734d2 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -937,10 +937,22 @@ async function collectTurnResponse( content += `\n\n[claude-code error] ${text}`; }; + const modelNotices: ModelNotice[] = []; + try { + bridge.requestedModel = model || null; + for await (const event of events) { - const mapped = mapSdkEvent(event); - if (mapped.kind === "park") { + const mapped = mapSdkEvent(event, bridge); + if ( + (mapped.kind === "usage-delta" || mapped.kind === "ignore") && + mapped.note + ) { + modelNotices.push({ tag: mapped.noteTag ?? "answer", text: mapped.note }); + } + if (mapped.kind === "notice") { + modelNotices.push({ tag: mapped.tag, text: mapped.text }); + } else if (mapped.kind === "park") { toolCalls.push(...mapped.tools); sawContent = true; } else if (mapped.kind === "text") { @@ -972,6 +984,17 @@ async function collectTurnResponse( noteError(message); } + // One model note per leg, after the text (never mid-stream: it could land + // inside an open code fence). + { + const channel = noticeChannel(); + const block = renderModelNoticeBlock(modelNotices, channel); + if (block) { + if (channel === "content") content += block; + else if (!suppressReasoning) reasoning += block; + } + } + const usage = resolveTurnUsage(turnUsage, resultUsage); // Buffered responses have not committed HTTP headers yet. Even if an agent @@ -1292,9 +1315,26 @@ function streamOpenAIResponse( }); }; + const modelNotices: ModelNotice[] = []; + try { + bridge.requestedModel = model || null; + for await (const event of events) { - const mapped = mapSdkEvent(event); + const mapped = mapSdkEvent(event, bridge); + if ( + (mapped.kind === "usage-delta" || mapped.kind === "ignore") && + mapped.note + ) { + modelNotices.push({ + tag: mapped.noteTag ?? "answer", + text: mapped.note, + }); + } + if (mapped.kind === "notice") { + modelNotices.push({ tag: mapped.tag, text: mapped.text }); + continue; + } if (mapped.kind === "park") { finishReason = "tool_calls"; for (let i = 0; i < mapped.tools.length; i++) { @@ -1401,6 +1441,30 @@ function streamOpenAIResponse( finishReason = "stop"; } + // One model note per leg, after the text (see collectTurnResponse). + if (!streamClosed) { + const channel = noticeChannel(); + const block = renderModelNoticeBlock(modelNotices, channel); + if (block && (channel === "content" || !suppressReasoning)) { + send({ + id: completionId, + object: "chat.completion.chunk", + created, + model, + choices: [ + { + index: 0, + delta: + channel === "content" + ? { content: block } + : { reasoning_content: block }, + finish_reason: null, + }, + ], + }); + } + } + const usage = resolveTurnUsage(turnUsage, resultUsage); if (!streamClosed) { send({ @@ -1436,14 +1500,49 @@ function streamOpenAIResponse( return new Response(readable, { headers: SSE_HEADERS }); } +/** Model-note kinds, most authoritative first. */ +type ModelNoticeTag = "switch" | "refusal" | "session" | "answer"; + type MappedEvent = | { kind: "text"; text: string } | { kind: "reasoning"; text: string } + | { kind: "notice"; text: string; tag: ModelNoticeTag } | { kind: "park"; tools: ParkedToolCall[] } | { kind: "usage"; usage: OpenAIUsage } - | { kind: "usage-delta"; usage: OpenAIUsage; messageId: string | null } + | { + kind: "usage-delta"; + usage: OpenAIUsage; + messageId: string | null; + note?: string; + noteTag?: ModelNoticeTag; + } | { kind: "error"; text: string; usage?: OpenAIUsage | null } - | { kind: "ignore" }; + | { kind: "ignore"; note?: string; noteTag?: ModelNoticeTag }; + +type ModelNotice = { tag: ModelNoticeTag; text: string }; + +const MODEL_NOTICE_PRIORITY: ModelNoticeTag[] = [ + "switch", + "refusal", + "session", + "answer", +]; + +/** + * Pick the one note a turn shows: highest-priority tag, first occurrence. + * A fallback event beats the id-derived notes for the same switch. + */ +export function renderModelNoticeBlock( + notices: ModelNotice[], + channel: "content" | "reasoning", +): string { + for (const tag of MODEL_NOTICE_PRIORITY) { + const hit = notices.find((n) => n.tag === tag); + if (!hit) continue; + return renderModelNotice(hit.text, channel); + } + return ""; +} /** Text carried by Claude's synthetic assistant API-error message. */ function assistantErrorText(event: Record): string | null { @@ -1482,6 +1581,161 @@ function forgetDeadSession(conversationKey: string, errorText: string): void { clearForeignSessionId(conversationKey); } +/** + * Where model notes go: `content` (default, a Markdown callout in the + * answer) or `reasoning` (the thinking trace, like compact/rate-limit notes). + */ +export function noticeChannel(): "content" | "reasoning" { + const value = (process.env.OPENCODE_CLAUDE_MODEL_NOTES ?? "") + .trim() + .toLowerCase(); + return value === "reasoning" ? "reasoning" : "content"; +} + +/** `[tag] headline\nmore` → blockquote callout for content; as-is for reasoning. */ +export function renderModelNotice( + text: string, + channel: "content" | "reasoning", +): string { + if (channel === "reasoning") return text; + const lines = text + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + if (lines.length === 0) return ""; + const [head, ...rest] = lines; + const headline = head.replace(/^\[([^\]]+)\]\s*/, "⚠️ **$1:** "); + const body = rest.map((line) => `_${line}_`); + return `\n\n> ${[headline, ...body].join("\n> ")}\n\n`; +} + +/** First non-empty string among the keys (SDK is snake_case, CLI transcripts camelCase). */ +function stringField( + e: Record, + ...keys: string[] +): string | null { + for (const key of keys) { + const value = e[key]; + if (typeof value === "string" && value.trim()) return value.trim(); + } + return null; +} + +/** + * `system/model_refusal_fallback`: safety classifiers refused the request on + * the selected model and the CLI re-ran it on a fallback (e.g. Fable 5 → + * Opus 4.8). Scope `session` (or absent) means later turns stay on the + * fallback too. + */ +export function formatModelFallbackNote(e: Record): string { + const from = stringField(e, "original_model", "originalModel") ?? "unknown"; + const to = stringField(e, "fallback_model", "fallbackModel") ?? "unknown"; + const category = stringField(e, "api_refusal_category", "apiRefusalCategory"); + const explanation = stringField( + e, + "api_refusal_explanation", + "apiRefusalExplanation", + ); + const direction = stringField(e, "direction"); + log.warn("[opencode-claude] model fallback", { + from, + to, + category, + scope: e.scope ?? "session", + direction, + explanation, + }); + // Models, category, and only the unusual flags; the CLI banner is not repeated. + const parts = [ + `${from} → ${to}`, + ...(category ? [`safeguards [${category}]`] : []), + ...(e.scope === "local" ? ["this response only"] : []), + ...(direction && direction !== "retry" ? [direction] : []), + ]; + return ( + `\n[model switched] ${parts.join(" · ")}\n` + + (explanation ? `Reason given by the API: ${explanation}\n` : "") + ); +} + +/** + * Compare model ids ignoring the `claude-` prefix and a date suffix. A bare + * alias (`opus`, `fable`) matches any of its versions, so a note is only + * raised when the ids are unambiguously different. + */ +export function modelsMatch(a: string, b: string): boolean { + const norm = (id: string) => + id + .trim() + .toLowerCase() + .replace(/^claude-/, "") + .replace(/-\d{8}$/, "") + .replace(/-latest$/, ""); + const x = norm(a); + const y = norm(b); + if (!x || !y || x === y) return true; + const bareAlias = (id: string) => !/\d/.test(id); + if (bareAlias(x) && y.startsWith(`${x}-`)) return true; + if (bareAlias(y) && x.startsWith(`${y}-`)) return true; + return false; +} + +/** An assistant message came from a model other than the one last announced. */ +export function formatModelMismatchNote( + actual: string, + selected: string, +): string { + log.warn("[opencode-claude] answer came from a different model", { + actual, + selected, + }); + return `\n[model mismatch] answered by ${actual}, you selected ${selected}\n`; +} + +/** `init.model` differs from the user's pick: the session is latched on a fallback. */ +export function formatSessionModelNote( + sessionModel: string, + selected: string, +): string { + log.warn("[opencode-claude] session is on a different model than selected", { + sessionModel, + selected, + }); + return `\n[model mismatch] answer starts on ${sessionModel}, you selected ${selected}\n`; +} + +/** `system/notification`: the CLI's own notices (deprecations, limits). Low = tips, dropped. */ +export function formatNotificationNote( + e: Record, +): string | null { + const text = stringField(e, "text"); + if (!text) return null; + const priority = stringField(e, "priority") ?? "medium"; + if (priority === "low") return null; + log.warn("[opencode-claude] CLI notification", { + key: stringField(e, "key"), + priority, + text, + }); + return `\n[notice${priority === "medium" ? "" : `:${priority}`}] ${text}\n`; +} + +/** `model_refusal_no_fallback`: refused, no retry; the turn ends with the CLI's error. */ +export function formatModelRefusalNote(e: Record): string { + const from = stringField(e, "original_model", "originalModel") ?? "unknown"; + const category = stringField(e, "api_refusal_category", "apiRefusalCategory"); + log.warn("[opencode-claude] model refusal without fallback", { + from, + category, + }); + const parts = [ + from, + ...(category ? [`safeguards [${category}]`] : []), + "no fallback, turn ended with an error", + ]; + return `\n[model refused] ${parts.join(" · ")}\n`; +} + /** * Map Claude Agent SDK events to OpenAI-style deltas. * @@ -1489,7 +1743,7 @@ function forgetDeadSession(conversationKey: string, errorText: string): void { * `assistant` message payloads repeat the same content after partials and * would double-print if both were forwarded. */ -function mapSdkEvent(event: unknown): MappedEvent { +function mapSdkEvent(event: unknown, bridge?: ParkedBridge): MappedEvent { if (!event || typeof event !== "object") return { kind: "ignore" }; const e = event as Record; @@ -1497,6 +1751,25 @@ function mapSdkEvent(event: unknown): MappedEvent { return { kind: "park", tools: e.tools as ParkedToolCall[] }; } + // init comes on every turn. After a sticky fallback the CLI may report the + // fallback as the session model, which is itself worth a note. + if (e.type === "system" && e.subtype === "init") { + if (bridge) { + const requested = bridge.requestedModel ?? null; + const sessionModel = stringField(e, "model"); + if (requested && sessionModel && !modelsMatch(sessionModel, requested)) { + bridge.announcedModel = sessionModel; + return { + kind: "ignore", + note: formatSessionModelNote(sessionModel, requested), + noteTag: "session", + }; + } + bridge.announcedModel = sessionModel ?? requested; + } + return { kind: "ignore" }; + } + // Structured subscription limit telemetry from the Agent SDK — record for // the /v1/rate-limit counter; surface a note only on meaningful changes. // The note decision must use THIS event's own payload (fresh), never @@ -1511,6 +1784,23 @@ function mapSdkEvent(event: unknown): MappedEvent { return note ? { kind: "reasoning", text: note } : { kind: "ignore" }; } + // Safety-classifier fallback: the CLI swaps the model silently, OpenCode + // keeps showing the requested one. + if (e.type === "system" && e.subtype === "model_refusal_fallback") { + const to = stringField(e, "fallback_model", "fallbackModel"); + if (bridge && to) bridge.announcedModel = to; + return { kind: "notice", text: formatModelFallbackNote(e), tag: "switch" }; + } + + if (e.type === "system" && e.subtype === "model_refusal_no_fallback") { + return { kind: "notice", text: formatModelRefusalNote(e), tag: "refusal" }; + } + + if (e.type === "system" && e.subtype === "notification") { + const text = formatNotificationNote(e); + return text ? { kind: "reasoning", text } : { kind: "ignore" }; + } + // Auto-compact boundary — surface as a short reasoning note for the UI. if (e.type === "system" && e.subtype === "compact_boundary") { return { @@ -1575,14 +1865,35 @@ function mapSdkEvent(event: unknown): MappedEvent { } return { kind: "error", text: note, usage }; } + // Catch-all: `message.model` is what actually answered. Announce a change + // once (the fallback note already set announcedModel to its target). + let note: string | undefined; + const actualModel = + message && typeof message.model === "string" && message.model.trim() + ? message.model.trim() + : null; + if (bridge && actualModel) { + const announced = bridge.announcedModel; + if (announced && !modelsMatch(actualModel, announced)) { + // Name the user's pick, not a previously announced fallback target. + const requested = bridge.requestedModel; + const selected = + requested && !modelsMatch(requested, announced) ? requested : announced; + note = formatModelMismatchNote(actualModel, selected); + } + bridge.announcedModel = actualModel; + } if (usage) { return { kind: "usage-delta", usage, messageId: typeof message?.id === "string" ? message.id : null, + ...(note ? { note, noteTag: "answer" as const } : {}), }; } - return { kind: "ignore" }; + return note + ? { kind: "ignore", note, noteTag: "answer" } + : { kind: "ignore" }; } if (e.type === "result") { diff --git a/test/smoke.ts b/test/smoke.ts index 048b4cb..76aabd3 100644 --- a/test/smoke.ts +++ b/test/smoke.ts @@ -39,6 +39,15 @@ async function main() { getProxyPort, getClaudeProxyBaseUrl, PROXY_IDLE_TIMEOUT_SECONDS, + formatModelFallbackNote, + formatModelRefusalNote, + formatModelMismatchNote, + formatNotificationNote, + formatSessionModelNote, + modelsMatch, + noticeChannel, + renderModelNotice, + renderModelNoticeBlock, } = await import("../src/proxy.ts"); // Auth env stripping @@ -1133,6 +1142,221 @@ async function main() { assert.match(sysPrompt.append ?? "", /mcp__opencode__todowrite/); assert.match(sysPrompt.append ?? "", /[Bb]atch independent tool calls/); + // Model notes: formatters. + { + const sessionNote = formatModelFallbackNote({ + original_model: "claude-fable-5", + fallback_model: "claude-opus-4-8", + api_refusal_category: "cyber", + direction: "sticky", + scope: "session", + }); + assert.equal( + sessionNote, + "\n[model switched] claude-fable-5 → claude-opus-4-8 · safeguards [cyber] · sticky\n", + ); + assert.match( + formatModelFallbackNote({ original_model: "a", fallback_model: "b", scope: "local" }), + /this response only/, + ); + // Real wire event (CLI 2.1.197–2.1.237): snake_case, no scope, banner + // in `content`. Only the models and category are shown. + const realNote = formatModelFallbackNote({ + type: "system", + subtype: "model_refusal_fallback", + trigger: "refusal", + direction: "retry", + original_model: "claude-fable-5", + fallback_model: "claude-opus-4-8", + api_refusal_category: "cyber", + api_refusal_explanation: null, + content: "Fable 5's safeguards flagged this message. Switched to Opus 4.8.", + }); + assert.equal(realNote, "\n[model switched] claude-fable-5 → claude-opus-4-8 · safeguards [cyber]\n"); + // camelCase (CLI transcript shape) is accepted too. + assert.match( + formatModelFallbackNote({ originalModel: "claude-fable-5", fallbackModel: "claude-opus-5" }), + /claude-fable-5 → claude-opus-5/, + ); + const explained = formatModelFallbackNote({ + original_model: "claude-fable-5", + fallback_model: "claude-opus-4-8", + api_refusal_explanation: "Working exploit code against a named target.", + }); + assert.match(explained, /\nReason given by the API: Working exploit code/); + + assert.equal( + formatModelRefusalNote({ original_model: "claude-fable-5", api_refusal_category: "bio" }), + "\n[model refused] claude-fable-5 · safeguards [bio] · no fallback, turn ended with an error\n", + ); + assert.equal( + formatModelMismatchNote("claude-opus-4-8", "claude-fable-5"), + "\n[model mismatch] answered by claude-opus-4-8, you selected claude-fable-5\n", + ); + assert.equal( + formatSessionModelNote("claude-opus-4-8", "fable"), + "\n[model mismatch] answer starts on claude-opus-4-8, you selected fable\n", + ); + + // Model ids: prefix/date-insensitive; a bare alias matches its versions. + assert.equal(modelsMatch("claude-haiku-4-5-20251001", "claude-haiku-4-5"), true); + assert.equal(modelsMatch("fable", "claude-fable-5"), true); + + assert.equal(modelsMatch("opus", "claude-opus-4-8"), true); + assert.equal(modelsMatch("claude-fable-5", "claude-opus-4-8"), false); + assert.equal(modelsMatch("claude-opus-5", "claude-opus-4-8"), false); + + // CLI notifications: medium plain, high tagged, low and empty dropped. + assert.equal( + formatNotificationNote({ text: "Opus 4.7 is deprecated", priority: "medium" }), + "\n[notice] Opus 4.7 is deprecated\n", + ); + assert.equal( + formatNotificationNote({ text: "Session limit nearly reached", priority: "high" }), + "\n[notice:high] Session limit nearly reached\n", + ); + assert.equal(formatNotificationNote({ text: "Try /btw", priority: "low" }), null); + assert.equal(formatNotificationNote({ priority: "high" }), null); + + // Channel: content by default, reasoning via env. + const prevNotes = process.env.OPENCODE_CLAUDE_MODEL_NOTES; + delete process.env.OPENCODE_CLAUDE_MODEL_NOTES; + assert.equal(noticeChannel(), "content"); + process.env.OPENCODE_CLAUDE_MODEL_NOTES = "reasoning"; + assert.equal(noticeChannel(), "reasoning"); + if (prevNotes === undefined) delete process.env.OPENCODE_CLAUDE_MODEL_NOTES; + else process.env.OPENCODE_CLAUDE_MODEL_NOTES = prevNotes; + + // Rendering: blockquote callout for content, untouched for reasoning. + assert.equal( + renderModelNotice(realNote, "content"), + "\n\n> ⚠️ **model switched:** claude-fable-5 → claude-opus-4-8 · safeguards [cyber]\n\n", + ); + assert.match(renderModelNotice(explained, "content"), /\n> _Reason given by the API: /); + assert.equal(renderModelNotice(sessionNote, "reasoning"), sessionNote); + assert.equal(renderModelNotice("\n\n", "content"), ""); + + // One note per turn: switch > refusal > session > answer, first wins. + const answerNote = formatModelMismatchNote("claude-opus-4-8", "claude-fable-5"); + const block = renderModelNoticeBlock( + [ + { tag: "answer", text: answerNote }, + { tag: "session", text: formatSessionModelNote("S1", "fable") }, + { tag: "switch", text: sessionNote }, + { tag: "session", text: formatSessionModelNote("S2", "fable") }, + ], + "content", + ); + assert.match(block, /model switched:/); + assert.doesNotMatch(block, /S1|S2|answered by/); + assert.equal(renderModelNoticeBlock([], "content"), ""); + assert.equal( + renderModelNoticeBlock([{ tag: "answer", text: answerNote }], "reasoning"), + answerNote, + ); + } + + // Model notes through the proxy. + const mockEvents = (events: unknown[]) => + setClaudeQueryStarter(async () => ({ + stream: (async function* () { + for (const ev of events) yield ev; + })(), + interrupt: async () => {}, + close: () => {}, + getPid: () => null, + })); + const askFable = (session: string, stream: boolean) => + fetch(`http://127.0.0.1:${port}/v1/chat/completions`, { + method: "POST", + headers: { + "content-type": "application/json", + "x-opencode-claude-session": session, + "x-opencode-claude-directory": "/data/projects/infra", + }, + body: JSON.stringify({ + model: "fable", + stream, + messages: [{ role: "user", content: "anything" }], + }), + }); + const textDelta = (text: string) => ({ + type: "stream_event", + event: { type: "content_block_delta", delta: { type: "text_delta", text } }, + }); + const assistantFrom = (model: string, text: string) => ({ + type: "assistant", + message: { + id: `msg_${text}`, + model, + role: "assistant", + content: [{ type: "text", text }], + usage: { input_tokens: 5, output_tokens: 1 }, + }, + }); + const result = { + type: "result", + is_error: false, + total_cost_usd: 0.001, + usage: { input_tokens: 5, output_tokens: 1 }, + }; + + // Buffered: a fallback event becomes a callout after the answer text; + // the fallback model's own message does not add a second note; CLI + // notifications go to the reasoning trace (low priority stays silent). + mockEvents([ + { type: "system", subtype: "init", session_id: "s-fb", model: "claude-fable-5" }, + { + type: "system", + subtype: "model_refusal_fallback", + direction: "retry", + scope: "session", + original_model: "claude-fable-5", + fallback_model: "claude-opus-4-8", + api_refusal_category: "cyber", + content: "Switched to Opus", + }, + textDelta("FALLBACK_ANSWER"), + assistantFrom("claude-opus-4-8", "FALLBACK_ANSWER"), + { type: "system", subtype: "notification", text: "Session limit nearly reached", priority: "high" }, + { type: "system", subtype: "notification", text: "Try /btw", priority: "low" }, + result, + ]); + const fbRes = await askFable("smoke-mock-fallback", false); + assert.equal(fbRes.status, 200); + const fbJson = (await fbRes.json()) as { + choices?: Array<{ message?: Record }>; + }; + const fbMsg = fbJson.choices?.[0]?.message ?? {}; + const fbContent = String(fbMsg.content ?? ""); + const fbReasoning = String(fbMsg.reasoning_content ?? ""); + assert.match( + fbContent, + /^FALLBACK_ANSWER\n\n> ⚠️ \*\*model switched:\*\* claude-fable-5 → claude-opus-4-8 · safeguards \[cyber\]\n/, + ); + assert.doesNotMatch(fbContent + fbReasoning, /Switched to Opus|answered by|Try \/btw/); + assert.match(fbReasoning, /\[notice:high\] Session limit nearly reached/); + assert.doesNotMatch(fbContent, /Session limit/); + + // Streaming: a quiet switch (no fallback event) is caught from + // `assistant.message.model`, announced once, as a content chunk. + mockEvents([ + { type: "system", subtype: "init", session_id: "s-mm", model: "claude-fable-5" }, + assistantFrom("claude-opus-4-8", "STREAMED"), + textDelta("STREAMED"), + assistantFrom("claude-opus-4-8", "STREAMED2"), + result, + ]); + const mmStream = await askFable("smoke-mock-mismatch-stream", true); + assert.equal(mmStream.status, 200); + const mmSse = await mmStream.text(); + assert.match(mmSse, /STREAMED/); + assert.equal( + mmSse.match(/answered by claude-opus-4-8, you selected claude-fable-5/g)?.length ?? 0, + 1, + ); + assert.match(mmSse, /"content":"[^"]*answered by claude-opus-4-8/); + assert.doesNotMatch(mmSse, /reasoning_content[^\n]*answered by/); // Proxy + mock SDK: hard limit error BEFORE any content — the proxy // must answer with a truthful HTTP 429 (not a fake-200 error stream), // flip the store to limited, and fail fast on the next request.