From bd056d007400bedcbb37aa99b95642f52733bd73 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 11 Aug 2026 00:20:04 +0200 Subject: [PATCH] feat(models): add vision input capabilities --- CHANGELOG.md | 3 ++ README.md | 4 +- index.ts | 10 ++-- src/converters.ts | 59 +++++++++++++++++----- src/core.ts | 14 ++---- src/models.ts | 59 +++++++++++++++++++++- tests/test-models.ts | 15 +++++- tests/test-pure-functions.ts | 97 +++++++++++++++++++++++++++++------- tests/test-stream.ts | 60 ++++++++++++++++++++-- 9 files changed, 265 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d6ff0d..6bbe9b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Add model-specific image input capabilities from the `command-code@1.15.1` catalog and forward user and tool-result images using the current Command Code wire format. +- Update the Command Code client version header to `1.15.1`. + ## 0.5.0 - 2026-08-07 - Stop replaying completed assistant reasoning traces to Command Code while preserving visible text and completed tool calls in follow-up request history. diff --git a/README.md b/README.md index 3c8ff79..91b8085 100644 --- a/README.md +++ b/README.md @@ -132,9 +132,9 @@ The following environment variables are intended for tests, local mocks, and com ## Image input -This provider currently advertises and accepts **text input only**. The extension uses Command Code's legacy `/alpha/generate` protocol, while the public Provider API documentation describes image parts for its documented `/provider/v1` endpoints. The legacy request path has no documented image-part contract, and the model catalog fixture exposes model IDs and context lengths but no image capability or limit fields. +The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.15.1`; unknown models default to text-only until their upstream metadata is reviewed. -To avoid silently dropping or changing image data, the provider rejects image content in user messages and tool results before making a network request. It does not claim image capability or define image-size/count limits. This limitation can be revisited when Command Code documents image parts and limits for the protocol used here. +For vision-capable models, image blocks from user messages and tool results are forwarded in Command Code's current data-URL wire format. Text-only models reject image content before making a network request instead of silently dropping it. ## Pricing display diff --git a/index.ts b/index.ts index e5c5166..24a77b1 100644 --- a/index.ts +++ b/index.ts @@ -15,16 +15,12 @@ import { } from "@earendil-works/pi-coding-agent" import { join } from "node:path" -import { - COMMAND_CODE_CLI_VERSION, - COMMAND_CODE_INPUT_TYPES, - createStreamCommandCode, - DEFAULT_API_BASE, -} from "./src/core.ts" +import { COMMAND_CODE_CLI_VERSION, createStreamCommandCode, DEFAULT_API_BASE } from "./src/core.ts" import { calculateCommandCodeCost } from "./src/cost.ts" import { DEFAULT_MODELS_URL, getModelsTimeoutMs, + inputModalitiesForModel, loadCommandCodeModels, thinkingMetadataForModel, type CommandCodeModel, @@ -75,7 +71,7 @@ function createProviderModel(model: { name: model.name, reasoning: model.reasoning, ...(thinkingMetadataForModel(model.id) ?? {}), - input: COMMAND_CODE_INPUT_TYPES, + input: inputModalitiesForModel(model.id), cost: MODEL_COSTS[model.id] ?? ZERO_MODEL_COST, contextWindow: model.contextWindow, maxTokens: model.maxTokens, diff --git a/src/converters.ts b/src/converters.ts index d2bef47..4012fbb 100644 --- a/src/converters.ts +++ b/src/converters.ts @@ -55,26 +55,50 @@ function apiKeyFromCredentialRecord(value: unknown): string | undefined { return stringValue(value.key) ?? stringValue(value.access) } -function hasImageContent(value: unknown): boolean { - if (isRecord(value)) return value.type === "image" - return recordArray(value).some((part) => part.type === "image") +function imageParts(value: unknown): readonly Record[] { + if (isRecord(value)) return value.type === "image" ? [value] : [] + return recordArray(value).filter((part) => part.type === "image") } function imageContentError(role: string): Error { - return new Error( - `Command Code does not support image content in ${role}; refusing to send it to avoid lossy handling`, - ) + return new Error(`Selected Command Code model does not support image content in ${role}`) } export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void { for (const message of messages ?? []) { - if (hasImageContent(message.content)) { + if (imageParts(message.content).length > 0) { const role = message.role === "toolResult" ? "tool results" : `${message.role} messages` throw imageContentError(role) } } } +function imageToCommandCode(part: Record): Record { + const data = stringValue(part.data) + const mimeType = stringValue(part.mimeType) + if (!data || !mimeType) + throw new Error("Invalid image content: expected base64 data and mimeType") + + return { + type: "image", + image: `data:${mimeType};base64,${data}`, + mimeType, + } +} + +function userContentToCommandCode(content: unknown, allowImages: boolean): unknown { + if (typeof content === "string") return content + + return recordArray(content).flatMap((part) => { + if (part.type === "text") return [{ type: "text", text: stringValue(part.text) ?? "" }] + if (part.type === "image") { + if (!allowImages) throw imageContentError("user messages") + return [imageToCommandCode(part)] + } + return [] + }) +} + export function getApiKey( options: { env?: NodeJS.ProcessEnv @@ -115,8 +139,6 @@ export function getApiKey( } export function textContent(message: { content?: unknown }): string { - if (hasImageContent(message.content)) throw imageContentError("tool results") - return recordArray(message.content) .filter((part) => part.type === "text") .map((part) => stringValue(part.text) ?? "") @@ -157,8 +179,12 @@ function completeToolCallIds(messages?: readonly MessageLike[]): Set { return new Set([...callIds].filter((id) => resultIds.has(id))) } -export function messagesToCC(messages?: readonly MessageLike[]): unknown[] { - assertTextOnlyMessages(messages) +export function messagesToCC( + messages?: readonly MessageLike[], + options: { allowImages?: boolean } = {}, +): unknown[] { + const allowImages = options.allowImages ?? false + if (!allowImages) assertTextOnlyMessages(messages) const out: unknown[] = [] const pairedToolCallIds = completeToolCallIds(messages) @@ -167,7 +193,7 @@ export function messagesToCC(messages?: readonly MessageLike[]): unknown[] { if (message.role === "user") { out.push({ role: "user", - content: typeof message.content === "string" ? message.content : message.content, + content: userContentToCommandCode(message.content, allowImages), }) } else if (message.role === "assistant") { const parts: unknown[] = [] @@ -201,6 +227,15 @@ export function messagesToCC(messages?: readonly MessageLike[]): unknown[] { }, ], }) + + const images = imageParts(message.content) + if (images.length > 0) { + if (!allowImages) throw imageContentError("tool results") + out.push({ + role: "user", + content: images.map(imageToCommandCode), + }) + } } } return out diff --git a/src/core.ts b/src/core.ts index ef69049..f1898dd 100644 --- a/src/core.ts +++ b/src/core.ts @@ -8,6 +8,7 @@ import { randomUUID } from "node:crypto" import { commandCodeErrorMessage, redactCommandCodeErrorText } from "./overflow.ts" +import { modelSupportsImageInput } from "./models.ts" import { getApiKey, getEnvironmentInfo, @@ -42,13 +43,7 @@ export * from "./overflow.ts" export * from "./types.ts" export const DEFAULT_API_BASE = "https://api.commandcode.ai" -export const COMMAND_CODE_CLI_VERSION = "0.29.0" -/** - * The legacy /alpha/generate request path used by this provider has no - * documented image-part contract. Keep the advertised capability text-only - * until Command Code documents and tests image handling for this endpoint. - */ -export const COMMAND_CODE_INPUT_TYPES = ["text"] as const +export const COMMAND_CODE_CLI_VERSION = "1.15.1" const DEFAULT_GENERATE_MAX_TOKENS = 64_000 const DEFAULT_MAX_RETRIES = 0 @@ -472,7 +467,8 @@ export function createStreamCommandCode(deps: CoreDependencies) { const reasoningEffort = mappedReasoningEffort(model, options) const timeoutMs = options?.timeoutMs - assertTextOnlyMessages(context.messages) + const allowImages = modelSupportsImageInput(model.id) + if (!allowImages) assertTextOnlyMessages(context.messages) let body: unknown = { config: { @@ -491,7 +487,7 @@ export function createStreamCommandCode(deps: CoreDependencies) { skills: null, params: { model: model.id, - messages: messagesToCC(context.messages), + messages: messagesToCC(context.messages, { allowImages }), tools: toolsToJson(context.tools), system: systemPromptToText(context.systemPrompt), max_tokens: generateMaxTokens(model, options), diff --git a/src/models.ts b/src/models.ts index 9f4c375..cd470ef 100644 --- a/src/models.ts +++ b/src/models.ts @@ -7,6 +7,63 @@ export const DEFAULT_MODELS_TIMEOUT_MS = 10_000 const DEFAULT_MAX_OUTPUT_TOKENS = 65_536 const MODEL_CACHE_VERSION = 1 +export type CommandCodeInputType = "text" | "image" + +/** + * Model input modalities from the command-code@1.15.1 bundled catalog. + * Models omitted here remain text-only so newly discovered IDs never claim + * image support without upstream evidence. + */ +export const MODEL_INPUT_MODALITIES: Readonly> = { + "MiniMaxAI/MiniMax-M3": ["text", "image"], + "Qwen/Qwen3.6-Plus": ["text", "image"], + "Qwen/Qwen3.7-Flash": ["text", "image"], + "Qwen/Qwen3.7-Plus": ["text", "image"], + "Qwen/Qwen3.8-Max": ["text", "image"], + "claude-fable-5": ["text", "image"], + "claude-haiku-4-5-20251001": ["text", "image"], + "claude-opus-4-7": ["text", "image"], + "claude-opus-4-8": ["text", "image"], + "claude-opus-5": ["text", "image"], + "claude-sonnet-4-6": ["text", "image"], + "claude-sonnet-5": ["text", "image"], + "google/gemini-3.1-flash-lite": ["text", "image"], + "google/gemini-3.5-flash": ["text", "image"], + "google/gemini-3.5-flash-lite": ["text", "image"], + "google/gemini-3.6-flash": ["text", "image"], + "gpt-5.3-codex": ["text", "image"], + "gpt-5.4": ["text", "image"], + "gpt-5.4-mini": ["text", "image"], + "gpt-5.5": ["text", "image"], + "gpt-5.6-luna": ["text", "image"], + "gpt-5.6-sol": ["text", "image"], + "gpt-5.6-terra": ["text", "image"], + "meta/muse-spark-1.1": ["text", "image"], + "meta/muse-spark-1.2": ["text", "image"], + "meta/muse-spark-1.2-contributor": ["text", "image"], + "moonshotai/Kimi-K2.5": ["text", "image"], + "moonshotai/Kimi-K2.6": ["text", "image"], + "moonshotai/Kimi-K2.7-Code": ["text", "image"], + "moonshotai/Kimi-K2.7-Code-Highspeed": ["text", "image"], + "moonshotai/Kimi-K3": ["text", "image"], + "sakana/fugu-ultra": ["text", "image"], + "stepfun/Step-3.7-Flash": ["text", "image"], + "thinkingmachines/inkling": ["text", "image"], + "thinkingmachines/inkling-small": ["text", "image"], + "xai/grok-4.5": ["text", "image"], + "xiaomi/mimo-v2.5": ["text", "image"], +} + +const TEXT_INPUT_ONLY = ["text"] as const + +export function inputModalitiesForModel(modelId: string): readonly CommandCodeInputType[] { + return MODEL_INPUT_MODALITIES[modelId] ?? TEXT_INPUT_ONLY +} + +export function modelSupportsImageInput(modelId: string): boolean { + return inputModalitiesForModel(modelId).includes("image") +} + export type PiThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" type CommandCodeReasoningEffort = Exclude @@ -15,7 +72,7 @@ type CommandCodeReasoningEffort = Exclude * Per-model reasoning efforts supported by Command Code's generate endpoint. * * The Provider API does not expose reasoning metadata. This is an exact - * snapshot of `reasoningEfforts` from the command-code@1.14.1 model catalog + * snapshot of `reasoningEfforts` from the command-code@1.15.1 model catalog * (`packages/shared/src/model-catalog.ts`, also published in the generated * `dist/bundled/command-code-knowledge/reference/models.md`). Models omitted * here let Command Code choose their reasoning depth, matching the CLI. diff --git a/tests/test-models.ts b/tests/test-models.ts index e0a34ad..af96ebb 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -9,8 +9,11 @@ import { commandCodeModelsFromCache, DEFAULT_MODELS_TIMEOUT_MS, getModelsTimeoutMs, + inputModalitiesForModel, loadCommandCodeModels, MODEL_EFFORTS, + MODEL_INPUT_MODALITIES, + modelSupportsImageInput, thinkingLevelMapForEfforts, thinkingMetadataForModel, type CommandCodeModel, @@ -81,6 +84,16 @@ describe("commandCodeModelsFromApiResponse()", () => { assert.deepEqual(commandCodeModelsFromApiResponse(API_RESPONSE), EXPECTED_MODELS) }) + it("matches command-code@1.15.1 image input capabilities", () => { + assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-pro"), ["text"]) + assert.deepEqual(inputModalitiesForModel("unknown-new-model"), ["text"]) + assert.equal(modelSupportsImageInput("gpt-5.6-luna"), true) + assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-pro"), false) + assert.equal(Object.keys(MODEL_INPUT_MODALITIES).length, 37) + }) + it("marks only known reasoning models as reasoning-capable", () => { const models = commandCodeModelsFromApiResponse({ object: "list", @@ -94,7 +107,7 @@ describe("commandCodeModelsFromApiResponse()", () => { assert.equal(models[1]?.reasoning, false) }) - it("matches the exact command-code@1.14.1 reasoning effort catalog", () => { + it("matches the exact command-code@1.15.1 reasoning effort catalog", () => { assert.deepEqual(MODEL_EFFORTS, { "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], diff --git a/tests/test-pure-functions.ts b/tests/test-pure-functions.ts index 134c437..136025f 100644 --- a/tests/test-pure-functions.ts +++ b/tests/test-pure-functions.ts @@ -11,7 +11,6 @@ import { describe, it } from "node:test" import { assertTextOnlyMessages, - COMMAND_CODE_INPUT_TYPES, getApiKey, getEnvironmentInfo, mapFinishReason, @@ -118,11 +117,7 @@ describe("projectSlugFromPath()", () => { }) describe("text-only image handling", () => { - it("does not advertise image input capability", () => { - assert.deepEqual(COMMAND_CODE_INPUT_TYPES, ["text"]) - }) - - it("rejects image content instead of dropping it", () => { + it("rejects image content for models without image support", () => { assert.throws( () => assertTextOnlyMessages([ @@ -131,7 +126,7 @@ describe("text-only image handling", () => { content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], }, ]), - /does not support image content.*refusing to send it/i, + /does not support image content/i, ) assert.throws( () => @@ -142,7 +137,7 @@ describe("text-only image handling", () => { content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], }, ]), - /does not support image content.*refusing to send it/i, + /does not support image content/i, ) }) }) @@ -160,17 +155,16 @@ describe("textContent()", () => { ) }) - it("rejects mixed text and image content instead of dropping the image", () => { - assert.throws( - () => - textContent({ - content: [ - { type: "text", text: "hello" }, - { type: "image", data: "x", mimeType: "image/png" }, - { type: "text", text: "world" }, - ], - }), - /does not support image content.*refusing to send it/i, + it("extracts text while images are handled separately", () => { + assert.equal( + textContent({ + content: [ + { type: "text", text: "hello" }, + { type: "image", data: "x", mimeType: "image/png" }, + { type: "text", text: "world" }, + ], + }), + "hello\nworld", ) }) @@ -506,6 +500,71 @@ describe("messagesToCC()", () => { assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "hello\nworld") }) + it("serializes image inputs in the current Command Code wire format", () => { + assert.deepEqual( + messagesToCC( + [ + { + role: "user", + content: [ + { type: "text", text: "inspect this" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], + }, + ], + { allowImages: true }, + ), + [ + { + role: "user", + content: [ + { type: "text", text: "inspect this" }, + { + type: "image", + image: "data:image/png;base64,aGVsbG8=", + mimeType: "image/png", + }, + ], + }, + ], + ) + }) + + it("preserves tool-result images as a following user image message", () => { + const result = messagesToCC( + [ + { role: "user", content: "read image" }, + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [ + { type: "text", text: "image attached" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }, + ], + }, + ], + { allowImages: true }, + ) + + assert.equal(objectAt(result, ["2", "role"]), "tool") + assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "image attached") + assert.deepEqual(objectAt(result, ["3"]), { + role: "user", + content: [ + { + type: "image", + image: "data:image/jpeg;base64,aGVsbG8=", + mimeType: "image/jpeg", + }, + ], + }) + }) + it("drops previous assistant reasoning while preserving text and tool calls", () => { const result = messagesToCC([ { role: "user", content: "first question" }, diff --git a/tests/test-stream.ts b/tests/test-stream.ts index 75271dd..e30ac49 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -142,6 +142,59 @@ describe("streamCommandCode — successful streams", () => { assert.equal(calculatedUsages.length, 1) }) + it("sends images for vision-capable models", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode( + makeModel({ id: "gpt-5.6-luna" }), + makeContext({ + messages: [ + { + role: "user", + content: [ + { type: "text", text: "inspect" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], + }, + ], + }), + { apiKey: "mock-key" }, + ), + ) + + assert.equal( + objectAt(server.lastRequestBody(), ["params", "messages", "0", "content", "1", "image"]), + "data:image/png;base64,aGVsbG8=", + ) + }) + + it("rejects images before network access for text-only models", async () => { + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode( + makeModel({ id: "deepseek/deepseek-v4-pro" }), + makeContext({ + messages: [ + { + role: "user", + content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }], + }, + ], + }), + { apiKey: "mock-key" }, + ), + ) + + assert.equal(events.at(-1)?.type, "error") + assert.equal(server.requestCount(), 0) + }) + it("derives uncached input when noCacheTokens is missing", async () => { server.mockResponse({ type: "success", @@ -375,10 +428,7 @@ describe("streamCommandCode — request serialization", () => { const lastEvent = events.at(-1) assert.equal(lastEvent?.type, "error") if (lastEvent?.type === "error") { - assert.match( - lastEvent.error.errorMessage ?? "", - /does not support image content.*refusing to send it/i, - ) + assert.match(lastEvent.error.errorMessage ?? "", /does not support image content/i) } assert.equal(server.requestCount(), 0) }) @@ -438,7 +488,7 @@ describe("streamCommandCode — request serialization", () => { const headers = server.lastRequestHeaders() assert.equal(headers.authorization, "Bearer mock-key") - assert.equal(headers["x-command-code-version"], "0.29.0") + assert.equal(headers["x-command-code-version"], "1.15.1") assert.equal(headers["x-project-slug"], "repo") assert.equal(headers["x-taste-learning"], "true") assert.equal(headers["x-co-flag"], "false")