Skip to content

Commit d325ce8

Browse files
A8Cl233395A8Cl233395
committed
fix: send images as native image parts instead of inlined base64 text
Convert OpenAI image_url parts into native {type:"image"} parts for /alpha/generate so vision models receive real image input instead of a text part full of base64. Image parts are built only from base64 data URIs and always carry the CLI's mimeType field; remote URLs and non-base64 data URIs fall back to the short [image_url: ...] / [image: ...] text placeholders, so the bridge never emits an image part shape the CommandCode CLI cannot produce. Tool-result images are forwarded as a following user message because the upstream ModelMessage schema rejects image parts inside tool-result output.
1 parent 7c27016 commit d325ce8

3 files changed

Lines changed: 267 additions & 2 deletions

File tree

‎src/converter.ts‎

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { cwd as processCwd } from "node:process";
44
import type {
55
CommandCodeContentPart,
66
CommandCodeGenerateBody,
7+
CommandCodeImagePart,
78
CommandCodeMessage,
89
CommandCodeTool,
910
CommandCodeToolResultPart,
@@ -37,18 +38,76 @@ function imageUrlToText(value: unknown): string {
3738
return "";
3839
}
3940

41+
function imageMediaType(dataUri: string): string | undefined {
42+
const match = /^data:([^;,]+);base64,/.exec(dataUri);
43+
return match?.[1];
44+
}
45+
46+
function imagePartFromUrl(value: unknown): CommandCodeImagePart | undefined {
47+
const url = imageUrlToText(value);
48+
const mimeType = url ? imageMediaType(url) : undefined;
49+
if (!mimeType) return undefined;
50+
return { type: "image", image: url, mimeType };
51+
}
52+
53+
function imagePlaceholder(value: unknown): string {
54+
const url = imageUrlToText(value);
55+
if (!url) return "[image_url]";
56+
if (url.startsWith("data:")) {
57+
const mediaType = /^data:([^;,]+)/.exec(url)?.[1];
58+
return `[image: ${mediaType ?? "image"}]`;
59+
}
60+
return `[image_url: ${url.slice(0, 256)}]`;
61+
}
62+
4063
export function flattenOpenAIContent(content: OpenAIMessageContent | undefined): string {
4164
if (content === undefined || content === null) return "";
4265
if (typeof content === "string") return content;
4366
return content
4467
.map((part) => {
4568
if (isTextPart(part)) return part.text;
46-
if (part.type === "image_url") return `[image_url: ${imageUrlToText(part.image_url)}]`;
69+
if (part.type === "image_url") return imagePlaceholder(part.image_url);
4770
return "";
4871
})
4972
.join("");
5073
}
5174

75+
function toolResultImages(content: OpenAIMessageContent | undefined): CommandCodeContentPart[] {
76+
if (content === undefined || content === null || typeof content === "string") return [];
77+
const images: CommandCodeContentPart[] = [];
78+
for (const part of content) {
79+
if (part.type !== "image_url") continue;
80+
const imagePart = imagePartFromUrl(part.image_url);
81+
if (imagePart) images.push(imagePart);
82+
}
83+
return images;
84+
}
85+
86+
function convertUserContent(
87+
content: OpenAIMessageContent | undefined,
88+
prefix: string,
89+
): CommandCodeContentPart[] {
90+
const parts: CommandCodeContentPart[] = [];
91+
if (prefix.length > 0) parts.push({ type: "text", text: prefix });
92+
if (content === undefined || content === null) return parts;
93+
if (typeof content === "string") {
94+
if (content.length > 0) parts.push({ type: "text", text: content });
95+
return parts;
96+
}
97+
for (const part of content) {
98+
if (isTextPart(part)) {
99+
if (part.text.length > 0) parts.push({ type: "text", text: part.text });
100+
continue;
101+
}
102+
if (part.type === "image_url") {
103+
const imagePart = imagePartFromUrl(part.image_url);
104+
if (imagePart) parts.push(imagePart);
105+
else parts.push({ type: "text", text: imagePlaceholder(part.image_url) });
106+
}
107+
}
108+
return parts;
109+
}
110+
52111
function asTextContent(text: string): OpenAITextContentPart[] {
53112
return [{ type: "text", text }];
54113
}
@@ -130,13 +189,18 @@ function convertMessages(messages: OpenAIChatMessage[]): CommandCodeMessage[] {
130189
} else {
131190
converted.push({ role: "tool", content: [part] });
132191
}
192+
const images = toolResultImages(message.content);
193+
if (images.length > 0) {
194+
converted.push({ role: "user", content: images });
195+
}
133196
continue;
134197
}
135198

136199
const userPrefix = message.name ? `name: ${message.name}\n` : "";
200+
const userParts = convertUserContent(message.content, userPrefix);
137201
converted.push({
138202
role: "user",
139-
content: asTextContent(`${userPrefix}${flattenOpenAIContent(message.content)}`),
203+
content: userParts.length > 0 ? userParts : asTextContent(""),
140204
});
141205
}
142206

‎src/types.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,15 @@ export interface CommandCodeToolResultPart {
9595
output: { type: "text" | "error-text"; value: string };
9696
}
9797

98+
export interface CommandCodeImagePart {
99+
type: "image";
100+
image: string;
101+
mimeType: string;
102+
}
103+
98104
export type CommandCodeContentPart =
99105
| OpenAITextContentPart
106+
| CommandCodeImagePart
100107
| CommandCodeToolCallPart
101108
| CommandCodeToolResultPart;
102109

‎tests/converter.test.ts‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,200 @@ describe("OpenAI to CommandCode conversion", () => {
293293
]);
294294
});
295295

296+
it("converts base64 image_url parts into native image parts with mimeType", () => {
297+
const body = buildCommandCodeGenerateBody({
298+
request: {
299+
model: "deepseek/deepseek-v4-flash-vision-exp",
300+
messages: [
301+
{
302+
role: "user",
303+
content: [
304+
{ type: "text", text: "What is in this image?" },
305+
{ type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } },
306+
],
307+
},
308+
],
309+
},
310+
upstreamModel: "deepseek/deepseek-v4-flash-vision-exp",
311+
});
312+
313+
expect(body.params.messages).toEqual([
314+
{
315+
role: "user",
316+
content: [
317+
{ type: "text", text: "What is in this image?" },
318+
{ type: "image", image: "data:image/png;base64,AAAA", mimeType: "image/png" },
319+
],
320+
},
321+
]);
322+
});
323+
324+
it("flattens image parts to short placeholders instead of inlining base64", () => {
325+
expect(
326+
flattenOpenAIContent([
327+
{ type: "text", text: "look" },
328+
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,AAAA" } },
329+
]),
330+
).toBe("look[image: image/jpeg]");
331+
expect(
332+
flattenOpenAIContent([{ type: "image_url", image_url: "https://example.com/cat.png" }]),
333+
).toBe("[image_url: https://example.com/cat.png]");
334+
});
335+
336+
it("keeps remote image URLs as text placeholders instead of image parts", () => {
337+
const body = buildCommandCodeGenerateBody({
338+
request: {
339+
model: "deepseek/deepseek-v4-flash-vision-exp",
340+
messages: [
341+
{
342+
role: "user",
343+
content: [
344+
{ type: "text", text: "What is this?" },
345+
{ type: "image_url", image_url: { url: "https://example.com/cat.png" } },
346+
],
347+
},
348+
],
349+
},
350+
upstreamModel: "deepseek/deepseek-v4-flash-vision-exp",
351+
});
352+
353+
expect(body.params.messages).toEqual([
354+
{
355+
role: "user",
356+
content: [
357+
{ type: "text", text: "What is this?" },
358+
{ type: "text", text: "[image_url: https://example.com/cat.png]" },
359+
],
360+
},
361+
]);
362+
});
363+
364+
it("falls back to text for data URIs that are not base64 encoded", () => {
365+
const body = buildCommandCodeGenerateBody({
366+
request: {
367+
model: "deepseek/deepseek-v4-flash-vision-exp",
368+
messages: [
369+
{
370+
role: "user",
371+
content: [{ type: "image_url", image_url: "data:image/svg+xml,%3Csvg%3E" }],
372+
},
373+
],
374+
},
375+
upstreamModel: "deepseek/deepseek-v4-flash-vision-exp",
376+
});
377+
378+
expect(body.params.messages).toEqual([
379+
{
380+
role: "user",
381+
content: [{ type: "text", text: "[image: image/svg+xml]" }],
382+
},
383+
]);
384+
});
385+
386+
it("forwards tool-result images as a following user image message", () => {
387+
const body = buildCommandCodeGenerateBody({
388+
request: {
389+
model: "deepseek/deepseek-v4-flash-vision-exp",
390+
messages: [
391+
{ role: "user", content: "Inspect the screenshot." },
392+
{
393+
role: "assistant",
394+
content: null,
395+
tool_calls: [
396+
{
397+
id: "call_shot",
398+
type: "function",
399+
function: { name: "read_image", arguments: "{}" },
400+
},
401+
],
402+
},
403+
{
404+
role: "tool",
405+
tool_call_id: "call_shot",
406+
content: [
407+
{ type: "text", text: "Screenshot captured." },
408+
{ type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } },
409+
],
410+
},
411+
],
412+
},
413+
upstreamModel: "deepseek/deepseek-v4-flash-vision-exp",
414+
});
415+
416+
expect(body.params.messages.map((message) => message.role)).toEqual([
417+
"user",
418+
"assistant",
419+
"tool",
420+
"user",
421+
]);
422+
expect(body.params.messages[2]).toEqual({
423+
role: "tool",
424+
content: [
425+
{
426+
type: "tool-result",
427+
toolCallId: "call_shot",
428+
toolName: "read_image",
429+
output: { type: "text", value: "Screenshot captured.[image: image/png]" },
430+
},
431+
],
432+
});
433+
expect(body.params.messages[3]).toEqual({
434+
role: "user",
435+
content: [{ type: "image", image: "data:image/png;base64,AAAA", mimeType: "image/png" }],
436+
});
437+
});
438+
439+
it("keeps remote tool-result image URLs in the tool text only", () => {
440+
const body = buildCommandCodeGenerateBody({
441+
request: {
442+
model: "deepseek/deepseek-v4-flash-vision-exp",
443+
messages: [
444+
{ role: "user", content: "Inspect the screenshot." },
445+
{
446+
role: "assistant",
447+
content: null,
448+
tool_calls: [
449+
{
450+
id: "call_shot",
451+
type: "function",
452+
function: { name: "read_image", arguments: "{}" },
453+
},
454+
],
455+
},
456+
{
457+
role: "tool",
458+
tool_call_id: "call_shot",
459+
content: [
460+
{ type: "text", text: "Screenshot captured." },
461+
{ type: "image_url", image_url: { url: "https://example.com/shot.png" } },
462+
],
463+
},
464+
],
465+
},
466+
upstreamModel: "deepseek/deepseek-v4-flash-vision-exp",
467+
});
468+
469+
expect(body.params.messages.map((message) => message.role)).toEqual([
470+
"user",
471+
"assistant",
472+
"tool",
473+
]);
474+
expect(body.params.messages[2]).toEqual({
475+
role: "tool",
476+
content: [
477+
{
478+
type: "tool-result",
479+
toolCallId: "call_shot",
480+
toolName: "read_image",
481+
output: {
482+
type: "text",
483+
value: "Screenshot captured.[image_url: https://example.com/shot.png]",
484+
},
485+
},
486+
],
487+
});
488+
});
489+
296490
it("injects JSON-only guidance for OpenAI response_format", () => {
297491
const body = buildCommandCodeGenerateBody({
298492
request: {

0 commit comments

Comments
 (0)