diff --git a/internal/openai/chat_prepare.go b/internal/openai/chat_prepare.go index 50afaa8..4e4478e 100644 --- a/internal/openai/chat_prepare.go +++ b/internal/openai/chat_prepare.go @@ -467,6 +467,16 @@ func (p *chatPreparer) prepareMessageContent(raw any, path string, allowImages b } out = append(out, map[string]any{"type": "image_url", "image_url": image}) p.recordUnknownObjectFields(part, map[string]bool{"type": true, "image_url": true}, partPath) + case "image": + if !allowImages { + return nil, fmt.Errorf("%s.type image is only valid for user messages", partPath) + } + image, err := p.prepareAnthropicImage(part, partPath) + if err != nil { + return nil, err + } + out = append(out, map[string]any{"type": "image_url", "image_url": image}) + p.recordUnknownObjectFields(part, map[string]bool{"type": true, "source": true}, partPath) default: return nil, fmt.Errorf("%s.type %q is not supported", partPath, kind) } @@ -474,6 +484,34 @@ func (p *chatPreparer) prepareMessageContent(raw any, path string, allowImages b return out, nil } +func (p *chatPreparer) prepareAnthropicImage(block map[string]any, path string) (map[string]any, error) { + source, ok := block["source"].(map[string]any) + if !ok { + return nil, fmt.Errorf("%s.source must be an object", path) + } + var url string + switch sourceType, _ := source["type"].(string); sourceType { + case "url": + value, ok := source["url"].(string) + if !ok || strings.TrimSpace(value) == "" { + return nil, fmt.Errorf("%s.source.url must be a non-empty string", path) + } + url = value + case "base64": + mediaType, _ := source["media_type"].(string) + data, _ := source["data"].(string) + if strings.TrimSpace(mediaType) == "" || strings.TrimSpace(data) == "" { + return nil, fmt.Errorf("%s.source media_type and data are required for base64 images", path) + } + url = "data:" + mediaType + ";base64," + data + default: + return nil, fmt.Errorf("%s.source.type %q is not supported", path, sourceType) + } + p.recordUnknownObjectFields(source, map[string]bool{"type": true, "url": true, "media_type": true, "data": true}, path+".source") + p.change(path, "rewritten", "Anthropic image block is normalized to OpenAI image_url content") + return map[string]any{"url": url}, nil +} + func (p *chatPreparer) prepareImageURL(raw any, path string) (map[string]any, error) { if value, ok := raw.(string); ok && strings.TrimSpace(value) != "" { p.change(path, "rewritten", "string image URL is normalized to an object") diff --git a/internal/openai/chat_prepare_test.go b/internal/openai/chat_prepare_test.go index d2501b4..2b60d3e 100644 --- a/internal/openai/chat_prepare_test.go +++ b/internal/openai/chat_prepare_test.go @@ -66,6 +66,7 @@ func TestPrepareChatNormalizesMessagesAndToolHistory(t *testing.T) { map[string]any{"role": "user", "content": []any{ map[string]any{"type": "text", "text": "inspect", "cache_control": true}, map[string]any{"type": "image_url", "image_url": map[string]any{"url": "data:image/png;base64,AA", "detail": "low", "vendor": true}}, + map[string]any{"type": "image", "source": map[string]any{"type": "base64", "media_type": "image/png", "data": "BB", "vendor": true}, "cache_control": true}, }}, map[string]any{"role": "assistant", "content": nil, "reasoning_content": "need tool", "tool_calls": []any{ map[string]any{"id": "call-1", "type": "function", "function": map[string]any{"name": "lookup", "arguments": `{"id":1}`, "extra": true}}, @@ -82,11 +83,15 @@ func TestPrepareChatNormalizesMessagesAndToolHistory(t *testing.T) { if parts[0].(map[string]any)["cache_control"] != nil || parts[1].(map[string]any)["image_url"].(map[string]any)["vendor"] != nil { t.Fatalf("content metadata leaked: %#v", parts) } + convertedImage := parts[2].(map[string]any) + if convertedImage["type"] != "image_url" || convertedImage["image_url"].(map[string]any)["url"] != "data:image/png;base64,BB" { + t.Fatalf("Anthropic image block was not normalized: %#v", convertedImage) + } call := messages[2].(map[string]any)["tool_calls"].([]any)[0].(map[string]any) if call["function"].(map[string]any)["extra"] != nil { t.Fatalf("tool call metadata leaked: %#v", call) } - for _, path := range []string{"messages[0].role", "messages[0].cache_control", "messages[1].content[0].cache_control", "messages[2].tool_calls[0].function.extra"} { + for _, path := range []string{"messages[0].role", "messages[0].cache_control", "messages[1].content[0].cache_control", "messages[1].content[2]", "messages[2].tool_calls[0].function.extra"} { if !hasChatChange(prepared.Changes, path) { t.Fatalf("missing change for %s", path) }