diff --git a/CHANGELOG.md b/CHANGELOG.md index a5a8e3b..5efa64d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,22 @@ All notable changes to **Tiger Core** (`webtigers/tiger-core`). Format follows ## [Unreleased] +## [0.36.0-beta] — 2026-07-21 + +### Added +- **Vision (multimodal) input for AI providers.** A `user` turn may now carry + `images: [{mime, data:}, …]`; each adapter renders them in its native wire format — Anthropic + `image` blocks, the OpenAI-compatible `image_url` content parts, and Gemini `inlineData`. Text-only + turns are byte-identical to before, so this is fully backward-compatible. + `Tiger_Agent_Provider_Factory::supportsVision($provider, $model)` reports capability — deliberately + conservative (unknown model ⇒ `false`) so callers degrade to a caption/OCR pass rather than sending an + image to a text-only model. Consumed by TigerRoundtable to choose native-image vs. caption per room. + +### Fixed +- **OpenAI's newer models rejected `max_tokens`.** `Tiger_Agent_Provider_OpenAi` now sends + `max_completion_tokens` (required by gpt-5 / o-series); the OpenAI-compatible base still defaults to + `max_tokens` via an overridable `_maxTokensField()`. + ## [0.19.0-beta] — 2026-07-17 ### Added diff --git a/library/Tiger/Agent/Provider/Adapter.php b/library/Tiger/Agent/Provider/Adapter.php index c4686ec..929b0ec 100644 --- a/library/Tiger/Agent/Provider/Adapter.php +++ b/library/Tiger/Agent/Provider/Adapter.php @@ -11,7 +11,10 @@ * about the response contract, the Forge, or the ACL — the Loop owns all of that. Swapping * providers therefore changes the wire format only, never how Tiger reasons about a turn. * - * Messages are the provider-neutral shape `[{role:'user'|'assistant', content:'…'}, …]`. + * Messages are the provider-neutral shape `[{role:'user'|'assistant', content:'…'}, …]`. A `user` + * turn MAY additionally carry `images: [{mime:'image/png', data:''}, …]` — adapters whose + * model is multimodal render them in the provider's native image format; text-only adapters ignore + * them. Use Tiger_Agent_Provider_Factory::supportsVision() to decide before attaching images. * * @api */ diff --git a/library/Tiger/Agent/Provider/Anthropic.php b/library/Tiger/Agent/Provider/Anthropic.php index 02d7c6e..a1fa5f1 100644 --- a/library/Tiger/Agent/Provider/Anthropic.php +++ b/library/Tiger/Agent/Provider/Anthropic.php @@ -139,18 +139,29 @@ protected function _mapMessages(array $messages) foreach ($messages as $m) { $role = (($m['role'] ?? 'user') === 'assistant') ? 'assistant' : 'user'; $content = (string) ($m['content'] ?? ''); - if ($content === '') { + $images = ($role === 'user') ? (array) ($m['images'] ?? []) : []; // images only on user turns + if ($content === '' && !$images) { continue; } + // Content is a block array (a superset of a plain string) so text + images ride together. + $blocks = []; + if ($content !== '') { $blocks[] = ['type' => 'text', 'text' => $content]; } + foreach ($images as $img) { + $data = (string) ($img['data'] ?? ''); + if ($data === '') { continue; } + $blocks[] = ['type' => 'image', 'source' => [ + 'type' => 'base64', 'media_type' => (string) ($img['mime'] ?? 'image/png'), 'data' => $data, + ]]; + } if ($out && $out[count($out) - 1]['role'] === $role) { - $out[count($out) - 1]['content'] .= "\n\n" . $content; + $out[count($out) - 1]['content'] = array_merge((array) $out[count($out) - 1]['content'], $blocks); } else { - $out[] = ['role' => $role, 'content' => $content]; + $out[] = ['role' => $role, 'content' => $blocks]; } } // The API requires the first message to be a user turn. if ($out && $out[0]['role'] !== 'user') { - array_unshift($out, ['role' => 'user', 'content' => '(continue)']); + array_unshift($out, ['role' => 'user', 'content' => [['type' => 'text', 'text' => '(continue)']]]); } return $out; } diff --git a/library/Tiger/Agent/Provider/Factory.php b/library/Tiger/Agent/Provider/Factory.php index 880c629..4f51ed9 100644 --- a/library/Tiger/Agent/Provider/Factory.php +++ b/library/Tiger/Agent/Provider/Factory.php @@ -119,4 +119,41 @@ public static function options() } return $out; } + + /** + * Can this provider+model accept image input (multimodal / "vision")? + * + * Deliberately CONSERVATIVE: return true only for models we're confident take images. When unsure + * we return false so the caller degrades to a caption/OCR pass (which works for every model) rather + * than sending an image to a text-only model and getting an API error. Model names churn, so this + * is a heuristic on the id — the safe direction is "no" (caption) not "yes" (risk a hard failure). + * + * @param string $provider provider key + * @param string $model model id + * @return bool + */ + public static function supportsVision($provider, $model) + { + $m = strtolower((string) $model); + switch ((string) $provider) { + case 'anthropic': // every current Claude is multimodal + return strpos($m, 'claude') !== false; + case 'openai': // 4o / 4.1 / 5 / o-series see images; not audio-only + return (bool) preg_match('~gpt-4o|gpt-4\.1|gpt-5|chatgpt-4o|(?:^|[^a-z])o[134](?:$|[^a-z])~', $m) + && strpos($m, 'audio') === false && strpos($m, 'realtime') === false; + case 'gemini': // 1.5+ and 2.x are all multimodal + return strpos($m, 'gemini') !== false; + case 'grok': + return strpos($m, 'vision') !== false || (bool) preg_match('~grok-(4|2-vision)~', $m); + case 'mistral': // pixtral + the newer vision-capable small/medium + return strpos($m, 'pixtral') !== false || (bool) preg_match('~mistral-(small|medium)-2[45]~', $m); + case 'openrouter': // route id names its upstream — reuse the same cues + return (bool) preg_match('~vision|gpt-4o|gpt-4\.1|gpt-5|claude|gemini|pixtral|llama-3\.2~', $m); + case 'groq': + return strpos($m, 'vision') !== false || strpos($m, 'llama-3.2') !== false; + case 'deepseek': // text-only today + default: + return false; + } + } } diff --git a/library/Tiger/Agent/Provider/Gemini.php b/library/Tiger/Agent/Provider/Gemini.php index 98874ec..26e3b33 100644 --- a/library/Tiger/Agent/Provider/Gemini.php +++ b/library/Tiger/Agent/Provider/Gemini.php @@ -99,12 +99,21 @@ protected function _contents(array $messages) $out = []; foreach ($messages as $m) { $content = (string) ($m['content'] ?? ''); - if ($content === '') { continue; } - $role = (($m['role'] ?? 'user') === 'assistant') ? 'model' : 'user'; + $role = (($m['role'] ?? 'user') === 'assistant') ? 'model' : 'user'; + $images = ($role === 'user') ? (array) ($m['images'] ?? []) : []; // images only on user turns + if ($content === '' && !$images) { continue; } + + $parts = []; + if ($content !== '') { $parts[] = ['text' => $content]; } + foreach ($images as $img) { + $data = (string) ($img['data'] ?? ''); + if ($data === '') { continue; } + $parts[] = ['inlineData' => ['mimeType' => (string) ($img['mime'] ?? 'image/png'), 'data' => $data]]; + } if ($out && $out[count($out) - 1]['role'] === $role) { - $out[count($out) - 1]['parts'][0]['text'] .= "\n\n" . $content; + $out[count($out) - 1]['parts'] = array_merge($out[count($out) - 1]['parts'], $parts); } else { - $out[] = ['role' => $role, 'parts' => [['text' => $content]]]; + $out[] = ['role' => $role, 'parts' => $parts]; } } if ($out && $out[0]['role'] !== 'user') { diff --git a/library/Tiger/Agent/Provider/OpenAi.php b/library/Tiger/Agent/Provider/OpenAi.php index 3651974..b09caac 100644 --- a/library/Tiger/Agent/Provider/OpenAi.php +++ b/library/Tiger/Agent/Provider/OpenAi.php @@ -8,4 +8,7 @@ class Tiger_Agent_Provider_OpenAi extends Tiger_Agent_Provider_OpenAiCompatible { protected function _base() { return 'https://api.openai.com/v1'; } protected function _providerKey() { return 'openai'; } + + /** OpenAI's gpt-5 / o-series reject `max_tokens` and require `max_completion_tokens`. */ + protected function _maxTokensField() { return 'max_completion_tokens'; } } diff --git a/library/Tiger/Agent/Provider/OpenAiCompatible.php b/library/Tiger/Agent/Provider/OpenAiCompatible.php index 4d6741a..5632332 100644 --- a/library/Tiger/Agent/Provider/OpenAiCompatible.php +++ b/library/Tiger/Agent/Provider/OpenAiCompatible.php @@ -30,6 +30,13 @@ protected function _headers($apiKey) return ['Content-Type: application/json', 'Authorization: Bearer ' . $apiKey]; } + /** The output-token limit field. Most OpenAI-compatible APIs use `max_tokens`; OpenAI's own newer + * models (gpt-5 / o-series) reject it and require `max_completion_tokens` — overridden per adapter. */ + protected function _maxTokensField() + { + return 'max_tokens'; + } + /** * Run one completion against a chat/completions endpoint. * @@ -46,15 +53,30 @@ public function complete($system, array $messages, $model, $apiKey) if ((string) $system !== '') { $turns[] = ['role' => 'system', 'content' => (string) $system]; } foreach ($messages as $m) { $content = (string) ($m['content'] ?? ''); - if ($content === '') { continue; } - $role = (($m['role'] ?? 'user') === 'assistant') ? 'assistant' : 'user'; - $turns[] = ['role' => $role, 'content' => $content]; + $role = (($m['role'] ?? 'user') === 'assistant') ? 'assistant' : 'user'; + $images = ($role === 'user') ? (array) ($m['images'] ?? []) : []; // images only on user turns + if ($content === '' && !$images) { continue; } + + if ($images) { + // Multimodal turn: content becomes an array of parts (OpenAI vision wire format). + $parts = []; + if ($content !== '') { $parts[] = ['type' => 'text', 'text' => $content]; } + foreach ($images as $img) { + $data = (string) ($img['data'] ?? ''); + if ($data === '') { continue; } + $mime = (string) ($img['mime'] ?? 'image/png'); + $parts[] = ['type' => 'image_url', 'image_url' => ['url' => 'data:' . $mime . ';base64,' . $data]]; + } + $turns[] = ['role' => $role, 'content' => $parts]; + } else { + $turns[] = ['role' => $role, 'content' => $content]; + } } $body = $this->_post($this->_base() . '/chat/completions', [ - 'model' => $model, - 'messages' => $turns, - 'max_tokens' => self::MAX_TOKENS, + 'model' => $model, + 'messages' => $turns, + $this->_maxTokensField() => self::MAX_TOKENS, ], $this->_headers($apiKey)); $text = (string) ($body['choices'][0]['message']['content'] ?? ''); diff --git a/library/Tiger/Version.php b/library/Tiger/Version.php index 853fc56..26dcfbc 100644 --- a/library/Tiger/Version.php +++ b/library/Tiger/Version.php @@ -9,5 +9,5 @@ class Tiger_Version { /** Current Tiger Core version. Keep in lockstep with the git tag cut for a release. */ - const VERSION = '0.35.0-beta'; + const VERSION = '0.36.0-beta'; }