feat(agent): vision (multimodal) input + OpenAI max_completion_tokens fix (v0.36.0-beta)

CHANGELOG.md

@@ -6,6 +6,22 @@ All notable changes to **Tiger Core** (`webtigers/tiger-core`). Format follows
 
 ## [Unreleased]
 
+## [0.36.0-beta] — 2026-07-21
+
+### Added
+- **Vision (multimodal) input for AI providers.** A `user` turn may now carry
+  `images: [{mime, data:<base64>}, …]`; each adapter renders them in its native wire format — Anthropic
+  `image` blocks, the OpenAI-compatible `image_url` content parts, and Gemini `inlineData`. Text-only
+  turns are byte-identical to before, so this is fully backward-compatible.
+  `Tiger_Agent_Provider_Factory::supportsVision($provider, $model)` reports capability — deliberately
+  conservative (unknown model ⇒ `false`) so callers degrade to a caption/OCR pass rather than sending an
+  image to a text-only model. Consumed by TigerRoundtable to choose native-image vs. caption per room.
+
+### Fixed
+- **OpenAI's newer models rejected `max_tokens`.** `Tiger_Agent_Provider_OpenAi` now sends
+  `max_completion_tokens` (required by gpt-5 / o-series); the OpenAI-compatible base still defaults to
+  `max_tokens` via an overridable `_maxTokensField()`.
+
 ## [0.19.0-beta] — 2026-07-17
 
 ### Added

library/Tiger/Agent/Provider/Adapter.php

@@ -11,7 +11,10 @@
  * about the response contract, the Forge, or the ACL — the Loop owns all of that. Swapping
  * providers therefore changes the wire format only, never how Tiger reasons about a turn.
  *
- * Messages are the provider-neutral shape `[{role:'user'|'assistant', content:'…'}, …]`.
+ * Messages are the provider-neutral shape `[{role:'user'|'assistant', content:'…'}, …]`. A `user`
+ * turn MAY additionally carry `images: [{mime:'image/png', data:'<base64>'}, …]` — adapters whose
+ * model is multimodal render them in the provider's native image format; text-only adapters ignore
+ * them. Use Tiger_Agent_Provider_Factory::supportsVision() to decide before attaching images.
  *
  * @api
  */

library/Tiger/Agent/Provider/Anthropic.php

@@ -139,18 +139,29 @@ protected function _mapMessages(array $messages)
         foreach ($messages as $m) {
             $role    = (($m['role'] ?? 'user') === 'assistant') ? 'assistant' : 'user';
             $content = (string) ($m['content'] ?? '');
-            if ($content === '') {
+            $images  = ($role === 'user') ? (array) ($m['images'] ?? []) : [];   // images only on user turns
+            if ($content === '' && !$images) {
                 continue;
             }
+            // Content is a block array (a superset of a plain string) so text + images ride together.
+            $blocks = [];
+            if ($content !== '') { $blocks[] = ['type' => 'text', 'text' => $content]; }
+            foreach ($images as $img) {
+                $data = (string) ($img['data'] ?? '');
+                if ($data === '') { continue; }
+                $blocks[] = ['type' => 'image', 'source' => [
+                    'type' => 'base64', 'media_type' => (string) ($img['mime'] ?? 'image/png'), 'data' => $data,
+                ]];
+            }
             if ($out && $out[count($out) - 1]['role'] === $role) {
-                $out[count($out) - 1]['content'] .= "\n\n" . $content;
+                $out[count($out) - 1]['content'] = array_merge((array) $out[count($out) - 1]['content'], $blocks);
             } else {
-                $out[] = ['role' => $role, 'content' => $content];
+                $out[] = ['role' => $role, 'content' => $blocks];
             }
         }
         // The API requires the first message to be a user turn.
         if ($out && $out[0]['role'] !== 'user') {
-            array_unshift($out, ['role' => 'user', 'content' => '(continue)']);
+            array_unshift($out, ['role' => 'user', 'content' => [['type' => 'text', 'text' => '(continue)']]]);
         }
         return $out;
     }

library/Tiger/Agent/Provider/Factory.php

@@ -119,4 +119,41 @@ public static function options()
         }
         return $out;
     }
+
+    /**
+     * Can this provider+model accept image input (multimodal / "vision")?
+     *
+     * Deliberately CONSERVATIVE: return true only for models we're confident take images. When unsure
+     * we return false so the caller degrades to a caption/OCR pass (which works for every model) rather
+     * than sending an image to a text-only model and getting an API error. Model names churn, so this
+     * is a heuristic on the id — the safe direction is "no" (caption) not "yes" (risk a hard failure).
+     *
+     * @param  string $provider provider key
+     * @param  string $model    model id
+     * @return bool
+     */
+    public static function supportsVision($provider, $model)
+    {
+        $m = strtolower((string) $model);
+        switch ((string) $provider) {
+            case 'anthropic':                                   // every current Claude is multimodal
+                return strpos($m, 'claude') !== false;
+            case 'openai':                                      // 4o / 4.1 / 5 / o-series see images; not audio-only
+                return (bool) preg_match('~gpt-4o|gpt-4\.1|gpt-5|chatgpt-4o|(?:^|[^a-z])o[134](?:$|[^a-z])~', $m)
+                       && strpos($m, 'audio') === false && strpos($m, 'realtime') === false;
+            case 'gemini':                                      // 1.5+ and 2.x are all multimodal
+                return strpos($m, 'gemini') !== false;
+            case 'grok':
+                return strpos($m, 'vision') !== false || (bool) preg_match('~grok-(4|2-vision)~', $m);
+            case 'mistral':                                     // pixtral + the newer vision-capable small/medium
+                return strpos($m, 'pixtral') !== false || (bool) preg_match('~mistral-(small|medium)-2[45]~', $m);
+            case 'openrouter':                                  // route id names its upstream — reuse the same cues
+                return (bool) preg_match('~vision|gpt-4o|gpt-4\.1|gpt-5|claude|gemini|pixtral|llama-3\.2~', $m);
+            case 'groq':
+                return strpos($m, 'vision') !== false || strpos($m, 'llama-3.2') !== false;
+            case 'deepseek':                                    // text-only today
+            default:
+                return false;
+        }
+    }
 }

library/Tiger/Agent/Provider/Gemini.php

@@ -99,12 +99,21 @@ protected function _contents(array $messages)
         $out = [];
         foreach ($messages as $m) {
             $content = (string) ($m['content'] ?? '');
-            if ($content === '') { continue; }
-            $role = (($m['role'] ?? 'user') === 'assistant') ? 'model' : 'user';
+            $role    = (($m['role'] ?? 'user') === 'assistant') ? 'model' : 'user';
+            $images  = ($role === 'user') ? (array) ($m['images'] ?? []) : [];   // images only on user turns
+            if ($content === '' && !$images) { continue; }
+
+            $parts = [];
+            if ($content !== '') { $parts[] = ['text' => $content]; }
+            foreach ($images as $img) {
+                $data = (string) ($img['data'] ?? '');
+                if ($data === '') { continue; }
+                $parts[] = ['inlineData' => ['mimeType' => (string) ($img['mime'] ?? 'image/png'), 'data' => $data]];
+            }
             if ($out && $out[count($out) - 1]['role'] === $role) {
-                $out[count($out) - 1]['parts'][0]['text'] .= "\n\n" . $content;
+                $out[count($out) - 1]['parts'] = array_merge($out[count($out) - 1]['parts'], $parts);
             } else {
-                $out[] = ['role' => $role, 'parts' => [['text' => $content]]];
+                $out[] = ['role' => $role, 'parts' => $parts];
             }
         }
         if ($out && $out[0]['role'] !== 'user') {

library/Tiger/Agent/Provider/OpenAi.php

@@ -8,4 +8,7 @@ class Tiger_Agent_Provider_OpenAi extends Tiger_Agent_Provider_OpenAiCompatible
 {
     protected function _base()        { return 'https://api.openai.com/v1'; }
     protected function _providerKey() { return 'openai'; }
+
+    /** OpenAI's gpt-5 / o-series reject `max_tokens` and require `max_completion_tokens`. */
+    protected function _maxTokensField() { return 'max_completion_tokens'; }
 }

library/Tiger/Agent/Provider/OpenAiCompatible.php

@@ -30,6 +30,13 @@ protected function _headers($apiKey)
         return ['Content-Type: application/json', 'Authorization: Bearer ' . $apiKey];
     }
 
+    /** The output-token limit field. Most OpenAI-compatible APIs use `max_tokens`; OpenAI's own newer
+     *  models (gpt-5 / o-series) reject it and require `max_completion_tokens` — overridden per adapter. */
+    protected function _maxTokensField()
+    {
+        return 'max_tokens';
+    }
+
     /**
      * Run one completion against a chat/completions endpoint.
      *
@@ -46,15 +53,30 @@ public function complete($system, array $messages, $model, $apiKey)
         if ((string) $system !== '') { $turns[] = ['role' => 'system', 'content' => (string) $system]; }
         foreach ($messages as $m) {
             $content = (string) ($m['content'] ?? '');
-            if ($content === '') { continue; }
-            $role = (($m['role'] ?? 'user') === 'assistant') ? 'assistant' : 'user';
-            $turns[] = ['role' => $role, 'content' => $content];
+            $role    = (($m['role'] ?? 'user') === 'assistant') ? 'assistant' : 'user';
+            $images  = ($role === 'user') ? (array) ($m['images'] ?? []) : [];   // images only on user turns
+            if ($content === '' && !$images) { continue; }
+
+            if ($images) {
+                // Multimodal turn: content becomes an array of parts (OpenAI vision wire format).
+                $parts = [];
+                if ($content !== '') { $parts[] = ['type' => 'text', 'text' => $content]; }
+                foreach ($images as $img) {
+                    $data = (string) ($img['data'] ?? '');
+                    if ($data === '') { continue; }
+                    $mime = (string) ($img['mime'] ?? 'image/png');
+                    $parts[] = ['type' => 'image_url', 'image_url' => ['url' => 'data:' . $mime . ';base64,' . $data]];
+                }
+                $turns[] = ['role' => $role, 'content' => $parts];
+            } else {
+                $turns[] = ['role' => $role, 'content' => $content];
+            }
         }
 
         $body = $this->_post($this->_base() . '/chat/completions', [
-            'model'      => $model,
-            'messages'   => $turns,
-            'max_tokens' => self::MAX_TOKENS,
+            'model'                  => $model,
+            'messages'               => $turns,
+            $this->_maxTokensField() => self::MAX_TOKENS,
         ], $this->_headers($apiKey));
 
         $text = (string) ($body['choices'][0]['message']['content'] ?? '');

library/Tiger/Version.php

@@ -9,5 +9,5 @@
 class Tiger_Version
 {
     /** Current Tiger Core version. Keep in lockstep with the git tag cut for a release. */
-    const VERSION = '0.35.0-beta';
+    const VERSION = '0.36.0-beta';
 }