From 58b0ba5c47a288ae4a10442654696a6b20857dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=9F=E5=AE=9D=E5=9D=A4?= Date: Fri, 26 Jun 2026 10:03:59 +0800 Subject: [PATCH 1/4] feat(openai): add per-capability default model configuration via Builder Replace the single defaultModelName field with per-capability defaults (defaultVisionModel, defaultImageGenModel, defaultTtsModel, defaultSttModel) via a new Builder, matching OpenAIChatModel.Builder conventions. Existing constructors remain for backward compatibility. Default values unchanged: gpt-4o, dall-e-3, tts-1, whisper-1. Closes #1924 --- .../openai/tool/OpenAIMultiModalTool.java | 189 ++++++++++++++++-- .../openai/tool/OpenAIMultiModalToolTest.java | 34 +++- 2 files changed, 206 insertions(+), 17 deletions(-) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalTool.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalTool.java index 2097c96f90..88ba0b1d2a 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalTool.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalTool.java @@ -58,6 +58,11 @@ public class OpenAIMultiModalTool { private static final Logger log = LoggerFactory.getLogger(OpenAIMultiModalTool.class); + private static final String DEFAULT_VISION_MODEL = "gpt-4o"; + private static final String DEFAULT_IMAGE_GEN_MODEL = "dall-e-3"; + private static final String DEFAULT_TTS_MODEL = "tts-1"; + private static final String DEFAULT_STT_MODEL = "whisper-1"; + /** OpenAI API key. */ private final String apiKey; @@ -68,7 +73,16 @@ public class OpenAIMultiModalTool { private final String baseUrl; /** Default vision model used when the caller does not specify one. */ - private final String defaultModelName; + private final String defaultVisionModel; + + /** Default image generation model used when the caller does not specify one. */ + private final String defaultImageGenModel; + + /** Default TTS model used when the caller does not specify one. */ + private final String defaultTtsModel; + + /** Default speech-to-text model used when the caller does not specify one. */ + private final String defaultSttModel; /** * Create a new OpenAIMultiModalTool with default base URL. @@ -94,19 +108,38 @@ public OpenAIMultiModalTool(String apiKey, String baseUrl) { * * @param apiKey the OpenAI API key * @param baseUrl the base URL (null for default https://api.openai.com) - * @param defaultModelName the default vision model name used when the caller omits the model + * @param defaultVisionModel the default vision model name used when the caller omits the model * parameter (e.g., "gpt-4o" for OpenAI, or your backend's vision model name) */ - public OpenAIMultiModalTool(String apiKey, String baseUrl, String defaultModelName) { + public OpenAIMultiModalTool(String apiKey, String baseUrl, String defaultVisionModel) { + this( + apiKey, + baseUrl, + defaultVisionModel, + DEFAULT_IMAGE_GEN_MODEL, + DEFAULT_TTS_MODEL, + DEFAULT_STT_MODEL); + } + + private OpenAIMultiModalTool( + String apiKey, + String baseUrl, + String defaultVisionModel, + String defaultImageGenModel, + String defaultTtsModel, + String defaultSttModel) { if (apiKey == null || apiKey.trim().isEmpty()) { throw new IllegalArgumentException("OpenAI API key cannot be empty."); } - if (defaultModelName == null || defaultModelName.trim().isEmpty()) { - throw new IllegalArgumentException("defaultModelName cannot be empty."); + if (defaultVisionModel == null || defaultVisionModel.trim().isEmpty()) { + throw new IllegalArgumentException("defaultVisionModel cannot be empty."); } this.apiKey = apiKey; this.baseUrl = baseUrl; - this.defaultModelName = defaultModelName; + this.defaultVisionModel = defaultVisionModel; + this.defaultImageGenModel = defaultImageGenModel; + this.defaultTtsModel = defaultTtsModel; + this.defaultSttModel = defaultSttModel; this.client = new OpenAIClient(); } @@ -123,18 +156,138 @@ protected OpenAIMultiModalTool(OpenAIClient client) { * Create a new OpenAIMultiModalTool with custom client and default model (for testing). * * @param client the OpenAI client - * @param defaultModelName the default vision model name + * @param defaultVisionModel the default vision model name */ - protected OpenAIMultiModalTool(OpenAIClient client, String defaultModelName) { - if (defaultModelName == null || defaultModelName.trim().isEmpty()) { - throw new IllegalArgumentException("defaultModelName cannot be empty."); + protected OpenAIMultiModalTool(OpenAIClient client, String defaultVisionModel) { + if (defaultVisionModel == null || defaultVisionModel.trim().isEmpty()) { + throw new IllegalArgumentException("defaultVisionModel cannot be empty."); } this.apiKey = "test-key"; this.baseUrl = null; - this.defaultModelName = defaultModelName; + this.defaultVisionModel = defaultVisionModel; + this.defaultImageGenModel = DEFAULT_IMAGE_GEN_MODEL; + this.defaultTtsModel = DEFAULT_TTS_MODEL; + this.defaultSttModel = DEFAULT_STT_MODEL; this.client = client; } + /** + * Creates a new builder for OpenAIMultiModalTool. + * + * @return a new Builder instance + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for OpenAIMultiModalTool. + * + *

Each capability has an independent default model. When not set via builder, + * the existing hardcoded default is used — no change to current behavior. + */ + public static class Builder { + private String apiKey; + private String baseUrl; + private String defaultVisionModel; + private String defaultImageGenModel; + private String defaultTtsModel; + private String defaultSttModel; + + /** + * Sets the API key for OpenAI authentication. + * + * @param apiKey the API key + * @return this builder instance + */ + public Builder apiKey(String apiKey) { + this.apiKey = apiKey; + return this; + } + + /** + * Sets a custom base URL for OpenAI API. + * + * @param baseUrl the base URL (null for default) + * @return this builder instance + */ + public Builder baseUrl(String baseUrl) { + this.baseUrl = baseUrl; + return this; + } + + /** + * Sets the default vision model for image-to-text. + * + *

When not set, falls back to {@code "gpt-4o"}. + * + * @param defaultVisionModel the vision model name + * @return this builder instance + */ + public Builder defaultVisionModel(String defaultVisionModel) { + this.defaultVisionModel = defaultVisionModel; + return this; + } + + /** + * Sets the default model for text-to-image generation. + * + *

When not set, falls back to {@code "dall-e-3"}. + * + * @param defaultImageGenModel the image generation model name + * @return this builder instance + */ + public Builder defaultImageGenModel(String defaultImageGenModel) { + this.defaultImageGenModel = defaultImageGenModel; + return this; + } + + /** + * Sets the default TTS model for text-to-audio. + * + *

When not set, falls back to {@code "tts-1"}. + * + * @param defaultTtsModel the TTS model name + * @return this builder instance + */ + public Builder defaultTtsModel(String defaultTtsModel) { + this.defaultTtsModel = defaultTtsModel; + return this; + } + + /** + * Sets the default speech-to-text model for audio-to-text. + * + *

When not set, falls back to {@code "whisper-1"}. + * + * @param defaultSttModel the STT model name + * @return this builder instance + */ + public Builder defaultSttModel(String defaultSttModel) { + this.defaultSttModel = defaultSttModel; + return this; + } + + /** + * Builds the OpenAIMultiModalTool instance. + * + * @return configured OpenAIMultiModalTool instance + * @throws IllegalArgumentException if apiKey is not set + */ + public OpenAIMultiModalTool build() { + if (apiKey == null || apiKey.trim().isEmpty()) { + throw new IllegalArgumentException("apiKey must be set"); + } + return new OpenAIMultiModalTool( + apiKey, + baseUrl, + defaultVisionModel != null ? defaultVisionModel : DEFAULT_VISION_MODEL, + defaultImageGenModel != null ? defaultImageGenModel : DEFAULT_IMAGE_GEN_MODEL, + defaultTtsModel != null ? defaultTtsModel : DEFAULT_TTS_MODEL, + defaultSttModel != null ? defaultSttModel : DEFAULT_STT_MODEL); + } + } + /** * Generate image(s) based on the given prompt. * @@ -186,7 +339,9 @@ public Mono openaiTextToImage( String responseFormat) { String finalModel = - Optional.ofNullable(model).filter(s -> !s.trim().isEmpty()).orElse("dall-e-3"); + Optional.ofNullable(model) + .filter(s -> !s.trim().isEmpty()) + .orElse(this.defaultImageGenModel); Integer finalN = Optional.ofNullable(n).orElse(1); String finalSize = Optional.ofNullable(size).filter(s -> !s.trim().isEmpty()).orElse("1024x1024"); @@ -318,7 +473,7 @@ public Mono openaiImageToText( String finalModel = Optional.ofNullable(model) .filter(s -> !s.trim().isEmpty()) - .orElse(this.defaultModelName); + .orElse(this.defaultVisionModel); String finalPrompt = Optional.ofNullable(prompt) .filter(s -> !s.trim().isEmpty()) @@ -447,7 +602,9 @@ public Mono openaiTextToAudio( Double speed) { String finalModel = - Optional.ofNullable(model).filter(s -> !s.trim().isEmpty()).orElse("tts-1"); + Optional.ofNullable(model) + .filter(s -> !s.trim().isEmpty()) + .orElse(this.defaultTtsModel); String finalVoice = Optional.ofNullable(voice).filter(s -> !s.trim().isEmpty()).orElse("alloy"); String finalResponseFormat = @@ -541,7 +698,9 @@ public Mono openaiAudioToText( Double temperature) { String finalModel = - Optional.ofNullable(model).filter(s -> !s.trim().isEmpty()).orElse("whisper-1"); + Optional.ofNullable(model) + .filter(s -> !s.trim().isEmpty()) + .orElse(this.defaultSttModel); String finalResponseFormat = Optional.ofNullable(responseFormat).filter(s -> !s.trim().isEmpty()).orElse("text"); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalToolTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalToolTest.java index 602dd89022..779a2115ce 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalToolTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalToolTest.java @@ -31,6 +31,7 @@ import io.agentscope.core.message.URLSource; import io.agentscope.extensions.model.openai.OpenAIClient; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; @@ -81,8 +82,7 @@ void testImageToText_usesCustomDefaultModel() { argThat( req -> { @SuppressWarnings("unchecked") - java.util.Map map = - (java.util.Map) req; + Map map = (Map) req; return "my-custom-vision-model".equals(map.get("model")); }))) .thenReturn(jsonResponse); @@ -104,4 +104,34 @@ void testConstructor_rejectsBlankDefaultModelName() { assertThrows( IllegalArgumentException.class, () -> new OpenAIMultiModalTool("key", null, null)); } + + @Test + void testBuilder_withAllCustomModels() { + OpenAIMultiModalTool tool = + OpenAIMultiModalTool.builder() + .apiKey("sk-test") + .baseUrl("https://custom.api.com") + .defaultVisionModel("gpt-4o-mini") + .defaultImageGenModel("dall-e-2") + .defaultTtsModel("tts-1-hd") + .defaultSttModel("whisper-1") + .build(); + assertNotNull(tool); + } + + @Test + void testBuilder_usesDefaultsWhenNotSet() { + OpenAIMultiModalTool tool = OpenAIMultiModalTool.builder().apiKey("sk-test").build(); + assertNotNull(tool); + } + + @Test + void testBuilder_rejectsBlankApiKey() { + assertThrows( + IllegalArgumentException.class, + () -> OpenAIMultiModalTool.builder().apiKey("").build()); + assertThrows( + IllegalArgumentException.class, + () -> OpenAIMultiModalTool.builder().apiKey(null).build()); + } } From b68a525b3c3dae327f1df80e3ff8c69255f681fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=9F=E5=AE=9D=E5=9D=A4?= Date: Fri, 26 Jun 2026 10:11:28 +0800 Subject: [PATCH 2/4] chore: retrigger CI From 808d1fe116f291f1e04157283ff0a5c23b2b06c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=9F=E5=AE=9D=E5=9D=A4?= Date: Fri, 26 Jun 2026 10:14:22 +0800 Subject: [PATCH 3/4] test(core): fix flaky SubAgentTool timeout test on boundedElastic When the outer timeout cancels the subscription, boundedElastic interrupts the sleeping thread via Future.cancel(true). The old code wrapped both the sleep and the file write in one try-catch, so the interrupt silently skipped the write. Split them so the write always executes even if sleep is interrupted. Closes #1924 (part of PR #1925) --- .../subagent/SubAgentToolTimeoutRetryIntegrationTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/agentscope-core/src/test/java/io/agentscope/core/tool/subagent/SubAgentToolTimeoutRetryIntegrationTest.java b/agentscope-core/src/test/java/io/agentscope/core/tool/subagent/SubAgentToolTimeoutRetryIntegrationTest.java index e49c2c0338..4b304776d1 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/tool/subagent/SubAgentToolTimeoutRetryIntegrationTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/tool/subagent/SubAgentToolTimeoutRetryIntegrationTest.java @@ -103,6 +103,10 @@ public Mono callAsync(ToolCallParam p) { () -> { try { Thread.sleep(4_000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { Files.writeString( tmpFile, "slow_tool round done\n"); } catch (Exception ignored) { From d33f44f9ba3883bcff993dd39bd626ab33e6e32f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=9F=E5=AE=9D=E5=9D=A4?= Date: Fri, 3 Jul 2026 15:54:22 +0800 Subject: [PATCH 4/4] fix(tool): drop model param from OpenAIMultiModalTool to enforce configured default The optional model param was documented with OpenAI-specific examples (e.g. 'dall-e-3'), which lured the LLM into passing a model name and bypassing the configured defaultXxxModel. Remove the param from all four capabilities (text-to-image / image-to-text / text-to-audio / audio-to-text) so the configured default is always used. Adapt existing tests to the new signatures. --- .../openai/tool/OpenAIMultiModalTool.java | 46 ++----------------- .../openai/tool/OpenAIMultiModalToolTest.java | 4 +- 2 files changed, 6 insertions(+), 44 deletions(-) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalTool.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalTool.java index 88ba0b1d2a..ccb056e65e 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalTool.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalTool.java @@ -292,7 +292,6 @@ public OpenAIMultiModalTool build() { * Generate image(s) based on the given prompt. * * @param prompt the text prompt to generate image - * @param model the model to use (e.g., "dall-e-3", "dall-e-2") * @param n the number of images to generate (1 for dall-e-3, 1-10 for dall-e-2) * @param size the size of the image (e.g., "1024x1024", "1792x1024", "1024x1792") * @param quality the quality of the image ("standard" or "hd" for dall-e-3) @@ -307,11 +306,6 @@ public OpenAIMultiModalTool build() { public Mono openaiTextToImage( @ToolParam(name = "prompt", description = "The text prompt to generate image") String prompt, - @ToolParam( - name = "model", - description = "The model to use, e.g., 'dall-e-3', 'dall-e-2'", - required = false) - String model, @ToolParam( name = "n", description = @@ -338,10 +332,7 @@ public Mono openaiTextToImage( required = false) String responseFormat) { - String finalModel = - Optional.ofNullable(model) - .filter(s -> !s.trim().isEmpty()) - .orElse(this.defaultImageGenModel); + String finalModel = this.defaultImageGenModel; Integer finalN = Optional.ofNullable(n).orElse(1); String finalSize = Optional.ofNullable(size).filter(s -> !s.trim().isEmpty()).orElse("1024x1024"); @@ -437,7 +428,6 @@ public Mono openaiTextToImage( * * @param imageUrls the URLs of the images to analyze * @param prompt the text prompt describing what to extract from the images - * @param model the vision model to use (leave empty to use the configured default) * @param maxTokens the maximum number of tokens in the response * @return a ToolResultBlock containing the text description of the images */ @@ -457,23 +447,13 @@ public Mono openaiImageToText( "The text prompt describing what to extract from the images", required = false) String prompt, - @ToolParam( - name = "model", - description = - "The vision model to use (leave empty to use the configured" - + " default)", - required = false) - String model, @ToolParam( name = "max_tokens", description = "The maximum number of tokens in the response", required = false) Integer maxTokens) { - String finalModel = - Optional.ofNullable(model) - .filter(s -> !s.trim().isEmpty()) - .orElse(this.defaultVisionModel); + String finalModel = this.defaultVisionModel; String finalPrompt = Optional.ofNullable(prompt) .filter(s -> !s.trim().isEmpty()) @@ -565,7 +545,6 @@ public Mono openaiImageToText( * Convert text to audio (speech) using OpenAI TTS models. * * @param text the text to convert to speech - * @param model the TTS model to use (e.g., "tts-1", "tts-1-hd") * @param voice the voice to use ("alloy", "echo", "fable", "onyx", "nova", "shimmer") * @param responseFormat the audio format ("mp3", "opus", "aac", "flac") * @param speed the speed of the speech (0.25 to 4.0) @@ -578,11 +557,6 @@ public Mono openaiImageToText( + "Returns audio as base64 data.") public Mono openaiTextToAudio( @ToolParam(name = "text", description = "The text to convert to speech") String text, - @ToolParam( - name = "model", - description = "The TTS model to use, e.g., 'tts-1', 'tts-1-hd'", - required = false) - String model, @ToolParam( name = "voice", description = @@ -601,10 +575,7 @@ public Mono openaiTextToAudio( required = false) Double speed) { - String finalModel = - Optional.ofNullable(model) - .filter(s -> !s.trim().isEmpty()) - .orElse(this.defaultTtsModel); + String finalModel = this.defaultTtsModel; String finalVoice = Optional.ofNullable(voice).filter(s -> !s.trim().isEmpty()).orElse("alloy"); String finalResponseFormat = @@ -654,7 +625,6 @@ public Mono openaiTextToAudio( * This is a placeholder implementation. * * @param audioUrl the URL of the audio file to transcribe - * @param model the transcription model to use (e.g., "whisper-1") * @param language the language of the audio (ISO-639-1 code, optional) * @param prompt optional text to guide the model's style * @param responseFormat the format of the response ("json", "text", "verbose_json", etc.) @@ -669,11 +639,6 @@ public Mono openaiTextToAudio( public Mono openaiAudioToText( @ToolParam(name = "audio_url", description = "The URL of the audio file to transcribe") String audioUrl, - @ToolParam( - name = "model", - description = "The transcription model to use, e.g., 'whisper-1'", - required = false) - String model, @ToolParam( name = "language", description = "The language of the audio (ISO-639-1 code, optional)", @@ -697,10 +662,7 @@ public Mono openaiAudioToText( required = false) Double temperature) { - String finalModel = - Optional.ofNullable(model) - .filter(s -> !s.trim().isEmpty()) - .orElse(this.defaultSttModel); + String finalModel = this.defaultSttModel; String finalResponseFormat = Optional.ofNullable(responseFormat).filter(s -> !s.trim().isEmpty()).orElse("text"); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalToolTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalToolTest.java index 779a2115ce..950cbf3b62 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalToolTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/tool/OpenAIMultiModalToolTest.java @@ -56,7 +56,7 @@ void testTextToImage_Url() { .thenReturn(jsonResponse); Mono resultMono = - tool.openaiTextToImage(prompt, "dall-e-3", 1, "1024x1024", "standard", "url"); + tool.openaiTextToImage(prompt, 1, "1024x1024", "standard", "url"); ToolResultBlock result = resultMono.block(); assertNotNull(result); @@ -91,7 +91,7 @@ void testImageToText_usesCustomDefaultModel() { new OpenAIMultiModalTool(client, "my-custom-vision-model"); Mono resultMono = - toolWithCustomModel.openaiImageToText(imageUrl, null, null, null); + toolWithCustomModel.openaiImageToText(imageUrl, null, null); ToolResultBlock result = resultMono.block(); assertNotNull(result);