From a88cb48caf92d3b7ee3f696853ee6b241ca4d26d Mon Sep 17 00:00:00 2001 From: guslegend <1670547022@qq.com> Date: Tue, 25 Aug 2026 13:24:50 +0800 Subject: [PATCH 01/10] feat(core): establish prompt cache contracts --- .../core/formatter/AbstractBaseFormatter.java | 85 ++++++++- .../core/message/MessageMetadataKeys.java | 5 +- .../io/agentscope/core/model/ChatUsage.java | 51 +++++- .../core/model/GenerateOptions.java | 11 +- .../AbstractBaseFormatterPromptCacheTest.java | 163 ++++++++++++++++++ .../message/MsgUsageSerializationTest.java | 29 +++- .../core/model/GenerateOptionsTest.java | 22 +++ 7 files changed, 351 insertions(+), 15 deletions(-) create mode 100644 agentscope-core/src/test/java/io/agentscope/core/formatter/AbstractBaseFormatterPromptCacheTest.java diff --git a/agentscope-core/src/main/java/io/agentscope/core/formatter/AbstractBaseFormatter.java b/agentscope-core/src/main/java/io/agentscope/core/formatter/AbstractBaseFormatter.java index 16bab108e7..e7806ee716 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/formatter/AbstractBaseFormatter.java +++ b/agentscope-core/src/main/java/io/agentscope/core/formatter/AbstractBaseFormatter.java @@ -37,8 +37,10 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Base64; +import java.util.Collections; import java.util.List; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.slf4j.Logger; @@ -62,6 +64,8 @@ public abstract class AbstractBaseFormatter implements Formatter { + protected static final int MAX_PROMPT_CACHE_BREAKPOINTS = 4; + private static final Logger log = LoggerFactory.getLogger(AbstractBaseFormatter.class); /** @@ -160,8 +164,10 @@ protected String formatRoleLabel(MsgRole role) { /** * Check if a message should bypass history merging in multiagent formatters. - * Messages with the {@link MessageMetadataKeys#BYPASS_MULTIAGENT_HISTORY_MERGE} flag set to {@code true} - * should be kept as separate messages rather than merged into the conversation history. + * Messages with the {@link MessageMetadataKeys#BYPASS_MULTIAGENT_HISTORY_MERGE} or {@link + * MessageMetadataKeys#CACHE_CONTROL} flag set to {@code true} should be kept as separate messages + * rather than merged into the conversation history. Preserving explicitly marked cache + * boundaries prevents formatter-level history merging from discarding the marker. * * @param msg The message to check * @return true if message should bypass history merging @@ -172,7 +178,80 @@ protected boolean shouldBypassHistory(Msg msg) { } Object bypassFlag = msg.getMetadata().get(MessageMetadataKeys.BYPASS_MULTIAGENT_HISTORY_MERGE); - return Boolean.TRUE.equals(bypassFlag); + Object cacheControlFlag = msg.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL); + return Boolean.TRUE.equals(bypassFlag) || Boolean.TRUE.equals(cacheControlFlag); + } + + /** + * Select prompt cache breakpoints while respecting the provider limit. + * + *

Explicitly marked items always take priority. When automatic selection is enabled, the + * first cacheable system item and the last cacheable non-system item fill any remaining slots. + * The returned items preserve their original request order. + * + * @param items provider request items to inspect + * @param automatic whether automatic cache breakpoint selection is enabled + * @param explicitlyMarked predicate identifying explicitly marked items + * @param systemItem predicate identifying system items + * @param cacheableItem predicate identifying items eligible for automatic caching + * @param provider request item type + * @return selected cache breakpoint items in request order + * @throws IllegalArgumentException when more than four items are explicitly marked + */ + protected List selectPromptCacheBreakpoints( + List items, + boolean automatic, + Predicate explicitlyMarked, + Predicate systemItem, + Predicate cacheableItem) { + if (items == null || items.isEmpty()) { + return List.of(); + } + + List selectedIndices = new ArrayList<>(); + for (int i = 0; i < items.size(); i++) { + if (explicitlyMarked.test(items.get(i))) { + selectedIndices.add(i); + } + } + + if (selectedIndices.size() > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "Prompt cache supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " explicit breakpoints, but got " + + selectedIndices.size()); + } + + if (automatic && selectedIndices.size() < MAX_PROMPT_CACHE_BREAKPOINTS) { + for (int i = 0; i < items.size(); i++) { + T item = items.get(i); + if (systemItem.test(item) && cacheableItem.test(item)) { + addBreakpointIfAvailable(selectedIndices, i); + break; + } + } + + for (int i = items.size() - 1; + i >= 0 && selectedIndices.size() < MAX_PROMPT_CACHE_BREAKPOINTS; + i--) { + T item = items.get(i); + if (!systemItem.test(item) && cacheableItem.test(item)) { + addBreakpointIfAvailable(selectedIndices, i); + break; + } + } + } + + Collections.sort(selectedIndices); + return selectedIndices.stream().map(items::get).toList(); + } + + private void addBreakpointIfAvailable(List selectedIndices, int index) { + if (selectedIndices.size() < MAX_PROMPT_CACHE_BREAKPOINTS + && !selectedIndices.contains(index)) { + selectedIndices.add(index); + } } /** diff --git a/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java b/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java index d34e72adef..207c27b3e7 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java +++ b/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java @@ -113,10 +113,11 @@ private MessageMetadataKeys() { * {"type": "ephemeral"} to this message during formatting. This allows users to manually * mark specific * messages for caching, independent of the automatic cache control strategy configured via - * {@link io.agentscope.core.model.GenerateOptions#getCacheControl()}. + * {@link io.agentscope.core.model.GenerateOptions#getCacheControl()}. Explicit markers remain + * effective when automatic cache control is disabled or unset. * *

Manually marked messages take priority over the automatic strategy — they will not be - * overwritten. + * overwritten. A request may contain at most four explicitly marked cache breakpoints. * *

Type: Boolean *

Example: diff --git a/agentscope-core/src/main/java/io/agentscope/core/model/ChatUsage.java b/agentscope-core/src/main/java/io/agentscope/core/model/ChatUsage.java index 1e22b8ddeb..2a9a13cf27 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/model/ChatUsage.java +++ b/agentscope-core/src/main/java/io/agentscope/core/model/ChatUsage.java @@ -22,14 +22,15 @@ * Represents token usage information for chat completion responses. * *

This immutable data class tracks the number of tokens used during a chat completion, - * including input tokens (prompt), output tokens (generated response), cached input tokens, and - * execution time. + * including input tokens (prompt), output tokens (generated response), prompt cache reads and + * writes, and execution time. */ public class ChatUsage { private final int inputTokens; private final int outputTokens; private final int cachedTokens; + private final int cacheCreationInputTokens; private final double time; /** @@ -55,15 +56,33 @@ public ChatUsage(int inputTokens, int outputTokens, double time) { * {@code inputTokens}); {@code 0} when the provider does not report cache information * @param time the execution time in seconds */ + public ChatUsage(int inputTokens, int outputTokens, int cachedTokens, double time) { + this(inputTokens, outputTokens, cachedTokens, 0, time); + } + + /** + * Creates a new ChatUsage instance with prompt cache read and creation information. + * + * @param inputTokens the total number of tokens used for the input/prompt + * @param outputTokens the number of tokens used for the output/generated response + * @param cachedTokens the number of input tokens served from the prompt cache (a subset of + * {@code inputTokens}); {@code 0} when the provider does not report cache information + * @param cacheCreationInputTokens the number of input tokens written to the prompt cache (a + * subset of {@code inputTokens}); {@code 0} when the provider does not report cache + * creation information + * @param time the execution time in seconds + */ @JsonCreator public ChatUsage( @JsonProperty("inputTokens") int inputTokens, @JsonProperty("outputTokens") int outputTokens, @JsonProperty("cachedTokens") int cachedTokens, + @JsonProperty("cacheCreationInputTokens") int cacheCreationInputTokens, @JsonProperty("time") double time) { this.inputTokens = inputTokens; this.outputTokens = outputTokens; this.cachedTokens = cachedTokens; + this.cacheCreationInputTokens = cacheCreationInputTokens; this.time = time; } @@ -98,6 +117,18 @@ public int getCachedTokens() { return cachedTokens; } + /** + * Gets the number of input tokens written to the prompt cache. + * + *

Cache creation tokens are a subset of {@link #getInputTokens()}, not an additional amount. + * Returns {@code 0} when the provider does not report cache creation information. + * + * @return the number of input tokens written to the prompt cache + */ + public int getCacheCreationInputTokens() { + return cacheCreationInputTokens; + } + /** * Gets the total number of tokens used. * @@ -132,6 +163,7 @@ public static class Builder { private int inputTokens; private int outputTokens; private int cachedTokens; + private int cacheCreationInputTokens; private double time; /** @@ -168,6 +200,18 @@ public Builder cachedTokens(int cachedTokens) { return this; } + /** + * Sets the number of input tokens written to the prompt cache. + * + * @param cacheCreationInputTokens the number of input tokens written to the prompt cache + * (a subset of {@code inputTokens}) + * @return this builder instance + */ + public Builder cacheCreationInputTokens(int cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + return this; + } + /** * Sets the execution time. * @@ -185,7 +229,8 @@ public Builder time(double time) { * @return a new ChatUsage instance */ public ChatUsage build() { - return new ChatUsage(inputTokens, outputTokens, cachedTokens, time); + return new ChatUsage( + inputTokens, outputTokens, cachedTokens, cacheCreationInputTokens, time); } } } diff --git a/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java b/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java index 87fc0243ec..9eb95c1ee1 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java +++ b/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java @@ -309,15 +309,14 @@ public Long getSeed() { /** * Gets whether cache control is enabled for prompt caching. * - *

When true, the formatter will automatically add cache_control: - * {"type": "ephemeral"} to system messages and the last message in the request. This - * enables prompt - * caching on supported providers (e.g., Anthropic, DashScope, OpenAI-compatible APIs) to reduce - * latency and cost. + *

When true, a supporting formatter automatically selects the first cacheable system message + * and the last cacheable non-system message as prompt cache breakpoints. Provider-specific + * formatters are responsible for encoding the selected breakpoints in the protocol they support. * *

Users can also manually mark individual messages for caching via {@link * io.agentscope.core.message.MessageMetadataKeys#CACHE_CONTROL} metadata. Manually marked - * messages take priority over the automatic strategy. + * messages take priority over the automatic strategy and remain effective when this option is + * false or unset. Providers currently support at most four explicit breakpoints per request. * * @return true if cache control is enabled, false or null if not set */ diff --git a/agentscope-core/src/test/java/io/agentscope/core/formatter/AbstractBaseFormatterPromptCacheTest.java b/agentscope-core/src/test/java/io/agentscope/core/formatter/AbstractBaseFormatterPromptCacheTest.java new file mode 100644 index 0000000000..69b0b152eb --- /dev/null +++ b/agentscope-core/src/test/java/io/agentscope/core/formatter/AbstractBaseFormatterPromptCacheTest.java @@ -0,0 +1,163 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.formatter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.agentscope.core.message.MessageMetadataKeys; +import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; +import io.agentscope.core.model.ChatResponse; +import io.agentscope.core.model.GenerateOptions; +import io.agentscope.core.model.ToolSchema; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("AbstractBaseFormatter prompt cache tests") +class AbstractBaseFormatterPromptCacheTest { + + private final TestFormatter formatter = new TestFormatter(); + + @Test + void selectsFirstSystemAndLastCacheableConversationItemAutomatically() { + CacheItem firstSystem = new CacheItem("system-1", false, true, true); + CacheItem secondSystem = new CacheItem("system-2", false, true, true); + CacheItem user = new CacheItem("user", false, false, true); + CacheItem emptyAssistant = new CacheItem("assistant-empty", false, false, false); + + List selected = + formatter.select(List.of(firstSystem, secondSystem, user, emptyAssistant), true); + + assertEquals(List.of(firstSystem, user), selected); + } + + @Test + void keepsExplicitBreakpointsAndPreservesRequestOrder() { + CacheItem system = new CacheItem("system", false, true, true); + CacheItem explicitUser = new CacheItem("user", true, false, true); + CacheItem assistant = new CacheItem("assistant", false, false, true); + CacheItem explicitTool = new CacheItem("tool", true, false, true); + + List selected = + formatter.select(List.of(system, explicitUser, assistant, explicitTool), true); + + assertEquals(List.of(system, explicitUser, explicitTool), selected); + } + + @Test + void explicitBreakpointsConsumeCapacityBeforeAutomaticBreakpoints() { + CacheItem system = new CacheItem("system", false, true, true); + CacheItem explicitOne = new CacheItem("one", true, false, true); + CacheItem explicitTwo = new CacheItem("two", true, false, true); + CacheItem explicitThree = new CacheItem("three", true, false, true); + CacheItem last = new CacheItem("last", false, false, true); + + List selected = + formatter.select( + List.of(system, explicitOne, explicitTwo, explicitThree, last), true); + + assertEquals(List.of(system, explicitOne, explicitTwo, explicitThree), selected); + } + + @Test + void disablingAutomaticSelectionKeepsOnlyExplicitBreakpoints() { + CacheItem system = new CacheItem("system", false, true, true); + CacheItem explicit = new CacheItem("explicit", true, false, true); + CacheItem last = new CacheItem("last", false, false, true); + + assertEquals(List.of(explicit), formatter.select(List.of(system, explicit, last), false)); + } + + @Test + void rejectsMoreThanFourExplicitBreakpoints() { + List items = + List.of( + new CacheItem("one", true, false, true), + new CacheItem("two", true, false, true), + new CacheItem("three", true, false, true), + new CacheItem("four", true, false, true), + new CacheItem("five", true, false, true)); + + IllegalArgumentException error = + assertThrows(IllegalArgumentException.class, () -> formatter.select(items, true)); + + assertTrue(error.getMessage().contains("at most 4 explicit breakpoints")); + } + + @Test + void returnsEmptySelectionForNullOrEmptyInput() { + assertTrue(formatter.select(null, true).isEmpty()); + assertTrue(formatter.select(List.of(), true).isEmpty()); + } + + @Test + void explicitCacheControlPreservesMultiAgentMessageBoundary() { + Msg cached = + Msg.builder() + .name("user") + .role(MsgRole.USER) + .textContent("cache me") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + Msg notCached = + Msg.builder() + .name("user") + .role(MsgRole.USER) + .textContent("merge me") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, false)) + .build(); + + assertTrue(formatter.bypassesHistory(cached)); + assertFalse(formatter.bypassesHistory(notCached)); + } + + private record CacheItem(String name, boolean explicit, boolean system, boolean cacheable) {} + + private static final class TestFormatter extends AbstractBaseFormatter { + + List select(List items, boolean automatic) { + return selectPromptCacheBreakpoints( + items, automatic, CacheItem::explicit, CacheItem::system, CacheItem::cacheable); + } + + boolean bypassesHistory(Msg msg) { + return shouldBypassHistory(msg); + } + + @Override + protected List doFormat(List msgs) { + return List.of(); + } + + @Override + public ChatResponse parseResponse(String response, Instant startTime) { + return null; + } + + @Override + public void applyOptions( + String paramsBuilder, GenerateOptions options, GenerateOptions defaultOptions) {} + + @Override + public void applyTools(String paramsBuilder, List tools) {} + } +} diff --git a/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java b/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java index 1fafc0233e..4ed34b52cd 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java @@ -50,7 +50,12 @@ void usageFieldRoundTripsViaJson() { @Test void cachedTokensRoundTripViaJson() { ChatUsage usage = - ChatUsage.builder().inputTokens(100).outputTokens(50).cachedTokens(30).build(); + ChatUsage.builder() + .inputTokens(100) + .outputTokens(50) + .cachedTokens(30) + .cacheCreationInputTokens(20) + .build(); Msg msg = Msg.builder() .name("assistant") @@ -68,12 +73,34 @@ void cachedTokensRoundTripViaJson() { assertEquals(100, deserialized.getUsage().getInputTokens()); assertEquals(50, deserialized.getUsage().getOutputTokens()); assertEquals(30, deserialized.getUsage().getCachedTokens()); + assertEquals(20, deserialized.getUsage().getCacheCreationInputTokens()); } @Test void cachedTokensDefaultsToZeroForLegacyConstructor() { ChatUsage usage = new ChatUsage(100, 50, 1.5); assertEquals(0, usage.getCachedTokens()); + assertEquals(0, usage.getCacheCreationInputTokens()); + } + + @Test + void cacheCreationTokensDefaultToZeroForFourArgumentConstructor() { + ChatUsage usage = new ChatUsage(100, 50, 30, 1.5); + + assertEquals(30, usage.getCachedTokens()); + assertEquals(0, usage.getCacheCreationInputTokens()); + } + + @Test + void legacyUsageJsonDefaultsCacheCreationTokensToZero() { + String json = "{\"inputTokens\":100,\"outputTokens\":50,\"cachedTokens\":30,\"time\":1.5}"; + + ChatUsage usage = JsonUtils.getJsonCodec().fromJson(json, ChatUsage.class); + + assertEquals(100, usage.getInputTokens()); + assertEquals(50, usage.getOutputTokens()); + assertEquals(30, usage.getCachedTokens()); + assertEquals(0, usage.getCacheCreationInputTokens()); } @Test diff --git a/agentscope-core/src/test/java/io/agentscope/core/model/GenerateOptionsTest.java b/agentscope-core/src/test/java/io/agentscope/core/model/GenerateOptionsTest.java index 9db349d158..01d44c5166 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/model/GenerateOptionsTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/model/GenerateOptionsTest.java @@ -356,6 +356,28 @@ void testMergeOptionsWithAdditionalParams() { assertEquals("fallback", merged.getAdditionalBodyParams().get("fallback_param")); } + @Test + @DisplayName("Should prefer primary cache control when merging options") + void testMergeOptionsPrefersPrimaryCacheControl() { + GenerateOptions primary = GenerateOptions.builder().cacheControl(false).build(); + GenerateOptions fallback = GenerateOptions.builder().cacheControl(true).build(); + + GenerateOptions merged = GenerateOptions.mergeOptions(primary, fallback); + + assertEquals(Boolean.FALSE, merged.getCacheControl()); + } + + @Test + @DisplayName("Should inherit fallback cache control when primary is unset") + void testMergeOptionsInheritsFallbackCacheControl() { + GenerateOptions primary = GenerateOptions.builder().temperature(0.8).build(); + GenerateOptions fallback = GenerateOptions.builder().cacheControl(true).build(); + + GenerateOptions merged = GenerateOptions.mergeOptions(primary, fallback); + + assertEquals(Boolean.TRUE, merged.getCacheControl()); + } + @Test @DisplayName("Should set additional headers using map") void testSetAdditionalHeadersMap() { From 38c13e040f24a054a72132f593aeb9662a44fcc2 Mon Sep 17 00:00:00 2001 From: guslegend <1670547022@qq.com> Date: Tue, 25 Aug 2026 13:45:15 +0800 Subject: [PATCH 02/10] fix(model): correct DashScope prompt cache protocol --- .../model/dashscope/DashScopeChatModel.java | 13 +- .../dashscope/dto/DashScopeContentPart.java | 35 ++ .../dashscope/dto/DashScopeTokenDetails.java | 46 +++ .../model/dashscope/dto/DashScopeUsage.java | 64 ++++ .../formatter/DashScopeChatFormatter.java | 125 +++++++- .../formatter/DashScopeMessageConverter.java | 2 +- .../DashScopeMultiAgentFormatter.java | 78 +++-- .../formatter/DashScopeResponseParser.java | 5 + .../dashscope/DashScopeChatModelTest.java | 56 +++- .../dto/DashScopeDtoSerializationTest.java | 24 ++ .../formatter/DashScopeCacheControlTest.java | 300 ++++++++---------- .../DashScopeResponseParserTest.java | 43 +++ .../model/openai/OpenAIChatModel.java | 36 ++- .../model/openai/dto/OpenAIContentPart.java | 18 ++ .../openai/formatter/OpenAIBaseFormatter.java | 106 +++++++ .../model/openai/OpenAIChatModelTest.java | 20 ++ .../formatter/OpenAICacheControlTest.java | 112 +++++++ 17 files changed, 859 insertions(+), 224 deletions(-) create mode 100644 agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/DashScopeChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/DashScopeChatModel.java index cb335dbf47..3a4b823b1a 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/DashScopeChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/DashScopeChatModel.java @@ -280,13 +280,12 @@ private Flux streamWithHttpClient( // Apply thinking mode if enabled applyThinkingMode(request, effectiveOptions); - // Apply cache control if enabled (adds cache_control to system msgs + last msg) - if (Boolean.TRUE.equals(effectiveOptions.getCacheControl())) { - if (formatter instanceof DashScopeChatFormatter chatFmt) { - chatFmt.applyCacheControl(request.getInput().getMessages()); - } else if (formatter instanceof DashScopeMultiAgentFormatter multiFmt) { - multiFmt.applyCacheControl(request.getInput().getMessages()); - } + // Always normalize explicit markers; automatic breakpoints remain opt-in. + boolean automaticCacheControl = Boolean.TRUE.equals(effectiveOptions.getCacheControl()); + if (formatter instanceof DashScopeChatFormatter chatFmt) { + chatFmt.applyCacheControl(request.getInput().getMessages(), automaticCacheControl); + } else if (formatter instanceof DashScopeMultiAgentFormatter multiFmt) { + multiFmt.applyCacheControl(request.getInput().getMessages(), automaticCacheControl); } // Set endpoint type for endpoint selection diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeContentPart.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeContentPart.java index ba7d3be21f..5abf031054 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeContentPart.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeContentPart.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; +import java.util.Map; /** * DashScope content part DTO for multimodal messages. @@ -43,6 +44,10 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class DashScopeContentPart { + /** Content type. Explicitly cached text blocks use {@code text}. */ + @JsonProperty("type") + private String type; + /** Text content. */ @JsonProperty("text") private String text; @@ -79,8 +84,20 @@ public class DashScopeContentPart { @JsonProperty("total_pixels") private Integer totalPixels; + /** Prompt cache control marker. */ + @JsonProperty("cache_control") + private Map cacheControl; + public DashScopeContentPart() {} + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + public String getText() { return text; } @@ -153,6 +170,14 @@ public void setTotalPixels(Integer totalPixels) { this.totalPixels = totalPixels; } + public Map getCacheControl() { + return cacheControl; + } + + public void setCacheControl(Map cacheControl) { + this.cacheControl = cacheControl; + } + /** * Get video as URL string. * @@ -245,6 +270,11 @@ public static Builder builder() { public static class Builder { private final DashScopeContentPart part = new DashScopeContentPart(); + public Builder type(String type) { + part.setType(type); + return this; + } + public Builder text(String text) { part.setText(text); return this; @@ -290,6 +320,11 @@ public Builder totalPixels(Integer totalPixels) { return this; } + public Builder cacheControl(Map cacheControl) { + part.setCacheControl(cacheControl); + return this; + } + public DashScopeContentPart build() { return part; } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java new file mode 100644 index 0000000000..90dcac4240 --- /dev/null +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java @@ -0,0 +1,46 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.extensions.model.dashscope.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Fine-grained DashScope input token usage details. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DashScopeTokenDetails { + + @JsonProperty("cached_tokens") + private Integer cachedTokens; + + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + public Integer getCachedTokens() { + return cachedTokens; + } + + public void setCachedTokens(Integer cachedTokens) { + this.cachedTokens = cachedTokens; + } + + public Integer getCacheCreationInputTokens() { + return cacheCreationInputTokens; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } +} diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeUsage.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeUsage.java index d80eec1877..0c4fb9c07b 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeUsage.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeUsage.java @@ -59,6 +59,22 @@ public class DashScopeUsage { @JsonProperty("audio_tokens") private Integer audioTokens; + /** Legacy regional field for cache hits. */ + @JsonProperty("cached_tokens") + private Integer cachedTokens; + + /** Legacy/top-level field for explicit cache creation. */ + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + /** Current DashScope prompt token details. */ + @JsonProperty("prompt_tokens_details") + private DashScopeTokenDetails promptTokensDetails; + + /** Alternate token detail shape returned by some endpoints. */ + @JsonProperty("input_tokens_details") + private DashScopeTokenDetails inputTokensDetails; + public DashScopeUsage() {} public Integer getInputTokens() { @@ -108,4 +124,52 @@ public Integer getAudioTokens() { public void setAudioTokens(Integer audioTokens) { this.audioTokens = audioTokens; } + + /** Returns cache hits across current and legacy DashScope response shapes. */ + public Integer getCachedTokens() { + if (promptTokensDetails != null && promptTokensDetails.getCachedTokens() != null) { + return promptTokensDetails.getCachedTokens(); + } + if (inputTokensDetails != null && inputTokensDetails.getCachedTokens() != null) { + return inputTokensDetails.getCachedTokens(); + } + return cachedTokens; + } + + public void setCachedTokens(Integer cachedTokens) { + this.cachedTokens = cachedTokens; + } + + /** Returns explicit cache creation tokens across current and legacy response shapes. */ + public Integer getCacheCreationInputTokens() { + if (promptTokensDetails != null + && promptTokensDetails.getCacheCreationInputTokens() != null) { + return promptTokensDetails.getCacheCreationInputTokens(); + } + if (inputTokensDetails != null + && inputTokensDetails.getCacheCreationInputTokens() != null) { + return inputTokensDetails.getCacheCreationInputTokens(); + } + return cacheCreationInputTokens; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } + + public DashScopeTokenDetails getPromptTokensDetails() { + return promptTokensDetails; + } + + public void setPromptTokensDetails(DashScopeTokenDetails promptTokensDetails) { + this.promptTokensDetails = promptTokensDetails; + } + + public DashScopeTokenDetails getInputTokensDetails() { + return inputTokensDetails; + } + + public void setInputTokensDetails(DashScopeTokenDetails inputTokensDetails) { + this.inputTokensDetails = inputTokensDetails; + } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeChatFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeChatFormatter.java index 587deaf74a..6a5b800b33 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeChatFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeChatFormatter.java @@ -21,6 +21,7 @@ import io.agentscope.core.model.GenerateOptions; import io.agentscope.core.model.ToolChoice; import io.agentscope.core.model.ToolSchema; +import io.agentscope.extensions.model.dashscope.dto.DashScopeContentPart; import io.agentscope.extensions.model.dashscope.dto.DashScopeInput; import io.agentscope.extensions.model.dashscope.dto.DashScopeMessage; import io.agentscope.extensions.model.dashscope.dto.DashScopeParameters; @@ -172,28 +173,126 @@ public DashScopeRequest buildRequest( return request; } + /** Apply automatic prompt cache control to DashScope messages. */ + public void applyCacheControl(List messages) { + applyCacheControl(messages, true); + } + /** - * Apply cache control to DashScope messages. + * Normalize explicit prompt cache markers and optionally apply the automatic strategy. * - *

Adds cache_control: {"type": "ephemeral"} to all system messages and the last - * message in the list. Messages that already have cache_control set (e.g., via manual metadata - * marking) will not be overwritten. + *

DashScope requires {@code cache_control} inside a content block. Legacy message-level + * markers are migrated to the last content block and removed from the message before + * serialization. * - * @param messages the list of formatted DashScope messages + * @param messages formatted DashScope messages + * @param automatic whether to add automatic breakpoints */ - public void applyCacheControl(List messages) { + public void applyCacheControl(List messages, boolean automatic) { if (messages == null || messages.isEmpty()) { return; } - for (DashScopeMessage msg : messages) { - if ("system".equals(msg.getRole()) && msg.getCacheControl() == null) { - msg.setCacheControl(EPHEMERAL_CACHE_CONTROL); - } + + List selected = + selectPromptCacheBreakpoints( + messages, + automatic, + DashScopeChatFormatter::hasExplicitCacheControl, + message -> "system".equals(message.getRole()), + DashScopeChatFormatter::isCacheable); + + for (DashScopeMessage message : selected) { + applyCacheControlToContentBlock(message); + } + for (DashScopeMessage message : messages) { + message.setCacheControl(null); + } + + int markerCount = countCacheControlMarkers(messages); + if (markerCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "DashScope supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " cache_control markers, but got " + + markerCount); + } + } + + static void applyCacheControlToContentBlock(DashScopeMessage message) { + if (hasContentBlockCacheControl(message)) { + message.setCacheControl(null); + return; + } + + List parts = ensureContentArray(message); + if (parts.isEmpty()) { + throw new IllegalArgumentException( + "Cannot apply cache_control to a message without cacheable content"); } - DashScopeMessage lastMsg = messages.get(messages.size() - 1); - if (lastMsg.getCacheControl() == null) { - lastMsg.setCacheControl(EPHEMERAL_CACHE_CONTROL); + + DashScopeContentPart lastPart = parts.get(parts.size() - 1); + if (lastPart.getType() == null && lastPart.getText() != null) { + lastPart.setType("text"); + } + Map cacheControl = + message.getCacheControl() != null + ? message.getCacheControl() + : EPHEMERAL_CACHE_CONTROL; + lastPart.setCacheControl(cacheControl); + message.setCacheControl(null); + } + + @SuppressWarnings("unchecked") + static List ensureContentArray(DashScopeMessage message) { + Object content = message.getContent(); + if (content instanceof List) { + return (List) content; + } + if (content instanceof String text) { + List parts = + new ArrayList<>( + List.of( + DashScopeContentPart.builder() + .type("text") + .text(text) + .build())); + message.setContent(parts); + return parts; + } + return List.of(); + } + + static boolean isCacheable(DashScopeMessage message) { + if (!("system".equals(message.getRole()) + || "user".equals(message.getRole()) + || "assistant".equals(message.getRole()) + || "tool".equals(message.getRole()))) { + return false; + } + Object content = message.getContent(); + return (content instanceof String text && !text.isEmpty()) + || (content instanceof List parts && !parts.isEmpty()); + } + + static boolean hasExplicitCacheControl(DashScopeMessage message) { + return message.getCacheControl() != null || hasContentBlockCacheControl(message); + } + + static boolean hasContentBlockCacheControl(DashScopeMessage message) { + List parts = message.getContentAsList(); + return parts != null && parts.stream().anyMatch(part -> part.getCacheControl() != null); + } + + static int countCacheControlMarkers(List messages) { + int count = 0; + for (DashScopeMessage message : messages) { + List parts = message.getContentAsList(); + if (parts != null) { + count += + (int) parts.stream().filter(part -> part.getCacheControl() != null).count(); + } } + return count; } /** diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMessageConverter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMessageConverter.java index f8818ed9e6..8176487a0b 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMessageConverter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMessageConverter.java @@ -279,7 +279,7 @@ private void applyCacheControlFromMetadata(Msg msg, DashScopeMessage result) { } Object cacheFlag = msg.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL); if (Boolean.TRUE.equals(cacheFlag)) { - result.setCacheControl(DashScopeChatFormatter.getEphemeralCacheControl()); + DashScopeChatFormatter.applyCacheControlToContentBlock(result); } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMultiAgentFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMultiAgentFormatter.java index 61a4ec14a8..fa92474f3b 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMultiAgentFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMultiAgentFormatter.java @@ -24,7 +24,6 @@ import io.agentscope.core.model.GenerateOptions; import io.agentscope.core.model.ToolChoice; import io.agentscope.core.model.ToolSchema; -import io.agentscope.extensions.model.dashscope.dto.DashScopeContentPart; import io.agentscope.extensions.model.dashscope.dto.DashScopeInput; import io.agentscope.extensions.model.dashscope.dto.DashScopeMessage; import io.agentscope.extensions.model.dashscope.dto.DashScopeParameters; @@ -33,7 +32,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.Map; /** * DashScope formatter for multi-agent conversations. @@ -84,11 +82,7 @@ protected List doFormat(List msgs) { // Process system message first (if any) - output separately if (!msgs.isEmpty() && msgs.get(0).getRole() == MsgRole.SYSTEM) { - result.add( - DashScopeMessage.builder() - .role("system") - .content(extractTextContent(msgs.get(0))) - .build()); + result.add(messageConverter.convertToMessage(msgs.get(0), false)); startIndex = 1; } @@ -111,6 +105,8 @@ protected List doFormat(List msgs) { } else if (group.type == GroupType.TOOL_SEQUENCE) { // Format tool sequence directly result.addAll(formatToolSeq(group.messages)); + } else if (group.type == GroupType.BYPASS) { + result.add(messageConverter.convertToMessage(group.messages.get(0), false)); } } @@ -171,14 +167,7 @@ public List formatMultiModal(List msgs) { // Process system message first (if any) if (!msgs.isEmpty() && msgs.get(0).getRole() == MsgRole.SYSTEM) { - result.add( - DashScopeMessage.builder() - .role("system") - .content( - List.of( - DashScopeContentPart.text( - extractTextContent(msgs.get(0))))) - .build()); + result.add(messageConverter.convertToMessage(msgs.get(0), true)); startIndex = 1; } @@ -200,6 +189,8 @@ public List formatMultiModal(List msgs) { } else if (group.type == GroupType.TOOL_SEQUENCE) { // Format tool sequence directly result.addAll(formatMultiModalToolSeq(group.messages)); + } else if (group.type == GroupType.BYPASS) { + result.add(messageConverter.convertToMessage(group.messages.get(0), true)); } } @@ -244,6 +235,16 @@ private List groupMessagesSequentially(List msgs) { List currentGroup = new ArrayList<>(); for (Msg msg : msgs) { + if (shouldBypassHistory(msg)) { + if (!currentGroup.isEmpty()) { + result.add(new MessageGroup(currentType, new ArrayList<>(currentGroup))); + currentGroup.clear(); + } + result.add(new MessageGroup(GroupType.BYPASS, List.of(msg))); + currentType = null; + continue; + } + boolean isToolRelated = msg.getRole() == MsgRole.TOOL || msg.hasContentBlocks(ToolUseBlock.class) @@ -347,7 +348,8 @@ private List formatMultiModalToolSeq(List msgs) { */ private enum GroupType { AGENT_MESSAGE, - TOOL_SEQUENCE + TOOL_SEQUENCE, + BYPASS } /** @@ -363,28 +365,38 @@ private static class MessageGroup { } } - /** - * Apply cache control to DashScope messages. - * - *

Adds cache_control: {"type": "ephemeral"} to all system messages and the last - * message in the list. Messages that already have cache_control set (e.g., via manual metadata - * marking) will not be overwritten. - * - * @param messages the list of formatted DashScope messages - */ + /** Apply automatic prompt cache control to DashScope messages. */ public void applyCacheControl(List messages) { + applyCacheControl(messages, true); + } + + /** Normalize explicit cache markers and optionally apply automatic cache breakpoints. */ + public void applyCacheControl(List messages, boolean automatic) { if (messages == null || messages.isEmpty()) { return; } - Map ephemeral = DashScopeChatFormatter.getEphemeralCacheControl(); - for (DashScopeMessage msg : messages) { - if ("system".equals(msg.getRole()) && msg.getCacheControl() == null) { - msg.setCacheControl(ephemeral); - } + + List selected = + selectPromptCacheBreakpoints( + messages, + automatic, + DashScopeChatFormatter::hasExplicitCacheControl, + message -> "system".equals(message.getRole()), + DashScopeChatFormatter::isCacheable); + for (DashScopeMessage message : selected) { + DashScopeChatFormatter.applyCacheControlToContentBlock(message); + } + for (DashScopeMessage message : messages) { + message.setCacheControl(null); } - DashScopeMessage lastMsg = messages.get(messages.size() - 1); - if (lastMsg.getCacheControl() == null) { - lastMsg.setCacheControl(ephemeral); + + int markerCount = DashScopeChatFormatter.countCacheControlMarkers(messages); + if (markerCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "DashScope supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " cache_control markers, but got " + + markerCount); } } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParser.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParser.java index b882d34ea7..f4f8cd7bcb 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParser.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParser.java @@ -101,6 +101,11 @@ public ChatResponse parseResponse(DashScopeResponse response, Instant startTime) ChatUsage.builder() .inputTokens(u.getInputTokens() != null ? u.getInputTokens() : 0) .outputTokens(u.getOutputTokens() != null ? u.getOutputTokens() : 0) + .cachedTokens(u.getCachedTokens() != null ? u.getCachedTokens() : 0) + .cacheCreationInputTokens( + u.getCacheCreationInputTokens() != null + ? u.getCacheCreationInputTokens() + : 0) .time( Duration.between(startTime, Instant.now()).toMillis() / 1000.0) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/DashScopeChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/DashScopeChatModelTest.java index 1dbac5c29b..5a85403ab0 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/DashScopeChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/DashScopeChatModelTest.java @@ -24,6 +24,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.fasterxml.jackson.databind.JsonNode; +import io.agentscope.core.message.MessageMetadataKeys; import io.agentscope.core.message.Msg; import io.agentscope.core.message.MsgRole; import io.agentscope.core.message.TextBlock; @@ -34,6 +36,7 @@ import io.agentscope.core.model.test.ModelTestUtils; import io.agentscope.core.model.transport.OkHttpTransport; import io.agentscope.core.model.transport.ProxyConfig; +import io.agentscope.core.util.JsonUtils; import io.agentscope.extensions.model.dashscope.dto.DashScopeParameters; import io.agentscope.extensions.model.dashscope.dto.DashScopeRequest; import io.agentscope.extensions.model.dashscope.dto.DashScopeSearchOptions; @@ -975,12 +978,53 @@ void testCacheControlApplied() throws Exception { RecordedRequest recorded = mockServer.takeRequest(); String body = recorded.getBody().readUtf8(); - assertTrue( - body.contains("\"cache_control\""), - "Request body should contain cache_control: " + body); - assertTrue( - body.contains("\"ephemeral\""), - "Request body should contain ephemeral cache type: " + body); + JsonNode tree = JsonUtils.getJsonCodec().fromJson(body, JsonNode.class); + JsonNode system = tree.at("/input/messages/0"); + JsonNode user = tree.at("/input/messages/1"); + assertFalse(system.has("cache_control"), body); + assertFalse(user.has("cache_control"), body); + assertTrue(system.get("content").isArray(), body); + assertTrue(user.get("content").isArray(), body); + assertEquals("text", system.at("/content/0/type").asText()); + assertEquals("ephemeral", system.at("/content/0/cache_control/type").asText()); + assertEquals("ephemeral", user.at("/content/0/cache_control/type").asText()); + + mockServer.shutdown(); + } + + @Test + @DisplayName("Should preserve explicit cache metadata when automatic cache control is disabled") + void testExplicitCacheControlAppliedWithoutAutomaticOption() throws Exception { + MockWebServer mockServer = new MockWebServer(); + mockServer.start(); + mockServer.enqueue( + new MockResponse() + .setResponseCode(200) + .setBody("{\"request_id\":\"test\",\"output\":{\"choices\":[]}}") + .setHeader("Content-Type", "application/json")); + + DashScopeChatModel chatModel = + DashScopeChatModel.builder().apiKey(mockApiKey).modelName("qwen-plus").stream(false) + .baseUrl(mockServer.url("/").toString().replaceAll("/$", "")) + .httpTransport(OkHttpTransport.builder().build()) + .build(); + Msg explicit = + Msg.builder() + .role(MsgRole.USER) + .textContent("stable context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + + chatModel + .doStream(List.of(explicit), List.of(), GenerateOptions.builder().build()) + .blockLast(); + + JsonNode tree = + JsonUtils.getJsonCodec() + .fromJson(mockServer.takeRequest().getBody().readUtf8(), JsonNode.class); + JsonNode message = tree.at("/input/messages/0"); + assertFalse(message.has("cache_control")); + assertEquals("ephemeral", message.at("/content/0/cache_control/type").asText()); mockServer.shutdown(); } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java index 50360e7183..f133d3a73f 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java @@ -238,6 +238,30 @@ void testDashScopeResponseDeserialization() throws Exception { assertEquals(8, usage.getOutputTokens()); } + @Test + void testDashScopeCacheUsageDeserialization() { + String json = + """ + { + "request_id": "req-cache", + "usage": { + "input_tokens": 2048, + "output_tokens": 128, + "prompt_tokens_details": { + "cached_tokens": 1536, + "cache_creation_input_tokens": 256 + } + } + } + """; + + DashScopeResponse response = jsonCodec.fromJson(json, DashScopeResponse.class); + + assertNotNull(response.getUsage()); + assertEquals(1536, response.getUsage().getCachedTokens()); + assertEquals(256, response.getUsage().getCacheCreationInputTokens()); + } + @Test void testDashScopeResponseWithToolCalls() throws Exception { String json = diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeCacheControlTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeCacheControlTest.java index 96b3257818..e0e155221a 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeCacheControlTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeCacheControlTest.java @@ -16,14 +16,20 @@ package io.agentscope.extensions.model.dashscope.formatter; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import com.fasterxml.jackson.databind.JsonNode; import io.agentscope.core.message.MessageMetadataKeys; import io.agentscope.core.message.Msg; import io.agentscope.core.message.MsgRole; +import io.agentscope.core.util.JsonUtils; +import io.agentscope.extensions.model.dashscope.dto.DashScopeContentPart; import io.agentscope.extensions.model.dashscope.dto.DashScopeMessage; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.jupiter.api.BeforeEach; @@ -31,9 +37,7 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -/** - * Tests for cache_control support in DashScope formatter. - */ +/** Tests DashScope content-block prompt cache markers. */ class DashScopeCacheControlTest { private static final Map EPHEMERAL = Map.of("type", "ephemeral"); @@ -46,217 +50,193 @@ void setUp() { } @Nested - @DisplayName("applyCacheControl - automatic strategy") - class ApplyCacheControlTest { + @DisplayName("automatic cache strategy") + class AutomaticCacheStrategyTest { @Test - @DisplayName("should add cache_control to system and last message") - void systemAndLastMessage() { - List messages = new ArrayList<>(); - messages.add( - DashScopeMessage.builder().role("system").content("You are helpful.").build()); - messages.add(DashScopeMessage.builder().role("user").content("Hello").build()); - messages.add(DashScopeMessage.builder().role("assistant").content("Hi").build()); - messages.add(DashScopeMessage.builder().role("user").content("Question").build()); - - formatter.applyCacheControl(messages); - - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - assertNull(messages.get(1).getCacheControl()); - assertNull(messages.get(2).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(3).getCacheControl()); + void marksFirstSystemAndLastCacheableNonSystemMessage() { + List messages = + new ArrayList<>( + List.of( + message("system", "stable system"), + message("system", "dynamic system"), + message("user", "question"), + message("assistant", "answer"))); + + formatter.applyCacheControl(messages, true); + + assertEquals(EPHEMERAL, lastPart(messages.get(0)).getCacheControl()); + assertNoCacheControl(messages.get(1)); + assertNoCacheControl(messages.get(2)); + assertEquals(EPHEMERAL, lastPart(messages.get(3)).getCacheControl()); + messages.forEach(message -> assertNull(message.getCacheControl())); } @Test - @DisplayName("should handle no system message - only last message") - void noSystemMessage() { - List messages = new ArrayList<>(); - messages.add(DashScopeMessage.builder().role("user").content("Hello").build()); - messages.add(DashScopeMessage.builder().role("assistant").content("Hi").build()); + void automaticFalseKeepsOnlyExplicitMarkers() { + DashScopeMessage system = message("system", "system"); + DashScopeMessage explicit = + DashScopeMessage.builder() + .role("user") + .content("explicit") + .cacheControl(Map.of("type", "custom")) + .build(); + DashScopeMessage last = message("assistant", "last"); - formatter.applyCacheControl(messages); + formatter.applyCacheControl(new ArrayList<>(List.of(system, explicit, last)), false); - assertNull(messages.get(0).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(1).getCacheControl()); + assertNoCacheControl(system); + assertEquals(Map.of("type", "custom"), lastPart(explicit).getCacheControl()); + assertNull(explicit.getCacheControl()); + assertNoCacheControl(last); } @Test - @DisplayName("should handle empty list without error") - void emptyList() { + void rejectsMoreThanFourExplicitMarkers() { List messages = new ArrayList<>(); - formatter.applyCacheControl(messages); - // No exception thrown + for (int i = 0; i < 5; i++) { + DashScopeContentPart part = + DashScopeContentPart.builder() + .type("text") + .text("part-" + i) + .cacheControl(EPHEMERAL) + .build(); + messages.add( + DashScopeMessage.builder() + .role("user") + .content(new ArrayList<>(List.of(part))) + .build()); + } + + IllegalArgumentException error = + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyCacheControl(messages, false)); + + assertTrue(error.getMessage().contains("at most 4")); } @Test - @DisplayName("should handle null list without error") - void nullList() { - formatter.applyCacheControl(null); - // No exception thrown + void handlesNullAndEmptyLists() { + formatter.applyCacheControl(null, true); + formatter.applyCacheControl(List.of(), true); } + } - @Test - @DisplayName("should handle single system message (both system and last)") - void singleSystemMessage() { - List messages = new ArrayList<>(); - messages.add( - DashScopeMessage.builder().role("system").content("You are helpful.").build()); - - formatter.applyCacheControl(messages); - - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - } + @Nested + @DisplayName("wire format") + class WireFormatTest { @Test - @DisplayName("should not overwrite manually marked cache_control") - void manuallyMarkedNotOverridden() { - Map customCacheControl = Map.of("type", "custom"); - - List messages = new ArrayList<>(); - messages.add( - DashScopeMessage.builder() - .role("system") - .content("System") - .cacheControl(customCacheControl) - .build()); - messages.add(DashScopeMessage.builder().role("user").content("User").build()); - - formatter.applyCacheControl(messages); - - // System message keeps its custom cache_control - assertEquals(customCacheControl, messages.get(0).getCacheControl()); - // Last message gets ephemeral - assertEquals(EPHEMERAL, messages.get(1).getCacheControl()); + void serializesMarkerInsideTextContentBlockOnly() { + List messages = + new ArrayList<>(List.of(message("system", "You are helpful"))); + + formatter.applyCacheControl(messages, true); + + JsonNode tree = + JsonUtils.getJsonCodec() + .fromJson( + JsonUtils.getJsonCodec().toJson(messages.get(0)), + JsonNode.class); + assertFalse(tree.has("cache_control")); + assertTrue(tree.get("content").isArray()); + assertEquals("text", tree.at("/content/0/type").asText()); + assertEquals("You are helpful", tree.at("/content/0/text").asText()); + assertEquals("ephemeral", tree.at("/content/0/cache_control/type").asText()); } @Test - @DisplayName("should not overwrite last message with existing cache_control") - void lastMessageManuallyMarkedNotOverridden() { - Map customCacheControl = Map.of("type", "custom"); - - List messages = new ArrayList<>(); - messages.add(DashScopeMessage.builder().role("system").content("System").build()); - messages.add( + void marksLastExistingMultimodalContentPart() { + DashScopeContentPart text = DashScopeContentPart.text("describe"); + DashScopeContentPart image = DashScopeContentPart.image("https://example.com/a.png"); + DashScopeMessage message = DashScopeMessage.builder() .role("user") - .content("User") - .cacheControl(customCacheControl) - .build()); - - formatter.applyCacheControl(messages); - - // System message gets ephemeral - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - // Last message keeps its custom cache_control - assertEquals(customCacheControl, messages.get(1).getCacheControl()); - } - - @Test - @DisplayName("should handle multiple system messages") - void multipleSystemMessages() { - List messages = new ArrayList<>(); - messages.add(DashScopeMessage.builder().role("system").content("System 1").build()); - messages.add(DashScopeMessage.builder().role("system").content("System 2").build()); - messages.add(DashScopeMessage.builder().role("user").content("User").build()); + .content(new ArrayList<>(List.of(text, image))) + .build(); - formatter.applyCacheControl(messages); + formatter.applyCacheControl(new ArrayList<>(List.of(message)), true); - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(1).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(2).getCacheControl()); + assertNull(text.getCacheControl()); + assertEquals(EPHEMERAL, image.getCacheControl()); + assertNull(message.getCacheControl()); } } @Nested - @DisplayName("metadata-based cache_control marking") - class MetadataMarkingTest { + @DisplayName("metadata and multi-agent formatting") + class MetadataAndMultiAgentTest { @Test - @DisplayName("should set cache_control from Msg metadata") - void metadataMarking() { - Map metadata = new HashMap<>(); - metadata.put(MessageMetadataKeys.CACHE_CONTROL, true); + void metadataMarkerIsPlacedOnContentBlockWithoutAutomaticOption() { Msg msg = Msg.builder() .role(MsgRole.USER) - .textContent("Important context") - .metadata(metadata) + .textContent("cache this") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) .build(); - List result = formatter.format(List.of(msg)); - - assertEquals(1, result.size()); - assertEquals(EPHEMERAL, result.get(0).getCacheControl()); - } - - @Test - @DisplayName("should not set cache_control when metadata flag is absent") - void noMetadata() { - Msg msg = Msg.builder().role(MsgRole.USER).textContent("Hello").build(); - - List result = formatter.format(List.of(msg)); + DashScopeMessage formatted = formatter.format(List.of(msg)).get(0); - assertEquals(1, result.size()); - assertNull(result.get(0).getCacheControl()); + assertNull(formatted.getCacheControl()); + assertEquals(EPHEMERAL, lastPart(formatted).getCacheControl()); } @Test - @DisplayName("should not set cache_control when metadata flag is false") - void metadataFalse() { - Map metadata = new HashMap<>(); - metadata.put(MessageMetadataKeys.CACHE_CONTROL, false); - Msg msg = + void explicitMetadataPreventsMultiAgentHistoryMerge() { + DashScopeMultiAgentFormatter multiFormatter = new DashScopeMultiAgentFormatter(); + Msg cached = + Msg.builder() + .name("agent-a") + .role(MsgRole.USER) + .textContent("stable context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + Msg following = Msg.builder() + .name("agent-b") .role(MsgRole.USER) - .textContent("Hello") - .metadata(metadata) + .textContent("dynamic question") .build(); - List result = formatter.format(List.of(msg)); + List formatted = multiFormatter.format(List.of(cached, following)); - assertEquals(1, result.size()); - assertNull(result.get(0).getCacheControl()); + assertEquals(2, formatted.size()); + assertEquals(EPHEMERAL, lastPart(formatted.get(0)).getCacheControl()); + assertNoCacheControl(formatted.get(1)); } @Test - @DisplayName("should set cache_control on system message via metadata") - void systemMessageMetadata() { - Map metadata = new HashMap<>(); - metadata.put(MessageMetadataKeys.CACHE_CONTROL, true); - Msg systemMsg = - Msg.builder() - .role(MsgRole.SYSTEM) - .textContent("System prompt") - .metadata(metadata) - .build(); - Msg userMsg = Msg.builder().role(MsgRole.USER).textContent("User msg").build(); + void multiAgentAutomaticStrategyUsesContentBlocks() { + DashScopeMultiAgentFormatter multiFormatter = new DashScopeMultiAgentFormatter(); + List messages = + new ArrayList<>(List.of(message("system", "system"), message("user", "user"))); - List result = formatter.format(List.of(systemMsg, userMsg)); + multiFormatter.applyCacheControl(messages, true); - assertEquals(2, result.size()); - assertEquals(EPHEMERAL, result.get(0).getCacheControl()); - assertNull(result.get(1).getCacheControl()); + assertEquals(EPHEMERAL, lastPart(messages.get(0)).getCacheControl()); + assertEquals(EPHEMERAL, lastPart(messages.get(1)).getCacheControl()); + messages.forEach(message -> assertNull(message.getCacheControl())); } } - @Nested - @DisplayName("DashScopeMultiAgentFormatter cache_control") - class MultiAgentFormatterTest { - - @Test - @DisplayName("should add cache_control to system and last message") - void applyCacheControl() { - DashScopeMultiAgentFormatter multiFormatter = new DashScopeMultiAgentFormatter(); - - List messages = new ArrayList<>(); - messages.add( - DashScopeMessage.builder().role("system").content("You are helpful.").build()); - messages.add(DashScopeMessage.builder().role("user").content("Hello").build()); + private static DashScopeMessage message(String role, String content) { + return DashScopeMessage.builder().role(role).content(content).build(); + } - multiFormatter.applyCacheControl(messages); + private static DashScopeContentPart lastPart(DashScopeMessage message) { + List parts = message.getContentAsList(); + assertNotNull(parts); + assertFalse(parts.isEmpty()); + return parts.get(parts.size() - 1); + } - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(1).getCacheControl()); + private static void assertNoCacheControl(DashScopeMessage message) { + assertNull(message.getCacheControl()); + List parts = message.getContentAsList(); + if (parts != null) { + assertTrue(parts.stream().allMatch(part -> part.getCacheControl() == null)); } } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java index 645d17abe0..593ea55230 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java @@ -30,6 +30,7 @@ import io.agentscope.extensions.model.dashscope.dto.DashScopeMessage; import io.agentscope.extensions.model.dashscope.dto.DashScopeOutput; import io.agentscope.extensions.model.dashscope.dto.DashScopeResponse; +import io.agentscope.extensions.model.dashscope.dto.DashScopeTokenDetails; import io.agentscope.extensions.model.dashscope.dto.DashScopeToolCall; import io.agentscope.extensions.model.dashscope.dto.DashScopeUsage; import java.time.Instant; @@ -191,6 +192,48 @@ void testParseResponseWithUsage() { assertEquals(5, chatResponse.getUsage().getOutputTokens()); } + @Test + void testParsePromptCacheUsageDetails() { + DashScopeTokenDetails details = new DashScopeTokenDetails(); + details.setCachedTokens(60); + details.setCacheCreationInputTokens(30); + + DashScopeUsage usage = new DashScopeUsage(); + usage.setInputTokens(100); + usage.setOutputTokens(20); + usage.setPromptTokensDetails(details); + + DashScopeResponse response = new DashScopeResponse(); + response.setRequestId("req-cache-usage"); + response.setUsage(usage); + + ChatResponse chatResponse = parser.parseResponse(response, startTime); + + assertNotNull(chatResponse.getUsage()); + assertEquals(100, chatResponse.getUsage().getInputTokens()); + assertEquals(20, chatResponse.getUsage().getOutputTokens()); + assertEquals(60, chatResponse.getUsage().getCachedTokens()); + assertEquals(30, chatResponse.getUsage().getCacheCreationInputTokens()); + } + + @Test + void testParseLegacyTopLevelCacheUsage() { + DashScopeUsage usage = new DashScopeUsage(); + usage.setInputTokens(100); + usage.setOutputTokens(20); + usage.setCachedTokens(40); + usage.setCacheCreationInputTokens(25); + + DashScopeResponse response = new DashScopeResponse(); + response.setRequestId("req-legacy-cache-usage"); + response.setUsage(usage); + + ChatResponse chatResponse = parser.parseResponse(response, startTime); + + assertEquals(40, chatResponse.getUsage().getCachedTokens()); + assertEquals(25, chatResponse.getUsage().getCacheCreationInputTokens()); + } + @Test void testParseEmptyResponse() { DashScopeResponse response = new DashScopeResponse(); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java index 0fb533a544..647cb99be5 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java @@ -35,8 +35,10 @@ import io.agentscope.extensions.model.openai.dto.OpenAIStreamOptions; import io.agentscope.extensions.model.openai.formatter.OpenAIBaseFormatter; import io.agentscope.extensions.model.openai.formatter.OpenAIChatFormatter; +import java.net.URI; import java.time.Instant; import java.util.List; +import java.util.Locale; import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -155,10 +157,16 @@ protected Flux doStream0( formatter.applyToolChoice(request, effectiveOptions.getToolChoice()); } - // Apply cache control if enabled (adds cache_control to system msgs + last msg) - if (Boolean.TRUE.equals(effectiveOptions.getCacheControl()) - && formatter instanceof OpenAIBaseFormatter openAIFormatter) { - openAIFormatter.applyCacheControl(request.getMessages()); + // DashScope-compatible endpoints require content-block markers. Normalize explicit + // metadata even when automatic cache control is disabled. + if (formatter instanceof OpenAIBaseFormatter openAIFormatter) { + boolean automaticCacheControl = Boolean.TRUE.equals(effectiveOptions.getCacheControl()); + if (isDashScopeCompatibleBaseUrl(baseUrl)) { + openAIFormatter.applyDashScopeCacheControl( + request.getMessages(), automaticCacheControl); + } else if (automaticCacheControl) { + openAIFormatter.applyCacheControl(request.getMessages()); + } } // Make the API call @@ -198,6 +206,26 @@ protected Flux doStream0( } } + static boolean isDashScopeCompatibleBaseUrl(String baseUrl) { + if (baseUrl == null || baseUrl.isBlank()) { + return false; + } + try { + String host = URI.create(baseUrl).getHost(); + if (host == null) { + return false; + } + String normalizedHost = host.toLowerCase(Locale.ROOT); + return normalizedHost.equals("dashscope.aliyuncs.com") + || (normalizedHost.startsWith("dashscope-") + && normalizedHost.endsWith(".aliyuncs.com")) + || normalizedHost.endsWith(".dashscope.aliyuncs.com") + || normalizedHost.endsWith(".maas.aliyuncs.com"); + } catch (IllegalArgumentException e) { + return false; + } + } + /** * Gets the model name for logging and identification. * diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java index 6a8612e33a..738e1f8369 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; /** * OpenAI content part DTO for multimodal messages. @@ -65,6 +66,10 @@ public class OpenAIContentPart { @JsonProperty("video_url") private OpenAIVideoUrl videoUrl; + /** Prompt cache control marker used by compatible providers such as DashScope. */ + @JsonProperty("cache_control") + private Map cacheControl; + public OpenAIContentPart() {} public String getType() { @@ -107,6 +112,14 @@ public void setVideoUrl(OpenAIVideoUrl videoUrl) { this.videoUrl = videoUrl; } + public Map getCacheControl() { + return cacheControl; + } + + public void setCacheControl(Map cacheControl) { + this.cacheControl = cacheControl; + } + /** * Create a text content part. * @@ -206,6 +219,11 @@ public Builder videoUrl(OpenAIVideoUrl videoUrl) { return this; } + public Builder cacheControl(Map cacheControl) { + part.setCacheControl(cacheControl); + return this; + } + public OpenAIContentPart build() { OpenAIContentPart result = part; part = new OpenAIContentPart(); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java index 439a85a594..2111819d8e 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java @@ -20,10 +20,12 @@ import io.agentscope.core.model.GenerateOptions; import io.agentscope.core.model.ToolChoice; import io.agentscope.core.model.ToolSchema; +import io.agentscope.extensions.model.openai.dto.OpenAIContentPart; import io.agentscope.extensions.model.openai.dto.OpenAIMessage; import io.agentscope.extensions.model.openai.dto.OpenAIRequest; import io.agentscope.extensions.model.openai.dto.OpenAIResponse; import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -193,6 +195,110 @@ public void applyCacheControl(List messages) { } } + /** + * Apply DashScope-compatible content-block cache control. + * + *

Legacy message-level markers are migrated to the last content block. Automatic + * breakpoints use the shared first-system and last-conversation strategy. + * + * @param messages formatted OpenAI-compatible messages + * @param automatic whether to add automatic breakpoints + */ + public void applyDashScopeCacheControl(List messages, boolean automatic) { + if (messages == null || messages.isEmpty()) { + return; + } + + List selected = + selectPromptCacheBreakpoints( + messages, + automatic, + OpenAIBaseFormatter::hasExplicitCacheControl, + message -> "system".equals(message.getRole()), + OpenAIBaseFormatter::isCacheable); + for (OpenAIMessage message : selected) { + applyDashScopeCacheControlToContentBlock(message); + } + for (OpenAIMessage message : messages) { + message.setCacheControl(null); + } + + int markerCount = countContentBlockCacheMarkers(messages); + if (markerCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "DashScope supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " cache_control markers, but got " + + markerCount); + } + } + + static void applyDashScopeCacheControlToContentBlock(OpenAIMessage message) { + if (hasContentBlockCacheControl(message)) { + message.setCacheControl(null); + return; + } + + List parts = ensureContentArray(message); + if (parts.isEmpty()) { + throw new IllegalArgumentException( + "Cannot apply cache_control to a message without cacheable content"); + } + Map cacheControl = + message.getCacheControl() != null + ? message.getCacheControl() + : EPHEMERAL_CACHE_CONTROL; + parts.get(parts.size() - 1).setCacheControl(cacheControl); + message.setCacheControl(null); + } + + @SuppressWarnings("unchecked") + static List ensureContentArray(OpenAIMessage message) { + Object content = message.getContent(); + if (content instanceof List) { + return (List) content; + } + if (content instanceof String text) { + List parts = new ArrayList<>(List.of(OpenAIContentPart.text(text))); + message.setContent(parts); + return parts; + } + return List.of(); + } + + private static boolean isCacheable(OpenAIMessage message) { + if (!("system".equals(message.getRole()) + || "user".equals(message.getRole()) + || "assistant".equals(message.getRole()) + || "tool".equals(message.getRole()))) { + return false; + } + Object content = message.getContent(); + return (content instanceof String text && !text.isEmpty()) + || (content instanceof List parts && !parts.isEmpty()); + } + + private static boolean hasExplicitCacheControl(OpenAIMessage message) { + return message.getCacheControl() != null || hasContentBlockCacheControl(message); + } + + private static boolean hasContentBlockCacheControl(OpenAIMessage message) { + List parts = message.getContentAsList(); + return parts != null && parts.stream().anyMatch(part -> part.getCacheControl() != null); + } + + private static int countContentBlockCacheMarkers(List messages) { + int count = 0; + for (OpenAIMessage message : messages) { + List parts = message.getContentAsList(); + if (parts != null) { + count += + (int) parts.stream().filter(part -> part.getCacheControl() != null).count(); + } + } + return count; + } + /** * Get the ephemeral cache control constant. * diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java index 9b3cbf5b44..a347a8a825 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java @@ -113,6 +113,26 @@ void testNativeStructuredOutputDisabled() { assertFalse(m.supportsNativeStructuredOutput()); } + @Test + @DisplayName("Should recognize Alibaba OpenAI-compatible endpoints") + void testDashScopeCompatibleBaseUrlDetection() { + assertTrue( + OpenAIChatModel.isDashScopeCompatibleBaseUrl( + "https://dashscope.aliyuncs.com/compatible-mode/v1")); + assertTrue( + OpenAIChatModel.isDashScopeCompatibleBaseUrl( + "https://dashscope-intl.aliyuncs.com/compatible-mode/v1")); + assertTrue( + OpenAIChatModel.isDashScopeCompatibleBaseUrl( + "https://dashscope-us.aliyuncs.com/compatible-mode/v1")); + assertTrue( + OpenAIChatModel.isDashScopeCompatibleBaseUrl( + "https://workspace.cn-beijing.maas.aliyuncs.com/compatible-mode/v1")); + assertFalse(OpenAIChatModel.isDashScopeCompatibleBaseUrl("https://api.openai.com/v1")); + assertFalse(OpenAIChatModel.isDashScopeCompatibleBaseUrl(null)); + assertFalse(OpenAIChatModel.isDashScopeCompatibleBaseUrl("not a valid URL")); + } + @Test @DisplayName("Should make non-streaming call successfully") void testNonStreamingCall() throws Exception { diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java index cbe229dc6e..e8ae81c6f1 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java @@ -16,11 +16,16 @@ package io.agentscope.extensions.model.openai.formatter; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import com.fasterxml.jackson.databind.JsonNode; import io.agentscope.core.message.MessageMetadataKeys; import io.agentscope.core.message.Msg; import io.agentscope.core.message.MsgRole; +import io.agentscope.core.util.JsonUtils; +import io.agentscope.extensions.model.openai.dto.OpenAIContentPart; import io.agentscope.extensions.model.openai.dto.OpenAIMessage; import java.util.ArrayList; import java.util.HashMap; @@ -238,4 +243,111 @@ void systemMessageMetadata() { assertNull(result.get(1).getCacheControl()); } } + + @Nested + @DisplayName("DashScope-compatible cache_control") + class DashScopeCompatibleTest { + + @Test + @DisplayName("should serialize markers on content blocks using shared strategy") + void exactContentBlockJson() { + List messages = + new ArrayList<>( + List.of( + OpenAIMessage.builder() + .role("system") + .content("System 1") + .build(), + OpenAIMessage.builder() + .role("system") + .content("System 2") + .build(), + OpenAIMessage.builder() + .role("user") + .content("Question") + .build())); + + formatter.applyDashScopeCacheControl(messages, true); + + JsonNode first = toJsonTree(messages.get(0)); + JsonNode second = toJsonTree(messages.get(1)); + JsonNode last = toJsonTree(messages.get(2)); + assertFalse(first.has("cache_control")); + assertEquals("text", first.path("content").get(0).path("type").asText()); + assertEquals( + "ephemeral", + first.path("content").get(0).path("cache_control").path("type").asText()); + assertFalse(second.has("cache_control")); + assertEquals("System 2", second.path("content").asText()); + assertEquals( + "ephemeral", + last.path("content").get(0).path("cache_control").path("type").asText()); + } + + @Test + @DisplayName("should retain explicit metadata marker when automatic strategy is disabled") + void explicitMetadataWithoutAutomaticStrategy() { + Msg msg = + Msg.builder() + .role(MsgRole.USER) + .textContent("Pinned context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + List messages = formatter.format(List.of(msg)); + + formatter.applyDashScopeCacheControl(messages, false); + + assertNull(messages.get(0).getCacheControl()); + assertEquals(EPHEMERAL, messages.get(0).getContentAsList().get(0).getCacheControl()); + } + + @Test + @DisplayName("should place multimodal marker on the final content block") + void multimodalFinalBlock() { + OpenAIMessage message = + OpenAIMessage.builder() + .role("user") + .content( + new ArrayList<>( + List.of( + OpenAIContentPart.text("Look"), + OpenAIContentPart.imageUrl( + "https://example.com/image.png")))) + .build(); + + formatter.applyDashScopeCacheControl(List.of(message), true); + + assertNull(message.getContentAsList().get(0).getCacheControl()); + assertEquals(EPHEMERAL, message.getContentAsList().get(1).getCacheControl()); + } + + @Test + @DisplayName("should reject more than four explicit content-block markers") + void tooManyExplicitMarkers() { + List messages = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + messages.add( + OpenAIMessage.builder() + .role("user") + .content( + new ArrayList<>( + List.of( + OpenAIContentPart.builder() + .type("text") + .text("Context " + i) + .cacheControl(EPHEMERAL) + .build()))) + .build()); + } + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyDashScopeCacheControl(messages, false)); + } + + private JsonNode toJsonTree(OpenAIMessage message) { + return JsonUtils.getJsonCodec() + .fromJson(JsonUtils.getJsonCodec().toJson(message), JsonNode.class); + } + } } From 28ec0a33942b2f3159919fdf4c8ebf4868949cee Mon Sep 17 00:00:00 2001 From: guslegend <1670547022@qq.com> Date: Tue, 25 Aug 2026 14:04:29 +0800 Subject: [PATCH 03/10] feat(model): add provider-native prompt caching --- .../agent/accumulator/ReasoningContext.java | 10 +- .../core/message/MessageMetadataKeys.java | 7 +- .../core/model/GenerateOptions.java | 10 +- .../accumulator/ReasoningContextTest.java | 3 + .../model/anthropic/AnthropicChatModel.java | 27 ++- .../formatter/AnthropicBaseFormatter.java | 72 ++++++++ .../formatter/AnthropicMessageConverter.java | 65 +++++++ .../AnthropicMultiAgentFormatter.java | 7 +- .../AnthropicPromptCacheSupport.java | 97 +++++++++++ .../formatter/AnthropicResponseParser.java | 98 ++++++++--- .../formatter/AnthropicPromptCacheTest.java | 140 +++++++++++++++ .../AnthropicResponseParserTest.java | 49 +++++- .../model/openai/OpenAIChatModel.java | 20 ++- .../model/openai/dto/OpenAIContentPart.java | 17 ++ .../model/openai/dto/OpenAIUsage.java | 11 ++ .../openai/formatter/OpenAIBaseFormatter.java | 105 ++++++++++++ .../formatter/OpenAIResponseParser.java | 12 ++ .../model/openai/OpenAIChatModelTest.java | 20 ++- .../formatter/OpenAICacheControlTest.java | 161 ++++++++++++++++++ .../formatter/OpenAIResponseParserTest.java | 4 + 20 files changed, 887 insertions(+), 48 deletions(-) create mode 100644 agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java create mode 100644 agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java diff --git a/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java b/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java index bf08581b4c..6aa0a96536 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java +++ b/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java @@ -56,6 +56,7 @@ public class ReasoningContext { private int inputTokens = 0; private int outputTokens = 0; private int cachedTokens = 0; + private int cacheCreationInputTokens = 0; private double time = 0; public ReasoningContext(String agentName) { @@ -85,6 +86,7 @@ public List processChunk(ChatResponse chunk) { inputTokens = usage.getInputTokens(); outputTokens = usage.getOutputTokens(); cachedTokens = usage.getCachedTokens(); + cacheCreationInputTokens = usage.getCacheCreationInputTokens(); time = usage.getTime(); } @@ -169,12 +171,17 @@ public Msg buildFinalMessage() { // Build metadata with accumulated ChatUsage Map metadata = new HashMap<>(); ChatUsage chatUsage = null; - if (inputTokens > 0 || outputTokens > 0 || time > 0) { + if (inputTokens > 0 + || outputTokens > 0 + || cachedTokens > 0 + || cacheCreationInputTokens > 0 + || time > 0) { chatUsage = ChatUsage.builder() .inputTokens(inputTokens) .outputTokens(outputTokens) .cachedTokens(cachedTokens) + .cacheCreationInputTokens(cacheCreationInputTokens) .time(time) .build(); metadata.put(MessageMetadataKeys.CHAT_USAGE, chatUsage); @@ -292,6 +299,7 @@ public ChatUsage getChatUsage() { .inputTokens(inputTokens) .outputTokens(outputTokens) .cachedTokens(cachedTokens) + .cacheCreationInputTokens(cacheCreationInputTokens) .time(time) .build(); } diff --git a/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java b/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java index 207c27b3e7..7824e44016 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java +++ b/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java @@ -109,10 +109,9 @@ private MessageMetadataKeys() { /** * Metadata key to mark a message for prompt caching. * - *

When set to {@code true}, the formatter will add cache_control: - * {"type": "ephemeral"} to this message during formatting. This allows users to manually - * mark specific - * messages for caching, independent of the automatic cache control strategy configured via + *

When set to {@code true}, a supporting formatter encodes a provider-specific cache + * breakpoint at this message. This allows users to manually mark specific messages for caching, + * independent of the automatic cache control strategy configured via * {@link io.agentscope.core.model.GenerateOptions#getCacheControl()}. Explicit markers remain * effective when automatic cache control is disabled or unset. * diff --git a/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java b/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java index 9eb95c1ee1..1970936493 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java +++ b/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java @@ -309,9 +309,9 @@ public Long getSeed() { /** * Gets whether cache control is enabled for prompt caching. * - *

When true, a supporting formatter automatically selects the first cacheable system message - * and the last cacheable non-system message as prompt cache breakpoints. Provider-specific - * formatters are responsible for encoding the selected breakpoints in the protocol they support. + *

When true, a supporting provider enables its automatic prompt-caching strategy. Some + * providers select request breakpoints in the formatter, while others expose a provider-native + * automatic caching mode. * *

Users can also manually mark individual messages for caching via {@link * io.agentscope.core.message.MessageMetadataKeys#CACHE_CONTROL} metadata. Manually marked @@ -779,8 +779,8 @@ public Builder seed(Long seed) { /** * Sets whether cache control is enabled for prompt caching. * - *

When true, the formatter will automatically add cache_control: - * {"type": "ephemeral"} to system messages and the last message in the request. + *

When true, a supporting provider enables its automatic prompt-caching strategy using + * the request shape defined by that provider. * * @param cacheControl true to enable cache control, false to disable * @return this builder instance diff --git a/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java b/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java index 83b00def5e..26f010203d 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java @@ -82,6 +82,7 @@ void testCachedTokensPropagation() { .inputTokens(100) .outputTokens(50) .cachedTokens(80) + .cacheCreationInputTokens(15) .time(1.5) .build(); @@ -98,10 +99,12 @@ void testCachedTokensPropagation() { assertNotNull(msg); assertNotNull(msg.getChatUsage()); assertEquals(80, msg.getChatUsage().getCachedTokens()); + assertEquals(15, msg.getChatUsage().getCacheCreationInputTokens()); ChatUsage resultUsage = context.getChatUsage(); assertNotNull(resultUsage); assertEquals(80, resultUsage.getCachedTokens()); + assertEquals(15, resultUsage.getCacheCreationInputTokens()); } @Test diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/AnthropicChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/AnthropicChatModel.java index cc90a9e60c..2a2cec2a96 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/AnthropicChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/AnthropicChatModel.java @@ -22,6 +22,7 @@ import com.anthropic.models.messages.MessageParam; import com.anthropic.models.messages.RawMessageStreamEvent; import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; import io.agentscope.core.model.ChatModelBase; import io.agentscope.core.model.ChatResponse; import io.agentscope.core.model.GenerateOptions; @@ -158,14 +159,28 @@ protected Flux doStream( .model(modelName) .maxTokens(4096); - // Extract and apply system message - // (Anthropic-specific requirement) - formatter.applySystemMessage(paramsBuilder, messages); - // Use formatter to convert Msg to Anthropic // MessageParam - List formattedMessages = formatter.format(messages); - for (MessageParam param : formattedMessages) { + List providerMessages = + messages != null + && !messages.isEmpty() + && messages.get(0).getRole() + == MsgRole.SYSTEM + ? messages.subList(1, messages.size()) + : messages; + List formattedMessages = + formatter.format(providerMessages); + GenerateOptions effectiveOptions = + GenerateOptions.mergeOptions(options, defaultOptions); + boolean automaticCache = + Boolean.TRUE.equals(effectiveOptions.getCacheControl()); + AnthropicBaseFormatter.PromptCachePlan cachePlan = + formatter.applyPromptCache( + paramsBuilder, + messages, + formattedMessages, + automaticCache); + for (MessageParam param : cachePlan.messages()) { paramsBuilder.addMessage(param); } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java index 9e0ac24d9f..8982bede63 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java @@ -15,13 +15,19 @@ */ package io.agentscope.extensions.model.anthropic.formatter; +import com.anthropic.core.JsonValue; import com.anthropic.models.messages.MessageCreateParams; import com.anthropic.models.messages.MessageParam; +import com.anthropic.models.messages.TextBlockParam; import io.agentscope.core.formatter.AbstractBaseFormatter; +import io.agentscope.core.message.MessageMetadataKeys; import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; import io.agentscope.core.model.GenerateOptions; import io.agentscope.core.model.ToolSchema; +import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Abstract base formatter for Anthropic API with shared logic for handling Anthropic-specific @@ -103,4 +109,70 @@ public void applySystemMessage(MessageCreateParams.Builder paramsBuilder, List originalMessages, + List formattedMessages, + boolean automatic) { + List plannedMessages = new ArrayList<>(formattedMessages); + Msg system = + originalMessages != null + && !originalMessages.isEmpty() + && originalMessages.get(0).getRole() == MsgRole.SYSTEM + ? originalMessages.get(0) + : null; + + boolean explicitSystem = + system != null + && system.getMetadata() != null + && Boolean.TRUE.equals( + system.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL)); + int explicitCount = explicitSystem ? 1 : 0; + explicitCount += + (int) + plannedMessages.stream() + .filter(AnthropicPromptCacheSupport::hasCacheControl) + .count(); + if (explicitCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "Prompt cache supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " explicit breakpoints, but got " + + explicitCount); + } + if (automatic && explicitCount == MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "Anthropic automatic prompt caching requires one of the four breakpoint slots"); + } + if (automatic) { + paramsBuilder.putAdditionalBodyProperty( + "cache_control", JsonValue.from(Map.of("type", "ephemeral"))); + } + + if (system != null) { + String systemText = messageConverter.extractSystemMessage(originalMessages); + if (systemText != null && !systemText.isEmpty()) { + TextBlockParam.Builder block = TextBlockParam.builder().text(systemText); + if (explicitSystem) { + block.cacheControl(AnthropicPromptCacheSupport.ephemeral()); + } + paramsBuilder.systemOfTextBlockParams(List.of(block.build())); + } else if (explicitSystem) { + throw new IllegalArgumentException( + "Explicit Anthropic system cache breakpoint has no text content"); + } + } + + return new PromptCachePlan(List.copyOf(plannedMessages)); + } + + /** Provider-ready messages and tool cache decision for one request. */ + public record PromptCachePlan(List messages) {} } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMessageConverter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMessageConverter.java index 60ec2f1035..5f7538bbac 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMessageConverter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMessageConverter.java @@ -80,12 +80,15 @@ public List convert(List messages) { for (int i = 0; i < messages.size(); i++) { Msg msg = messages.get(i); boolean isFirstMessage = (i == 0); + int resultStart = result.size(); if (msg.getRole() == MsgRole.ASSISTANT && msg.getContentBlocks(ToolUseBlock.class).size() > 1) { SplitToolResultSequence splitResults = collectSplitToolResults(messages, i + 1); if (shouldSplitParallelToolCalls(msg, splitResults)) { result.addAll(convertParallelToolCalls(msg, splitResults)); + applyExplicitBreakpoints( + result, resultStart, msg, messages, i + 1, splitResults); i += splitResults.consumedMessages(); continue; } @@ -123,11 +126,73 @@ public List convert(List messages) { result.add(param); } } + + markLastProducedMessage(result, resultStart, msg); } return result; } + private void markLastProducedMessage(List result, int start, Msg source) { + if (!AnthropicPromptCacheSupport.isExplicitlyMarked(source)) { + return; + } + if (result.size() == start) { + throw new IllegalArgumentException( + "Explicit Anthropic prompt cache breakpoint has no provider message"); + } + int last = result.size() - 1; + result.set(last, AnthropicPromptCacheSupport.markLastCacheableBlock(result.get(last))); + } + + private void applyExplicitBreakpoints( + List result, + int start, + Msg assistant, + List messages, + int toolResultStart, + SplitToolResultSequence splitResults) { + if (AnthropicPromptCacheSupport.isExplicitlyMarked(assistant)) { + int assistantIndex = start + (splitResults.resultsById().size() - 1) * 2; + result.set( + assistantIndex, + AnthropicPromptCacheSupport.markLastCacheableBlock(result.get(assistantIndex))); + } + + for (int i = toolResultStart; i < toolResultStart + splitResults.consumedMessages(); i++) { + Msg toolResultMsg = messages.get(i); + if (AnthropicPromptCacheSupport.isExplicitlyMarked(toolResultMsg)) { + List sourceResults = + toolResultMsg.getContentBlocks(ToolResultBlock.class); + String lastToolUseId = sourceResults.get(sourceResults.size() - 1).getId(); + int providerIndex = findToolResultMessage(result, start, lastToolUseId); + result.set( + providerIndex, + AnthropicPromptCacheSupport.markLastCacheableBlock( + result.get(providerIndex))); + } + } + } + + private int findToolResultMessage( + List result, int start, String expectedToolUseId) { + for (int i = result.size() - 1; i >= start; i--) { + MessageParam param = result.get(i); + if (!param.content().isBlockParams()) { + continue; + } + for (ContentBlockParam block : param.content().asBlockParams()) { + if (block.isToolResult() + && java.util.Objects.equals( + block.asToolResult().toolUseId(), expectedToolUseId)) { + return i; + } + } + } + throw new IllegalArgumentException( + "Explicit Anthropic tool-result cache breakpoint lost during conversion"); + } + /** * Decide whether the current assistant message should be expanded into Anthropic's required * tool-use/tool-result alternation. diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMultiAgentFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMultiAgentFormatter.java index 7ff7a644ba..e5cb8168d0 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMultiAgentFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMultiAgentFormatter.java @@ -89,6 +89,7 @@ public List doFormat(List msgs) { } case TOOL_SEQUENCE -> result.addAll(formatToolSequence(group.messages)); case AGENT_CONVERSATION -> result.addAll(formatAgentConversation(group.messages)); + case BYPASS -> result.addAll(messageConverter.convert(group.messages)); } } @@ -109,7 +110,8 @@ public ChatResponse parseResponse(Object response, Instant startTime) { private enum GroupType { SYSTEM, TOOL_SEQUENCE, - AGENT_CONVERSATION + AGENT_CONVERSATION, + BYPASS } private static class MessageGroup { @@ -158,6 +160,9 @@ private List groupMessages(List msgs) { /** Determine message type for grouping. */ private GroupType determineMessageType(Msg msg) { + if (shouldBypassHistory(msg)) { + return GroupType.BYPASS; + } if (msg.getRole() == MsgRole.SYSTEM && !msg.getContent().isEmpty() && !(msg.getContent().get(0) instanceof ToolResultBlock)) { diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java new file mode 100644 index 0000000000..8900b98c47 --- /dev/null +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java @@ -0,0 +1,97 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + */ +package io.agentscope.extensions.model.anthropic.formatter; + +import com.anthropic.models.messages.CacheControlEphemeral; +import com.anthropic.models.messages.ContentBlockParam; +import com.anthropic.models.messages.MessageParam; +import io.agentscope.core.message.MessageMetadataKeys; +import io.agentscope.core.message.Msg; +import java.util.ArrayList; +import java.util.List; + +/** Utilities for encoding AgentScope prompt-cache intent into Anthropic request blocks. */ +final class AnthropicPromptCacheSupport { + + private static final CacheControlEphemeral EPHEMERAL = CacheControlEphemeral.builder().build(); + + private AnthropicPromptCacheSupport() {} + + static CacheControlEphemeral ephemeral() { + return EPHEMERAL; + } + + static boolean isExplicitlyMarked(Msg msg) { + return msg != null + && msg.getMetadata() != null + && Boolean.TRUE.equals(msg.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL)); + } + + static boolean hasCacheControl(MessageParam message) { + if (!message.content().isBlockParams()) { + return false; + } + return message.content().asBlockParams().stream() + .anyMatch(AnthropicPromptCacheSupport::hasCacheControl); + } + + static MessageParam markLastCacheableBlock(MessageParam message) { + if (!message.content().isBlockParams()) { + throw new IllegalArgumentException( + "Anthropic prompt cache breakpoint requires block content"); + } + + List blocks = new ArrayList<>(message.content().asBlockParams()); + for (int i = blocks.size() - 1; i >= 0; i--) { + ContentBlockParam marked = mark(blocks.get(i)); + if (marked != null) { + blocks.set(i, marked); + return message.toBuilder() + .content(MessageParam.Content.ofBlockParams(blocks)) + .build(); + } + } + throw new IllegalArgumentException( + "Anthropic prompt cache breakpoint has no cacheable content block"); + } + + private static boolean hasCacheControl(ContentBlockParam block) { + if (block.isText()) { + return block.asText().cacheControl().isPresent(); + } + if (block.isImage()) { + return block.asImage().cacheControl().isPresent(); + } + if (block.isToolUse()) { + return block.asToolUse().cacheControl().isPresent(); + } + if (block.isToolResult()) { + return block.asToolResult().cacheControl().isPresent(); + } + return false; + } + + private static ContentBlockParam mark(ContentBlockParam block) { + if (block.isText()) { + return ContentBlockParam.ofText( + block.asText().toBuilder().cacheControl(EPHEMERAL).build()); + } + if (block.isImage()) { + return ContentBlockParam.ofImage( + block.asImage().toBuilder().cacheControl(EPHEMERAL).build()); + } + if (block.isToolUse()) { + return ContentBlockParam.ofToolUse( + block.asToolUse().toBuilder().cacheControl(EPHEMERAL).build()); + } + if (block.isToolResult()) { + return ContentBlockParam.ofToolResult( + block.asToolResult().toBuilder().cacheControl(EPHEMERAL).build()); + } + return null; + } +} diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParser.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParser.java index 1e63b86efc..7c57f483fa 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParser.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParser.java @@ -19,6 +19,7 @@ import com.anthropic.core.ObjectMappers; import com.anthropic.models.messages.Message; import com.anthropic.models.messages.RawMessageStreamEvent; +import com.anthropic.models.messages.Usage; import io.agentscope.core.message.ContentBlock; import io.agentscope.core.message.TextBlock; import io.agentscope.core.message.ThinkingBlock; @@ -87,14 +88,17 @@ public static ChatResponse parseMessage(Message message, Instant startTime) { } // Parse usage - long baseInput = message.usage().inputTokens(); - long cacheRead = message.usage().cacheReadInputTokens().orElse(0L); - long cacheCreate = message.usage().cacheCreationInputTokens().orElse(0L); + Usage anthropicUsage = message.usage(); + long cacheRead = anthropicUsage.cacheReadInputTokens().orElse(0L); + long cacheCreation = anthropicUsage.cacheCreationInputTokens().orElse(0L); ChatUsage usage = ChatUsage.builder() - .inputTokens((int) (baseInput + cacheRead + cacheCreate)) - .outputTokens((int) message.usage().outputTokens()) - .cachedTokens((int) cacheRead) + .inputTokens( + Math.toIntExact( + anthropicUsage.inputTokens() + cacheRead + cacheCreation)) + .outputTokens(Math.toIntExact(anthropicUsage.outputTokens())) + .cachedTokens(Math.toIntExact(cacheRead)) + .cacheCreationInputTokens(Math.toIntExact(cacheCreation)) .time(Duration.between(startTime, Instant.now()).toMillis() / 1000.0) .build(); @@ -106,30 +110,52 @@ public static ChatResponse parseMessage(Message message, Instant startTime) { */ public static Flux parseStreamEvents( Flux eventFlux, Instant startTime) { - return eventFlux - .flatMap( - event -> { - try { - return Flux.just(parseStreamEvent(event, startTime)); - } catch (Exception e) { - log.warn("Error parsing stream event: {}", e.getMessage()); - return Flux.empty(); - } - }) - .filter(response -> response != null && !response.getContent().isEmpty()); + return Flux.defer( + () -> { + StreamState state = new StreamState(); + return eventFlux + .concatMap( + event -> { + try { + return Flux.just( + parseStreamEvent(event, startTime, state)); + } catch (Exception e) { + log.warn( + "Error parsing stream event: {}", + e.getMessage()); + return Flux.empty(); + } + }) + .filter( + response -> + response != null + && (!response.getContent().isEmpty() + || response.getUsage() != null)); + }); } /** * Parse single stream event. */ private static ChatResponse parseStreamEvent(RawMessageStreamEvent event, Instant startTime) { + return parseStreamEvent(event, startTime, new StreamState()); + } + + private static ChatResponse parseStreamEvent( + RawMessageStreamEvent event, Instant startTime, StreamState state) { List contentBlocks = new ArrayList<>(); ChatUsage usage = null; - String messageId = null; // Message start if (event.isMessageStart()) { - messageId = event.asMessageStart().message().id(); + Message message = event.asMessageStart().message(); + state.messageId = message.id(); + Usage startUsage = message.usage(); + if (startUsage != null) { + state.inputTokens = startUsage.inputTokens(); + state.cachedTokens = startUsage.cacheReadInputTokens().orElse(0L); + state.cacheCreationInputTokens = startUsage.cacheCreationInputTokens().orElse(0L); + } } // Content block delta - text @@ -193,14 +219,44 @@ private static ChatResponse parseStreamEvent(RawMessageStreamEvent event, Instan // Message delta - usage information if (event.isMessageDelta()) { var messageDelta = event.asMessageDelta(); + messageDelta.usage().inputTokens().ifPresent(value -> state.inputTokens = value); + messageDelta + .usage() + .cacheReadInputTokens() + .ifPresent(value -> state.cachedTokens = value); + messageDelta + .usage() + .cacheCreationInputTokens() + .ifPresent(value -> state.cacheCreationInputTokens = value); + state.outputTokens = messageDelta.usage().outputTokens(); usage = ChatUsage.builder() - .outputTokens((int) messageDelta.usage().outputTokens()) + .inputTokens( + Math.toIntExact( + state.inputTokens + + state.cachedTokens + + state.cacheCreationInputTokens)) + .outputTokens(Math.toIntExact(state.outputTokens)) + .cachedTokens(Math.toIntExact(state.cachedTokens)) + .cacheCreationInputTokens( + Math.toIntExact(state.cacheCreationInputTokens)) .time(Duration.between(startTime, Instant.now()).toMillis() / 1000.0) .build(); } - return ChatResponse.builder().id(messageId).content(contentBlocks).usage(usage).build(); + return ChatResponse.builder() + .id(state.messageId) + .content(contentBlocks) + .usage(usage) + .build(); + } + + private static final class StreamState { + private String messageId; + private long inputTokens; + private long outputTokens; + private long cachedTokens; + private long cacheCreationInputTokens; } /** diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java new file mode 100644 index 0000000000..042a4d69d1 --- /dev/null +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java @@ -0,0 +1,140 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + */ +package io.agentscope.extensions.model.anthropic.formatter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.anthropic.models.messages.MessageCreateParams; +import com.anthropic.models.messages.MessageParam; +import io.agentscope.core.message.MessageMetadataKeys; +import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; +import io.agentscope.core.message.TextBlock; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class AnthropicPromptCacheTest { + + private final AnthropicChatFormatter formatter = new AnthropicChatFormatter(); + + @Test + void automaticCachingUsesTopLevelProviderField() { + Msg user = message(MsgRole.USER, "hello", false); + List formatted = formatter.format(List.of(user)); + MessageCreateParams.Builder builder = baseBuilder(); + + AnthropicBaseFormatter.PromptCachePlan plan = + formatter.applyPromptCache(builder, List.of(user), formatted, true); + plan.messages().forEach(builder::addMessage); + MessageCreateParams params = builder.build(); + + @SuppressWarnings("unchecked") + Map cacheControl = + params._additionalBodyProperties().get("cache_control").convert(Map.class); + assertEquals(Map.of("type", "ephemeral"), cacheControl); + assertFalse(AnthropicPromptCacheSupport.hasCacheControl(plan.messages().get(0))); + } + + @Test + void explicitMessageMarkerWorksWhenAutomaticCachingIsDisabled() { + Msg user = message(MsgRole.USER, "stable context", true); + List formatted = formatter.format(List.of(user)); + MessageCreateParams.Builder builder = baseBuilder(); + + AnthropicBaseFormatter.PromptCachePlan plan = + formatter.applyPromptCache(builder, List.of(user), formatted, false); + plan.messages().forEach(builder::addMessage); + MessageCreateParams params = builder.build(); + + assertTrue(AnthropicPromptCacheSupport.hasCacheControl(plan.messages().get(0))); + assertFalse(params._additionalBodyProperties().containsKey("cache_control")); + } + + @Test + void explicitSystemMarkerUsesTypedSystemBlockWithoutDuplicatingMessage() { + Msg system = message(MsgRole.SYSTEM, "You are helpful", true); + Msg user = message(MsgRole.USER, "hello", false); + List formatted = formatter.format(List.of(user)); + MessageCreateParams.Builder builder = baseBuilder(); + + AnthropicBaseFormatter.PromptCachePlan plan = + formatter.applyPromptCache(builder, List.of(system, user), formatted, false); + plan.messages().forEach(builder::addMessage); + MessageCreateParams params = builder.build(); + + assertEquals(1, params.messages().size()); + assertTrue(params.system().orElseThrow().isTextBlockParams()); + assertTrue( + params.system() + .orElseThrow() + .asTextBlockParams() + .get(0) + .cacheControl() + .isPresent()); + } + + @Test + void multiAgentFormatterPreservesExplicitBoundary() { + AnthropicMultiAgentFormatter multiAgentFormatter = new AnthropicMultiAgentFormatter(); + List formatted = + multiAgentFormatter.format( + List.of( + message(MsgRole.USER, "first", false), + message(MsgRole.ASSISTANT, "stable", true), + message(MsgRole.USER, "last", false))); + + assertEquals(3, formatted.size()); + assertTrue(AnthropicPromptCacheSupport.hasCacheControl(formatted.get(1))); + } + + @Test + void rejectsMoreThanFourExplicitBreakpoints() { + List messages = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + messages.add(message(MsgRole.USER, "context-" + i, true)); + } + List formatted = formatter.format(messages); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyPromptCache(baseBuilder(), messages, formatted, false)); + } + + @Test + void automaticCachingRejectsFourExplicitBreakpointsBecauseItUsesOneSlot() { + List messages = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + messages.add(message(MsgRole.USER, "context-" + i, true)); + } + List formatted = formatter.format(messages); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyPromptCache(baseBuilder(), messages, formatted, true)); + } + + private static MessageCreateParams.Builder baseBuilder() { + return MessageCreateParams.builder().model("claude-test").maxTokens(32); + } + + private static Msg message(MsgRole role, String text, boolean cache) { + Msg.Builder builder = + Msg.builder() + .name(role.name()) + .role(role) + .content(TextBlock.builder().text(text).build()); + if (cache) { + builder.metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)); + } + return builder.build(); + } +} diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java index d871d056e0..8d09123162 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java @@ -26,7 +26,9 @@ import com.anthropic.models.messages.ContentBlock; import com.anthropic.models.messages.Message; +import com.anthropic.models.messages.MessageDeltaUsage; import com.anthropic.models.messages.RawContentBlockDeltaEvent; +import com.anthropic.models.messages.RawMessageDeltaEvent; import com.anthropic.models.messages.RawMessageStartEvent; import com.anthropic.models.messages.RawMessageStreamEvent; import com.anthropic.models.messages.Usage; @@ -84,6 +86,8 @@ void testParseMessageWithTextBlock() { when(message.usage()).thenReturn(usage); when(usage.inputTokens()).thenReturn(100L); when(usage.outputTokens()).thenReturn(50L); + when(usage.cacheReadInputTokens()).thenReturn(Optional.of(30L)); + when(usage.cacheCreationInputTokens()).thenReturn(Optional.of(20L)); when(contentBlock.text()).thenReturn(Optional.of(textBlock)); when(contentBlock.toolUse()).thenReturn(Optional.empty()); @@ -101,8 +105,10 @@ void testParseMessageWithTextBlock() { ChatUsage responseUsage = response.getUsage(); assertNotNull(responseUsage); - assertEquals(100, responseUsage.getInputTokens()); + assertEquals(150, responseUsage.getInputTokens()); assertEquals(50, responseUsage.getOutputTokens()); + assertEquals(30, responseUsage.getCachedTokens()); + assertEquals(20, responseUsage.getCacheCreationInputTokens()); } @Test @@ -344,6 +350,47 @@ void testParseStreamEventMessageStart() throws Exception { assertTrue(response.getContent().isEmpty()); // MessageStart has no content } + @Test + void testStreamUsageRetainsCacheTokensAndMessageId() { + RawMessageStreamEvent startEvent = mock(RawMessageStreamEvent.class); + RawMessageStartEvent messageStart = mock(RawMessageStartEvent.class); + Message message = mock(Message.class); + Usage startUsage = mock(Usage.class); + when(startEvent.isMessageStart()).thenReturn(true); + when(startEvent.asMessageStart()).thenReturn(messageStart); + when(messageStart.message()).thenReturn(message); + when(message.id()).thenReturn("msg-cache"); + when(message.usage()).thenReturn(startUsage); + when(startUsage.inputTokens()).thenReturn(10L); + when(startUsage.cacheReadInputTokens()).thenReturn(Optional.of(70L)); + when(startUsage.cacheCreationInputTokens()).thenReturn(Optional.of(20L)); + + RawMessageStreamEvent deltaEvent = mock(RawMessageStreamEvent.class); + RawMessageDeltaEvent messageDelta = mock(RawMessageDeltaEvent.class); + MessageDeltaUsage deltaUsage = mock(MessageDeltaUsage.class); + when(deltaEvent.isMessageDelta()).thenReturn(true); + when(deltaEvent.asMessageDelta()).thenReturn(messageDelta); + when(messageDelta.usage()).thenReturn(deltaUsage); + when(deltaUsage.inputTokens()).thenReturn(Optional.empty()); + when(deltaUsage.cacheReadInputTokens()).thenReturn(Optional.empty()); + when(deltaUsage.cacheCreationInputTokens()).thenReturn(Optional.empty()); + when(deltaUsage.outputTokens()).thenReturn(5L); + + StepVerifier.create( + AnthropicResponseParser.parseStreamEvents( + Flux.just(startEvent, deltaEvent), Instant.now())) + .assertNext( + response -> { + assertEquals("msg-cache", response.getId()); + assertTrue(response.getContent().isEmpty()); + assertEquals(100, response.getUsage().getInputTokens()); + assertEquals(5, response.getUsage().getOutputTokens()); + assertEquals(70, response.getUsage().getCachedTokens()); + assertEquals(20, response.getUsage().getCacheCreationInputTokens()); + }) + .verifyComplete(); + } + @Test void testParseStreamEventThinkingDelta() throws Exception { RawContentBlockDeltaEvent deltaEvent = diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java index 647cb99be5..7f9deef654 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java @@ -164,8 +164,12 @@ protected Flux doStream0( if (isDashScopeCompatibleBaseUrl(baseUrl)) { openAIFormatter.applyDashScopeCacheControl( request.getMessages(), automaticCacheControl); - } else if (automaticCacheControl) { - openAIFormatter.applyCacheControl(request.getMessages()); + } else if (isOfficialOpenAIBaseUrl(baseUrl)) { + openAIFormatter.applyOpenAIPromptCache(request); + } else { + // The OpenAI-compatible surface does not imply a provider's cache protocol. + // Never leak AgentScope's internal legacy marker to an unknown endpoint. + openAIFormatter.clearLegacyCacheControl(request.getMessages()); } } @@ -226,6 +230,18 @@ static boolean isDashScopeCompatibleBaseUrl(String baseUrl) { } } + static boolean isOfficialOpenAIBaseUrl(String baseUrl) { + if (baseUrl == null || baseUrl.isBlank()) { + return true; + } + try { + String host = URI.create(baseUrl).getHost(); + return host != null && host.equalsIgnoreCase("api.openai.com"); + } catch (IllegalArgumentException e) { + return false; + } + } + /** * Gets the model name for logging and identification. * diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java index 738e1f8369..0f9491d830 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java @@ -70,6 +70,10 @@ public class OpenAIContentPart { @JsonProperty("cache_control") private Map cacheControl; + /** Explicit prompt cache breakpoint used by the official OpenAI API. */ + @JsonProperty("prompt_cache_breakpoint") + private Map promptCacheBreakpoint; + public OpenAIContentPart() {} public String getType() { @@ -120,6 +124,14 @@ public void setCacheControl(Map cacheControl) { this.cacheControl = cacheControl; } + public Map getPromptCacheBreakpoint() { + return promptCacheBreakpoint; + } + + public void setPromptCacheBreakpoint(Map promptCacheBreakpoint) { + this.promptCacheBreakpoint = promptCacheBreakpoint; + } + /** * Create a text content part. * @@ -224,6 +236,11 @@ public Builder cacheControl(Map cacheControl) { return this; } + public Builder promptCacheBreakpoint(Map promptCacheBreakpoint) { + part.setPromptCacheBreakpoint(promptCacheBreakpoint); + return this; + } + public OpenAIContentPart build() { OpenAIContentPart result = part; part = new OpenAIContentPart(); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java index 6a711355fb..13535480d1 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java @@ -108,6 +108,9 @@ public static class PromptTokensDetails { @JsonProperty("audio_tokens") private Integer audioTokens; + @JsonProperty("cache_write_tokens") + private Integer cacheWriteTokens; + public Integer getCachedTokens() { return cachedTokens; } @@ -123,6 +126,14 @@ public Integer getAudioTokens() { public void setAudioTokens(Integer audioTokens) { this.audioTokens = audioTokens; } + + public Integer getCacheWriteTokens() { + return cacheWriteTokens; + } + + public void setCacheWriteTokens(Integer cacheWriteTokens) { + this.cacheWriteTokens = cacheWriteTokens; + } } /** diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java index 2111819d8e..1ead4b43dd 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java @@ -45,6 +45,8 @@ public abstract class OpenAIBaseFormatter extends AbstractBaseFormatter { private static final Map EPHEMERAL_CACHE_CONTROL = Map.of("type", "ephemeral"); + private static final Map EXPLICIT_PROMPT_CACHE_BREAKPOINT = + Map.of("mode", "explicit"); protected final OpenAIMessageConverter messageConverter; protected final OpenAIResponseParser responseParser; @@ -233,6 +235,88 @@ public void applyDashScopeCacheControl(List messages, boolean aut } } + /** + * Normalize manually marked messages to the official OpenAI explicit prompt-cache protocol. + * + *

OpenAI enables automatic prompt caching by default, so this method never invents + * breakpoints for {@link GenerateOptions#getCacheControl() cacheControl=true}. It only + * migrates explicit AgentScope metadata (temporarily represented by legacy + * {@code cache_control}) to a content-part {@code prompt_cache_breakpoint}, and enables + * explicit mode at request level. + * + * @param request request to normalize + */ + public void applyOpenAIPromptCache(OpenAIRequest request) { + if (request == null || request.getMessages() == null) { + return; + } + + for (OpenAIMessage message : request.getMessages()) { + boolean legacyExplicit = message.getCacheControl() != null; + List parts = message.getContentAsList(); + if (parts != null) { + for (OpenAIContentPart part : parts) { + if (part.getCacheControl() != null) { + legacyExplicit = true; + part.setCacheControl(null); + } + } + } + + if (legacyExplicit && !hasPromptCacheBreakpoint(message)) { + parts = ensureContentArray(message); + if (parts.isEmpty()) { + throw new IllegalArgumentException( + "Cannot apply prompt_cache_breakpoint to a message without cacheable" + + " content"); + } + parts.get(parts.size() - 1) + .setPromptCacheBreakpoint(EXPLICIT_PROMPT_CACHE_BREAKPOINT); + } + message.setCacheControl(null); + } + + int markerCount = countPromptCacheBreakpoints(request.getMessages()); + if (markerCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "OpenAI supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " prompt_cache_breakpoint markers, but got " + + markerCount); + } + if (markerCount > 0) { + Map promptCacheOptions = new java.util.LinkedHashMap<>(); + Object configured = + request.getExtraParams() != null + ? request.getExtraParams().get("prompt_cache_options") + : null; + if (configured instanceof Map configuredMap) { + configuredMap.forEach( + (key, value) -> { + if (key instanceof String stringKey) { + promptCacheOptions.put(stringKey, value); + } + }); + } + promptCacheOptions.put("mode", "explicit"); + request.addExtraParam("prompt_cache_options", promptCacheOptions); + } + } + + /** Remove AgentScope's internal legacy markers before calling an unknown compatible API. */ + public void clearLegacyCacheControl(List messages) { + if (messages == null) { + return; + } + for (OpenAIMessage message : messages) { + message.setCacheControl(null); + List parts = message.getContentAsList(); + if (parts != null) { + parts.forEach(part -> part.setCacheControl(null)); + } + } + } + static void applyDashScopeCacheControlToContentBlock(OpenAIMessage message) { if (hasContentBlockCacheControl(message)) { message.setCacheControl(null); @@ -287,6 +371,27 @@ private static boolean hasContentBlockCacheControl(OpenAIMessage message) { return parts != null && parts.stream().anyMatch(part -> part.getCacheControl() != null); } + private static boolean hasPromptCacheBreakpoint(OpenAIMessage message) { + List parts = message.getContentAsList(); + return parts != null + && parts.stream().anyMatch(part -> part.getPromptCacheBreakpoint() != null); + } + + private static int countPromptCacheBreakpoints(List messages) { + int count = 0; + for (OpenAIMessage message : messages) { + List parts = message.getContentAsList(); + if (parts != null) { + count += + (int) + parts.stream() + .filter(part -> part.getPromptCacheBreakpoint() != null) + .count(); + } + } + return count; + } + private static int countContentBlockCacheMarkers(List messages) { int count = 0; for (OpenAIMessage message : messages) { diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java index 1e4e940238..ea2dceeba6 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java @@ -90,6 +90,14 @@ private long getSafeCachedTokens(OpenAIUsage usage) { return cachedTokens != null ? cachedTokens : 0; } + private long getSafeCacheWriteTokens(OpenAIUsage usage) { + if (usage == null || usage.getPromptTokensDetails() == null) { + return 0; + } + Integer cacheWriteTokens = usage.getPromptTokensDetails().getCacheWriteTokens(); + return cacheWriteTokens != null ? cacheWriteTokens : 0; + } + public OpenAIResponseParser() {} /** @@ -128,6 +136,8 @@ protected ChatResponse parseCompletionResponse(OpenAIResponse response, Instant .inputTokens((int) getSafePromptTokens(openAIUsage)) .outputTokens((int) getSafeCompletionTokens(openAIUsage)) .cachedTokens((int) getSafeCachedTokens(openAIUsage)) + .cacheCreationInputTokens( + (int) getSafeCacheWriteTokens(openAIUsage)) .time( Duration.between(startTime, Instant.now()).toMillis() / 1000.0) @@ -372,6 +382,8 @@ protected ChatResponse parseChunkResponse(OpenAIResponse response, Instant start ? openAIUsage.getCompletionTokens() : 0) .cachedTokens((int) getSafeCachedTokens(openAIUsage)) + .cacheCreationInputTokens( + (int) getSafeCacheWriteTokens(openAIUsage)) .time( Duration.between(startTime, Instant.now()).toMillis() / 1000.0) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java index a347a8a825..2677fb699a 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java @@ -131,6 +131,12 @@ void testDashScopeCompatibleBaseUrlDetection() { assertFalse(OpenAIChatModel.isDashScopeCompatibleBaseUrl("https://api.openai.com/v1")); assertFalse(OpenAIChatModel.isDashScopeCompatibleBaseUrl(null)); assertFalse(OpenAIChatModel.isDashScopeCompatibleBaseUrl("not a valid URL")); + + assertTrue(OpenAIChatModel.isOfficialOpenAIBaseUrl(null)); + assertTrue(OpenAIChatModel.isOfficialOpenAIBaseUrl("")); + assertTrue(OpenAIChatModel.isOfficialOpenAIBaseUrl("https://api.openai.com/v1")); + assertFalse( + OpenAIChatModel.isOfficialOpenAIBaseUrl("https://gateway.example.com/openai/v1")); } @Test @@ -356,8 +362,8 @@ void testBuildModelWithEndpointPath() throws Exception { } @Test - @DisplayName("Should apply cache_control to request when cacheControl option is enabled") - void testCacheControlApplied() throws Exception { + @DisplayName("Should not guess a cache protocol for an unknown compatible endpoint") + void testUnknownCompatibleCacheControlIsNotApplied() throws Exception { String responseJson = """ { @@ -405,12 +411,12 @@ void testCacheControlApplied() throws Exception { RecordedRequest request = mockServer.takeRequest(1, TimeUnit.SECONDS); assertNotNull(request); String body = request.getBody().readUtf8(); - assertTrue( + assertFalse( body.contains("\"cache_control\""), - "Request body should contain cache_control: " + body); - assertTrue( - body.contains("\"ephemeral\""), - "Request body should contain ephemeral cache type: " + body); + "Unknown compatible endpoint must not receive cache_control: " + body); + assertFalse( + body.contains("\"prompt_cache_breakpoint\""), + "Unknown compatible endpoint must not receive OpenAI cache fields: " + body); } @Test diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java index e8ae81c6f1..0a0f58fc1b 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java @@ -27,6 +27,7 @@ import io.agentscope.core.util.JsonUtils; import io.agentscope.extensions.model.openai.dto.OpenAIContentPart; import io.agentscope.extensions.model.openai.dto.OpenAIMessage; +import io.agentscope.extensions.model.openai.dto.OpenAIRequest; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -244,6 +245,161 @@ void systemMessageMetadata() { } } + @Nested + @DisplayName("Official OpenAI prompt caching") + class OfficialOpenAITest { + + @Test + @DisplayName("should serialize explicit metadata using the official protocol") + void exactExplicitProtocolJson() { + Msg msg = + Msg.builder() + .role(MsgRole.USER) + .textContent("Pinned context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + OpenAIRequest request = + OpenAIRequest.builder() + .model("gpt-5.6") + .messages(formatter.format(List.of(msg))) + .stream(false) + .build(); + + formatter.applyOpenAIPromptCache(request); + + JsonNode json = toJsonTree(request); + JsonNode message = json.path("messages").get(0); + assertFalse(message.has("cache_control")); + assertEquals( + "explicit", + message.path("content") + .get(0) + .path("prompt_cache_breakpoint") + .path("mode") + .asText()); + assertFalse(message.path("content").get(0).has("cache_control")); + assertEquals("explicit", json.path("prompt_cache_options").path("mode").asText()); + } + + @Test + @DisplayName("should rely on server automatic caching when there is no explicit marker") + void noInventedAutomaticMarkers() { + OpenAIRequest request = + OpenAIRequest.builder() + .model("gpt-5.6") + .messages( + new ArrayList<>( + List.of( + OpenAIMessage.builder() + .role("system") + .content("System") + .build(), + OpenAIMessage.builder() + .role("user") + .content("Question") + .build()))) + .build(); + + formatter.applyOpenAIPromptCache(request); + + JsonNode json = toJsonTree(request); + assertFalse(json.has("prompt_cache_options")); + assertFalse(json.toString().contains("prompt_cache_breakpoint")); + assertFalse(json.toString().contains("cache_control")); + } + + @Test + @DisplayName("should preserve other prompt cache options and force explicit mode") + void mergesConfiguredPromptCacheOptions() { + OpenAIMessage message = + OpenAIMessage.builder() + .role("user") + .content("Context") + .cacheControl(EPHEMERAL) + .build(); + OpenAIRequest request = + OpenAIRequest.builder() + .model("gpt-5.6") + .messages(List.of(message)) + .extraParam( + "prompt_cache_options", + Map.of("mode", "automatic", "ttl", "24h")) + .build(); + + formatter.applyOpenAIPromptCache(request); + + JsonNode json = toJsonTree(request); + assertEquals("explicit", json.path("prompt_cache_options").path("mode").asText()); + assertEquals("24h", json.path("prompt_cache_options").path("ttl").asText()); + } + + @Test + @DisplayName("should place a multimodal breakpoint on the final content part") + void multimodalFinalPart() { + OpenAIMessage message = + OpenAIMessage.builder() + .role("user") + .content( + new ArrayList<>( + List.of( + OpenAIContentPart.text("Look"), + OpenAIContentPart.imageUrl( + "https://example.com/image.png")))) + .cacheControl(EPHEMERAL) + .build(); + OpenAIRequest request = OpenAIRequest.builder().messages(List.of(message)).build(); + + formatter.applyOpenAIPromptCache(request); + + assertNull(message.getContentAsList().get(0).getPromptCacheBreakpoint()); + assertEquals( + Map.of("mode", "explicit"), + message.getContentAsList().get(1).getPromptCacheBreakpoint()); + } + + @Test + @DisplayName("should reject more than four explicit breakpoints") + void tooManyBreakpoints() { + List messages = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + messages.add( + OpenAIMessage.builder() + .role("user") + .content("Context " + i) + .cacheControl(EPHEMERAL) + .build()); + } + OpenAIRequest request = OpenAIRequest.builder().messages(messages).build(); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyOpenAIPromptCache(request)); + } + + @Test + @DisplayName("should clear legacy fields for unknown compatible providers") + void clearUnknownCompatibleMarkers() { + OpenAIContentPart part = + OpenAIContentPart.builder() + .type("text") + .text("Context") + .cacheControl(EPHEMERAL) + .build(); + OpenAIMessage message = + OpenAIMessage.builder() + .role("user") + .content(List.of(part)) + .cacheControl(EPHEMERAL) + .build(); + + formatter.clearLegacyCacheControl(List.of(message)); + + JsonNode json = toJsonTree(message); + assertFalse(json.toString().contains("cache_control")); + assertFalse(json.toString().contains("prompt_cache_breakpoint")); + } + } + @Nested @DisplayName("DashScope-compatible cache_control") class DashScopeCompatibleTest { @@ -350,4 +506,9 @@ private JsonNode toJsonTree(OpenAIMessage message) { .fromJson(JsonUtils.getJsonCodec().toJson(message), JsonNode.class); } } + + private JsonNode toJsonTree(Object value) { + return JsonUtils.getJsonCodec() + .fromJson(JsonUtils.getJsonCodec().toJson(value), JsonNode.class); + } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java index 9b909b0773..659662e2b0 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java @@ -937,6 +937,7 @@ void testParseResponseWithCachedTokens() { usage.setCompletionTokens(20); OpenAIUsage.PromptTokensDetails details = new OpenAIUsage.PromptTokensDetails(); details.setCachedTokens(80); + details.setCacheWriteTokens(12); usage.setPromptTokensDetails(details); response.setUsage(usage); @@ -958,6 +959,7 @@ void testParseResponseWithCachedTokens() { assertEquals(100, result.getUsage().getInputTokens()); assertEquals(20, result.getUsage().getOutputTokens()); assertEquals(80, result.getUsage().getCachedTokens()); + assertEquals(12, result.getUsage().getCacheCreationInputTokens()); } @Test @@ -1032,6 +1034,7 @@ void testParseChunkResponseWithCachedTokens() { usage.setCompletionTokens(50); OpenAIUsage.PromptTokensDetails details = new OpenAIUsage.PromptTokensDetails(); details.setCachedTokens(64); + details.setCacheWriteTokens(7); usage.setPromptTokensDetails(details); response.setUsage(usage); @@ -1053,6 +1056,7 @@ void testParseChunkResponseWithCachedTokens() { assertEquals(100, result.getUsage().getInputTokens()); assertEquals(50, result.getUsage().getOutputTokens()); assertEquals(64, result.getUsage().getCachedTokens()); + assertEquals(7, result.getUsage().getCacheCreationInputTokens()); } @Test From 45ae6bfe5ff40e701c013454541015397bff65be Mon Sep 17 00:00:00 2001 From: guslegend <1670547022@qq.com> Date: Tue, 25 Aug 2026 14:20:16 +0800 Subject: [PATCH 04/10] feat(model): complete Gemini prompt cache integration --- .../java/io/agentscope/core/ReActAgent.java | 3 + .../java/io/agentscope/core/message/Msg.java | 2 + .../agent/ReActAgentStructuredOutputTest.java | 14 +++- .../message/MsgUsageSerializationTest.java | 28 +++++++ .../model/gemini/GeminiChatModel.java | 44 ++++++----- .../gemini/formatter/GeminiChatFormatter.java | 58 +++++++++++++++ .../formatter/GeminiResponseParser.java | 7 +- .../model/gemini/GeminiChatModelTest.java | 38 ++++++++++ .../formatter/GeminiChatFormatterTest.java | 74 +++++++++++++++++++ .../formatter/GeminiResponseParserTest.java | 11 ++- .../adapter/strategy/AguiStreamContext.java | 11 ++- .../ModelCallUsageEventConverter.java | 1 + .../agui/adapter/AguiAgentAdapterV2Test.java | 19 +++-- docs/v2/en/blogs/usecases/aidc-logistics.md | 2 +- docs/v2/en/blogs/usecases/logistics.md | 2 +- docs/v2/en/integration/model/gemini.md | 34 +++++++++ docs/v2/zh/blogs/usecases/aidc-logistics.md | 2 +- docs/v2/zh/blogs/usecases/logistics.md | 2 +- docs/v2/zh/integration/model/gemini.md | 34 +++++++++ 19 files changed, 343 insertions(+), 43 deletions(-) diff --git a/agentscope-core/src/main/java/io/agentscope/core/ReActAgent.java b/agentscope-core/src/main/java/io/agentscope/core/ReActAgent.java index 2fd0eef59a..653956c069 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/ReActAgent.java +++ b/agentscope-core/src/main/java/io/agentscope/core/ReActAgent.java @@ -1448,6 +1448,7 @@ private ChatUsage collectAggregatedUsage(AgentState agentState) { int totalInput = 0; int totalOutput = 0; int totalCached = 0; + int totalCacheCreation = 0; double totalTime = 0; boolean hasUsage = false; for (Msg msg : agentState.getContext()) { @@ -1458,6 +1459,7 @@ private ChatUsage collectAggregatedUsage(AgentState agentState) { totalInput += usage.getInputTokens(); totalOutput += usage.getOutputTokens(); totalCached += usage.getCachedTokens(); + totalCacheCreation += usage.getCacheCreationInputTokens(); totalTime += usage.getTime(); } } @@ -1467,6 +1469,7 @@ private ChatUsage collectAggregatedUsage(AgentState agentState) { .inputTokens(totalInput) .outputTokens(totalOutput) .cachedTokens(totalCached) + .cacheCreationInputTokens(totalCacheCreation) .time(totalTime) .build() : null; diff --git a/agentscope-core/src/main/java/io/agentscope/core/message/Msg.java b/agentscope-core/src/main/java/io/agentscope/core/message/Msg.java index 82df1e01cf..1cded18b74 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/message/Msg.java +++ b/agentscope-core/src/main/java/io/agentscope/core/message/Msg.java @@ -590,6 +590,8 @@ public ChatUsage getChatUsage() { ChatUsage.builder() .inputTokens(toInt(map.get("inputTokens"))) .outputTokens(toInt(map.get("outputTokens"))) + .cachedTokens(toInt(map.get("cachedTokens"))) + .cacheCreationInputTokens(toInt(map.get("cacheCreationInputTokens"))) .time(toDouble(map.get("time"))) .build(); metadata.put(MessageMetadataKeys.CHAT_USAGE, chatUsage); diff --git a/agentscope-core/src/test/java/io/agentscope/core/agent/ReActAgentStructuredOutputTest.java b/agentscope-core/src/test/java/io/agentscope/core/agent/ReActAgentStructuredOutputTest.java index e6a1a49957..a537d6a1e5 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/agent/ReActAgentStructuredOutputTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/agent/ReActAgentStructuredOutputTest.java @@ -388,7 +388,14 @@ void testStructuredOutputPreservesChatUsage() { .toJson( toolInput)) .build())) - .usage(new ChatUsage(100, 50, 1.5)) + .usage( + ChatUsage.builder() + .inputTokens(100) + .outputTokens(50) + .cachedTokens(40) + .cacheCreationInputTokens(25) + .time(1.5) + .build()) .build()); } else { return List.of( @@ -435,6 +442,11 @@ void testStructuredOutputPreservesChatUsage() { assertNotNull(usage, "ChatUsage should be preserved after structured output compression"); assertEquals(100, usage.getInputTokens(), "Input tokens should be preserved"); assertEquals(50, usage.getOutputTokens(), "Output tokens should be preserved"); + assertEquals(40, usage.getCachedTokens(), "Cached tokens should be preserved"); + assertEquals( + 25, + usage.getCacheCreationInputTokens(), + "Cache creation tokens should be preserved"); assertEquals(1.5, usage.getTime(), 0.01, "Time should be preserved"); } diff --git a/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java b/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java index 4ed34b52cd..e68d7dbba2 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java @@ -144,6 +144,34 @@ void getChatUsageFallsBackToMetadata() { assertEquals(300, retrieved.getInputTokens()); } + @Test + void getChatUsageMapFallbackPreservesPromptCacheUsage() { + Msg msg = + Msg.builder() + .name("assistant") + .role(MsgRole.ASSISTANT) + .textContent("reply") + .metadata( + java.util.Map.of( + MessageMetadataKeys.CHAT_USAGE, + java.util.Map.of( + "inputTokens", 300, + "outputTokens", 150, + "cachedTokens", 120, + "cacheCreationInputTokens", 30, + "time", 3.0))) + .build(); + + ChatUsage retrieved = msg.getChatUsage(); + + assertNotNull(retrieved); + assertEquals(300, retrieved.getInputTokens()); + assertEquals(150, retrieved.getOutputTokens()); + assertEquals(120, retrieved.getCachedTokens()); + assertEquals(30, retrieved.getCacheCreationInputTokens()); + assertEquals(3.0, retrieved.getTime()); + } + @Test void usageDeserializesFromJsonWithoutField() { String json = diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/GeminiChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/GeminiChatModel.java index 9f1f08119a..23ef40e2f3 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/GeminiChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/GeminiChatModel.java @@ -59,6 +59,7 @@ *

  • Multi-agent conversation with history merging
    • *
  • Vision capabilities (images, audio, video)
    • *
  • Thinking mode (extended reasoning)
    • + *
  • Implicit caching and explicit cached content resource references
    • * */ public class GeminiChatModel extends ChatModelBase { @@ -250,29 +251,11 @@ protected Flux doStream0( return Flux.defer( () -> { try { - // Build generate content config - GenerateContentConfig.Builder configBuilder = - GenerateContentConfig.builder(); - // Use formatter to convert Msg to Gemini // Content List formattedMessages = formatter.format(messages); - - // Add tools if provided - if (tools != null && !tools.isEmpty()) { - formatter.applyTools(configBuilder, tools); - - // Apply tool choice if present - if (options != null && options.getToolChoice() != null) { - formatter.applyToolChoice( - configBuilder, options.getToolChoice()); - } - } - - // Apply generation options via formatter - formatter.applyOptions(configBuilder, options, defaultOptions); - - GenerateContentConfig config = configBuilder.build(); + GenerateContentConfig config = + buildGenerateContentConfig(tools, options); // Choose API based on streaming flag if (streamEnabled) { @@ -325,6 +308,27 @@ protected Flux doStream0( .subscribeOn(Schedulers.boundedElastic()); } + GenerateContentConfig buildGenerateContentConfig( + List tools, GenerateOptions options) { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + + // Apply options first because an explicit cached content resource also fixes tools and + // toolConfig. Gemini rejects those fields when cachedContent is set. + formatter.applyOptions(configBuilder, options, defaultOptions); + boolean usesCachedContent = configBuilder.build().cachedContent().isPresent(); + + if (!usesCachedContent && tools != null && !tools.isEmpty()) { + formatter.applyTools(configBuilder, tools); + GenerateOptions effectiveOptions = + GenerateOptions.mergeOptions(options, defaultOptions); + if (effectiveOptions != null && effectiveOptions.getToolChoice() != null) { + formatter.applyToolChoice(configBuilder, effectiveOptions.getToolChoice()); + } + } + + return configBuilder.build(); + } + @Override public String getModelName() { return modelName; diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java index ea37916c01..cbf84ffabe 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java @@ -29,6 +29,8 @@ import io.agentscope.core.model.ToolSchema; import java.time.Instant; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; import java.util.function.Function; @@ -53,6 +55,12 @@ public class GeminiChatFormatter extends AbstractBaseFormatter< Content, GenerateContentResponse, GenerateContentConfig.Builder> { + /** Canonical additional body parameter for referencing a Gemini cached content resource. */ + public static final String CACHED_CONTENT_PARAM = "cachedContent"; + + /** Snake-case alias accepted for consistency with other Google Gen AI client conventions. */ + public static final String CACHED_CONTENT_PARAM_ALIAS = "cached_content"; + private final GeminiMessageConverter messageConverter; private final GeminiResponseParser responseParser; private final GeminiToolsHelper toolsHelper; @@ -127,6 +135,56 @@ public void applyOptions( thinkingConfigBuilder.thinkingBudget(thinkingBudget); configBuilder.thinkingConfig(thinkingConfigBuilder.build()); } + + String cachedContent = resolveCachedContent(options, defaultOptions); + if (cachedContent != null) { + configBuilder.cachedContent(cachedContent); + } + } + + private String resolveCachedContent(GenerateOptions options, GenerateOptions defaultOptions) { + CacheReference requestReference = cacheReference(options); + CacheReference defaultReference = cacheReference(defaultOptions); + Object value = requestReference.present ? requestReference.value : defaultReference.value; + + if (value == null) { + return null; + } + if (!(value instanceof String cacheName) || cacheName.isBlank()) { + throw new IllegalArgumentException( + "Gemini cachedContent must be a non-blank String resource name"); + } + return cacheName; + } + + private CacheReference cacheReference(GenerateOptions options) { + if (options == null) { + return CacheReference.absent(); + } + + Map params = options.getAdditionalBodyParams(); + boolean hasCanonical = params.containsKey(CACHED_CONTENT_PARAM); + boolean hasAlias = params.containsKey(CACHED_CONTENT_PARAM_ALIAS); + Object canonical = params.get(CACHED_CONTENT_PARAM); + Object alias = params.get(CACHED_CONTENT_PARAM_ALIAS); + + if (hasCanonical && hasAlias && !Objects.equals(canonical, alias)) { + throw new IllegalArgumentException( + "Gemini cachedContent and cached_content must not specify different values"); + } + if (hasCanonical) { + return new CacheReference(true, canonical); + } + if (hasAlias) { + return new CacheReference(true, alias); + } + return CacheReference.absent(); + } + + private record CacheReference(boolean present, Object value) { + private static CacheReference absent() { + return new CacheReference(false, null); + } } /** diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParser.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParser.java index 605844f7c2..df8d7447b8 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParser.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParser.java @@ -95,13 +95,8 @@ public ChatResponse parseResponse(GenerateContentResponse response, Instant star GenerateContentResponseUsageMetadata metadata = response.usageMetadata().get(); int inputTokens = metadata.promptTokenCount().orElse(0); + int outputTokens = metadata.candidatesTokenCount().orElse(0); int cachedTokens = metadata.cachedContentTokenCount().orElse(0); - int totalOutputTokens = metadata.candidatesTokenCount().orElse(0); - int thinkingTokens = metadata.thoughtsTokenCount().orElse(0); - - // Output tokens exclude thinking tokens (following DashScope behavior) - // In Gemini, candidatesTokenCount includes thinking, so we subtract it - int outputTokens = totalOutputTokens - thinkingTokens; usage = ChatUsage.builder() diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/GeminiChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/GeminiChatModelTest.java index f78b03232e..e74735e9d7 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/GeminiChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/GeminiChatModelTest.java @@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.genai.types.ClientOptions; +import com.google.genai.types.GenerateContentConfig; import com.google.genai.types.HttpOptions; import com.google.genai.types.ProxyOptions; import io.agentscope.core.message.Msg; @@ -283,6 +284,43 @@ void testGenerateOptionsConfiguration() { assertNotNull(modelWithOptions); } + @Test + @DisplayName("Should reference cached content without resending tools or tool config") + void testCachedContentRequestSuppressesCachedToolConfiguration() { + GeminiChatModel model = + GeminiChatModel.builder().apiKey(mockApiKey).modelName("gemini-2.5-flash").build(); + ToolSchema tool = + ToolSchema.builder() + .name("search") + .description("Search") + .parameters(java.util.Map.of("type", "object")) + .build(); + GenerateOptions options = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/cache-123") + .toolChoice(new ToolChoice.Required()) + .build(); + + GenerateContentConfig config = model.buildGenerateContentConfig(List.of(tool), options); + + assertEquals("cachedContents/cache-123", config.cachedContent().orElseThrow()); + assertTrue(config.tools().isEmpty()); + assertTrue(config.toolConfig().isEmpty()); + } + + @Test + @DisplayName("Should leave Gemini implicit caching to the provider") + void testCacheControlDoesNotCreateExplicitCachedContentReference() { + GeminiChatModel model = + GeminiChatModel.builder().apiKey(mockApiKey).modelName("gemini-2.5-flash").build(); + + GenerateContentConfig config = + model.buildGenerateContentConfig( + List.of(), GenerateOptions.builder().cacheControl(true).build()); + + assertTrue(config.cachedContent().isEmpty()); + } + @Test @DisplayName("Should build with minimal parameters") void testMinimalBuilder() { diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java index b783a02c7b..4caae34784 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java @@ -17,6 +17,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.genai.types.Content; @@ -94,6 +95,79 @@ void testApplyOptions() { assertEquals(0.3f, config.presencePenalty().get(), 0.001f); } + @Test + void testApplyCachedContentFromCanonicalParameter() { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + GenerateOptions options = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/cache-123") + .build(); + + formatter.applyOptions(configBuilder, options, null); + + assertEquals( + "cachedContents/cache-123", configBuilder.build().cachedContent().orElseThrow()); + } + + @Test + void testApplyCachedContentFromSnakeCaseAlias() { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + GenerateOptions options = + GenerateOptions.builder() + .additionalBodyParam("cached_content", "cachedContents/cache-123") + .build(); + + formatter.applyOptions(configBuilder, options, null); + + assertEquals( + "cachedContents/cache-123", configBuilder.build().cachedContent().orElseThrow()); + } + + @Test + void testRequestCachedContentOverridesDefaultAlias() { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + GenerateOptions defaults = + GenerateOptions.builder() + .additionalBodyParam("cached_content", "cachedContents/default") + .build(); + GenerateOptions request = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/request") + .build(); + + formatter.applyOptions(configBuilder, request, defaults); + + assertEquals("cachedContents/request", configBuilder.build().cachedContent().orElseThrow()); + } + + @Test + void testRejectConflictingCachedContentAliases() { + GenerateOptions options = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/one") + .additionalBodyParam("cached_content", "cachedContents/two") + .build(); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyOptions(GenerateContentConfig.builder(), options, null)); + } + + @Test + void testRejectInvalidCachedContentReference() { + GenerateOptions blank = + GenerateOptions.builder().additionalBodyParam("cachedContent", " ").build(); + GenerateOptions nonString = + GenerateOptions.builder().additionalBodyParam("cachedContent", 123).build(); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyOptions(GenerateContentConfig.builder(), blank, null)); + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyOptions(GenerateContentConfig.builder(), nonString, null)); + } + @Test void testApplyTools() { GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParserTest.java index 0da20af442..08fb6c60c4 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParserTest.java @@ -216,8 +216,9 @@ void testParseUsageMetadata() { GenerateContentResponseUsageMetadata usageMetadata = GenerateContentResponseUsageMetadata.builder() .promptTokenCount(100) - .candidatesTokenCount(60) // Includes thinking - .thoughtsTokenCount(10) // Thinking tokens + .cachedContentTokenCount(60) + .candidatesTokenCount(50) + .thoughtsTokenCount(10) .totalTokenCount(160) .build(); @@ -238,9 +239,13 @@ void testParseUsageMetadata() { // Input tokens = promptTokenCount assertEquals(100, usage.getInputTokens()); - // Output tokens = candidatesTokenCount - thoughtsTokenCount + // Output tokens = candidatesTokenCount; thoughts are reported separately by Gemini assertEquals(50, usage.getOutputTokens()); + // Cached tokens are a subset of promptTokenCount + assertEquals(60, usage.getCachedTokens()); + assertEquals(0, usage.getCacheCreationInputTokens()); + // Time should be > 0 assertTrue(usage.getTime() >= 0); } diff --git a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/AguiStreamContext.java b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/AguiStreamContext.java index d1eb74e315..3f77759b01 100644 --- a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/AguiStreamContext.java +++ b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/AguiStreamContext.java @@ -341,30 +341,39 @@ static final class TokenUsageAccumulator { private long cumulativeInputTokens; private long cumulativeOutputTokens; private long cumulativeCachedTokens; + private long cumulativeCacheCreationInputTokens; private double cumulativeTime; TokenUsageSnapshot add(ChatUsage usage) { cumulativeInputTokens += usage.getInputTokens(); cumulativeOutputTokens += usage.getOutputTokens(); cumulativeCachedTokens += usage.getCachedTokens(); + cumulativeCacheCreationInputTokens += usage.getCacheCreationInputTokens(); cumulativeTime += usage.getTime(); return new TokenUsageSnapshot( new TokenUsage( usage.getInputTokens(), usage.getOutputTokens(), usage.getCachedTokens(), + usage.getCacheCreationInputTokens(), usage.getTime()), new TokenUsage( cumulativeInputTokens, cumulativeOutputTokens, cumulativeCachedTokens, + cumulativeCacheCreationInputTokens, cumulativeTime)); } } record TokenUsageSnapshot(TokenUsage delta, TokenUsage cumulative) {} - record TokenUsage(long inputTokens, long outputTokens, long cachedTokens, double time) { + record TokenUsage( + long inputTokens, + long outputTokens, + long cachedTokens, + long cacheCreationInputTokens, + double time) { long totalTokens() { return inputTokens + outputTokens; diff --git a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/ModelCallUsageEventConverter.java b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/ModelCallUsageEventConverter.java index 6ac475e1d8..7348bc4f91 100644 --- a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/ModelCallUsageEventConverter.java +++ b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/ModelCallUsageEventConverter.java @@ -67,6 +67,7 @@ private static Map tokenUsageMap(AguiStreamContext.TokenUsage us value.put("inputTokens", usage.inputTokens()); value.put("outputTokens", usage.outputTokens()); value.put("cachedTokens", usage.cachedTokens()); + value.put("cacheCreationInputTokens", usage.cacheCreationInputTokens()); value.put("totalTokens", usage.totalTokens()); value.put("time", usage.time()); return value; diff --git a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/test/java/io/agentscope/core/agui/adapter/AguiAgentAdapterV2Test.java b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/test/java/io/agentscope/core/agui/adapter/AguiAgentAdapterV2Test.java index 9435ec1edc..c90ff89955 100644 --- a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/test/java/io/agentscope/core/agui/adapter/AguiAgentAdapterV2Test.java +++ b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/test/java/io/agentscope/core/agui/adapter/AguiAgentAdapterV2Test.java @@ -1132,14 +1132,15 @@ void testEnabledTokenUsageEmitsCustomEventWithDeltaAndCumulativeUsage() { List events = runReActEvents( config, - new ModelCallEndEvent("reply-usage", new ChatUsage(100, 20, 40, 0.8))); + new ModelCallEndEvent( + "reply-usage", new ChatUsage(100, 20, 40, 25, 0.8))); assertEquals(List.of(AguiEventType.CUSTOM), types(events)); AguiEvent.Custom usageEvent = assertCustomEvent(events.get(0), "token_usage"); Map value = customValue(usageEvent); - assertUsage(value.get("delta"), 100L, 20L, 40L, 120L, 0.8); - assertUsage(value.get("cumulative"), 100L, 20L, 40L, 120L, 0.8); + assertUsage(value.get("delta"), 100L, 20L, 40L, 25L, 120L, 0.8); + assertUsage(value.get("cumulative"), 100L, 20L, 40L, 25L, 120L, 0.8); assertEquals(Map.of("replyId", "reply-usage"), value.get("modelCall")); } @@ -1149,8 +1150,8 @@ void testEnabledTokenUsageAccumulatesAcrossModelCalls() { List events = runReActEvents( config, - new ModelCallEndEvent("reply-1", new ChatUsage(100, 20, 40, 0.8)), - new ModelCallEndEvent("reply-2", new ChatUsage(50, 30, 10, 1.2))); + new ModelCallEndEvent("reply-1", new ChatUsage(100, 20, 40, 25, 0.8)), + new ModelCallEndEvent("reply-2", new ChatUsage(50, 30, 10, 15, 1.2))); assertEquals(List.of(AguiEventType.CUSTOM, AguiEventType.CUSTOM), types(events)); @@ -1158,9 +1159,9 @@ void testEnabledTokenUsageAccumulatesAcrossModelCalls() { customValue(assertCustomEvent(events.get(0), "token_usage")); Map secondValue = customValue(assertCustomEvent(events.get(1), "token_usage")); - assertUsage(firstValue.get("cumulative"), 100L, 20L, 40L, 120L, 0.8); - assertUsage(secondValue.get("delta"), 50L, 30L, 10L, 80L, 1.2); - assertUsage(secondValue.get("cumulative"), 150L, 50L, 50L, 200L, 2.0); + assertUsage(firstValue.get("cumulative"), 100L, 20L, 40L, 25L, 120L, 0.8); + assertUsage(secondValue.get("delta"), 50L, 30L, 10L, 15L, 80L, 1.2); + assertUsage(secondValue.get("cumulative"), 150L, 50L, 50L, 40L, 200L, 2.0); assertEquals(Map.of("replyId", "reply-2"), secondValue.get("modelCall")); } @@ -1927,6 +1928,7 @@ private static void assertUsage( long inputTokens, long outputTokens, long cachedTokens, + long cacheCreationInputTokens, long totalTokens, double time) { assertInstanceOf(Map.class, value); @@ -1934,6 +1936,7 @@ private static void assertUsage( assertEquals(inputTokens, usage.get("inputTokens")); assertEquals(outputTokens, usage.get("outputTokens")); assertEquals(cachedTokens, usage.get("cachedTokens")); + assertEquals(cacheCreationInputTokens, usage.get("cacheCreationInputTokens")); assertEquals(totalTokens, usage.get("totalTokens")); assertEquals(time, (Double) usage.get("time"), 0.000001); } diff --git a/docs/v2/en/blogs/usecases/aidc-logistics.md b/docs/v2/en/blogs/usecases/aidc-logistics.md index a1ffb14f75..ec31137b70 100644 --- a/docs/v2/en/blogs/usecases/aidc-logistics.md +++ b/docs/v2/en/blogs/usecases/aidc-logistics.md @@ -291,7 +291,7 @@ As the bottom-most layer of the architecture, its core is interacting with the L **5. Zero-intrusion integration of observability and advanced reasoning capabilities** - Rule: Trace instrumentation, Prompt caching, and tool invocation enhancements are automatically completed at the model layer; business code needs no manual handling. -- Source analysis: `ChatModelBase.stream()` automatically wraps calls via `TracerRegistry.get().callModel()`; when cacheControl=true, `OpenAIBaseFormatter.applyCacheControl()` automatically adds cache markers; toolChoice and parallelToolCalls parameters directly control tool behavior. +- Source analysis: `ChatModelBase.stream()` automatically wraps calls via `TracerRegistry.get().callModel()`; official OpenAI endpoints use server-managed automatic prompt caching, while manually marked messages are translated to provider-native explicit breakpoints when supported; toolChoice and parallelToolCalls parameters directly control tool behavior. ![Model layer architecture I](https://mmbiz.qpic.cn/sz_mmbiz_png/bvDbzNRia8j2vyuibOsbQibMibMjVQOymQcVxoTOX2VY8z2jHJ6XdAN5A5FCfD8zWgxt5Abdt2sGI95MLD7eJFMF6pKYduAc8jvaMYS0VfMWw8c/640?wx_fmt=png&from=appmsg) diff --git a/docs/v2/en/blogs/usecases/logistics.md b/docs/v2/en/blogs/usecases/logistics.md index a1ffb14f75..ec31137b70 100644 --- a/docs/v2/en/blogs/usecases/logistics.md +++ b/docs/v2/en/blogs/usecases/logistics.md @@ -291,7 +291,7 @@ As the bottom-most layer of the architecture, its core is interacting with the L **5. Zero-intrusion integration of observability and advanced reasoning capabilities** - Rule: Trace instrumentation, Prompt caching, and tool invocation enhancements are automatically completed at the model layer; business code needs no manual handling. -- Source analysis: `ChatModelBase.stream()` automatically wraps calls via `TracerRegistry.get().callModel()`; when cacheControl=true, `OpenAIBaseFormatter.applyCacheControl()` automatically adds cache markers; toolChoice and parallelToolCalls parameters directly control tool behavior. +- Source analysis: `ChatModelBase.stream()` automatically wraps calls via `TracerRegistry.get().callModel()`; official OpenAI endpoints use server-managed automatic prompt caching, while manually marked messages are translated to provider-native explicit breakpoints when supported; toolChoice and parallelToolCalls parameters directly control tool behavior. ![Model layer architecture I](https://mmbiz.qpic.cn/sz_mmbiz_png/bvDbzNRia8j2vyuibOsbQibMibMjVQOymQcVxoTOX2VY8z2jHJ6XdAN5A5FCfD8zWgxt5Abdt2sGI95MLD7eJFMF6pKYduAc8jvaMYS0VfMWw8c/640?wx_fmt=png&from=appmsg) diff --git a/docs/v2/en/integration/model/gemini.md b/docs/v2/en/integration/model/gemini.md index 68015f3cd3..1a5c9d9373 100644 --- a/docs/v2/en/integration/model/gemini.md +++ b/docs/v2/en/integration/model/gemini.md @@ -37,6 +37,40 @@ GeminiChatModel model = GeminiChatModel.builder() .build(); ``` +## Prompt caching + +### Implicit caching + +Gemini 2.5 and newer models enable implicit caching on the provider side. No request field is required. Setting `GenerateOptions.cacheControl(true)` does not create a Gemini cache resource, and setting it to `false` does not disable provider-managed implicit caching. + +Likewise, adding `MessageMetadataKeys.CACHE_CONTROL` to a `Msg` does not create or manage a Gemini `cachedContents` resource. Gemini explicit caching has a separate, provider-managed resource lifecycle. + +### Reference explicit cached content + +Create and manage the cache resource with the Google Gen AI SDK, then pass its resource name through `GenerateOptions`: + +```java +GenerateOptions options = GenerateOptions.builder() + .additionalBodyParam("cachedContent", cache.name().orElseThrow()) + .build(); + +model.stream(dynamicMessages, tools, options); +``` + +The resource name is passed to `GenerateContentConfig.cachedContent(...)` unchanged. Use a cache created for the same model and, for Vertex AI, the same project and location. The resource must also be unexpired. + +Cached content is a prefix to the request. Pass only the dynamic suffix in `dynamicMessages`; do not resend messages already stored in the cache. If the request uses a system instruction, tool declarations, or tool configuration, include them when creating the cache resource. Gemini does not allow these fields to be sent again in a `generateContent` request that references cached content. Tool schemas may still be supplied to AgentScope for local tool execution, but they must match the declarations stored in the cache. + +The Google SDK owns cache creation, TTL updates, lookup, and deletion. AgentScope only references the supplied resource name during generation. + +### Cache usage + +Gemini cache usage is normalized into `ChatUsage` as follows: + +- `inputTokens` is the total effective prompt size and already includes cached tokens. +- `cachedTokens` is the cached subset reported by Gemini's `cachedContentTokenCount`. +- `cacheCreationInputTokens` is `0` for `generateContent`; cache creation is a separate resource operation. + ## Spring Boot Spring Boot applications can use the Gemini starter: diff --git a/docs/v2/zh/blogs/usecases/aidc-logistics.md b/docs/v2/zh/blogs/usecases/aidc-logistics.md index dccb1f79e3..5b1c3e397c 100644 --- a/docs/v2/zh/blogs/usecases/aidc-logistics.md +++ b/docs/v2/zh/blogs/usecases/aidc-logistics.md @@ -291,7 +291,7 @@ return "NEED_CONFIRM: 金额过大，请确认"; **5. 可观测性与高级推理能力零侵入集成** - 规则：Trace 埋点、Prompt 缓存、工具调用增强等能力在模型层自动完成，业务代码无需手动处理。 -- 源码分析：`ChatModelBase.stream()` 通过 `TracerRegistry.get().callModel()` 自动包裹调用；cacheControl=true 时 `OpenAIBaseFormatter.applyCacheControl()` 自动添加缓存标记；toolChoice 与 parallelToolCalls 参数直接控制工具行为。 +- 源码分析：`ChatModelBase.stream()` 通过 `TracerRegistry.get().callModel()` 自动包裹调用；OpenAI 官方端点使用服务端管理的自动 Prompt 缓存，手动标记的消息则会在支持的端点上转换为提供商原生的显式断点；toolChoice 与 parallelToolCalls 参数直接控制工具行为。 ![模型层架构一](https://mmbiz.qpic.cn/sz_mmbiz_png/bvDbzNRia8j2vyuibOsbQibMibMjVQOymQcVxoTOX2VY8z2jHJ6XdAN5A5FCfD8zWgxt5Abdt2sGI95MLD7eJFMF6pKYduAc8jvaMYS0VfMWw8c/640?wx_fmt=png&from=appmsg) diff --git a/docs/v2/zh/blogs/usecases/logistics.md b/docs/v2/zh/blogs/usecases/logistics.md index 4bb3ef373a..9617bd8758 100644 --- a/docs/v2/zh/blogs/usecases/logistics.md +++ b/docs/v2/zh/blogs/usecases/logistics.md @@ -291,7 +291,7 @@ Thread.sleep(1000); // 占用宝贵线程资源 **5. 可观测性与高级推理能力零侵入集成** - 规则：Trace 埋点、Prompt 缓存、工具调用增强等能力在模型层自动完成，业务代码无需手动处理。 -- 源码分析：`ChatModelBase.stream()` 通过 `TracerRegistry.get().callModel()` 自动包裹调用；cacheControl=true 时 `OpenAIBaseFormatter.applyCacheControl()` 自动添加缓存标记；toolChoice 与 parallelToolCalls 参数直接控制工具行为。 +- 源码分析：`ChatModelBase.stream()` 通过 `TracerRegistry.get().callModel()` 自动包裹调用；OpenAI 官方端点使用服务端管理的自动 Prompt 缓存，手动标记的消息则会在支持的端点上转换为提供商原生的显式断点；toolChoice 与 parallelToolCalls 参数直接控制工具行为。 ![模型层架构一](https://mmbiz.qpic.cn/sz_mmbiz_png/bvDbzNRia8j2vyuibOsbQibMibMjVQOymQcVxoTOX2VY8z2jHJ6XdAN5A5FCfD8zWgxt5Abdt2sGI95MLD7eJFMF6pKYduAc8jvaMYS0VfMWw8c/640?wx_fmt=png&from=appmsg) diff --git a/docs/v2/zh/integration/model/gemini.md b/docs/v2/zh/integration/model/gemini.md index 20e2da57c1..1fbd676d7c 100644 --- a/docs/v2/zh/integration/model/gemini.md +++ b/docs/v2/zh/integration/model/gemini.md @@ -37,6 +37,40 @@ GeminiChatModel model = GeminiChatModel.builder() .build(); ``` +## Prompt 缓存 + +### 隐式缓存 + +Gemini 2.5 及更新模型由服务端默认启用隐式缓存，请求无需增加额外字段。设置 `GenerateOptions.cacheControl(true)` 不会创建 Gemini 缓存资源，设为 `false` 也不会关闭服务端管理的隐式缓存。 + +同样，为 `Msg` 添加 `MessageMetadataKeys.CACHE_CONTROL` 不会创建或管理 Gemini `cachedContents` 资源。Gemini 显式缓存拥有独立的服务端资源生命周期。 + +### 引用显式缓存资源 + +先使用 Google Gen AI SDK 创建和管理缓存资源，再通过 `GenerateOptions` 传入它的资源名称： + +```java +GenerateOptions options = GenerateOptions.builder() + .additionalBodyParam("cachedContent", cache.name().orElseThrow()) + .build(); + +model.stream(dynamicMessages, tools, options); +``` + +资源名称会原样传给 `GenerateContentConfig.cachedContent(...)`。必须使用由相同模型创建且未过期的缓存；对于 Vertex AI，project 和 location 也必须一致。 + +缓存内容是当前请求的前缀。`dynamicMessages` 中只传递动态后缀，不要重复传入已写入缓存的消息。如果请求需要 system instruction、工具声明或 tool config，应在创建缓存资源时写入。Gemini 不允许在引用缓存的 `generateContent` 请求中重复发送这些字段。AgentScope 仍可以接收工具 schema 用于本地工具执行，但它必须与缓存资源中的声明一致。 + +缓存的创建、TTL 更新、查询和删除由 Google SDK 管理。AgentScope 在生成请求中只引用用户传入的资源名称。 + +### 缓存用量 + +Gemini 的缓存用量会按以下规则归一化到 `ChatUsage`： + +- `inputTokens` 是完整的有效输入 token 数，已包含缓存 token。 +- `cachedTokens` 是 Gemini `cachedContentTokenCount` 报告的缓存子集。 +- `cacheCreationInputTokens` 在 `generateContent` 中为 `0`；缓存创建是独立的资源操作。 + ## Spring Boot Spring Boot 应用可以使用 Gemini starter： From 3bd7619f705a7eb2f97ca836519a162c4e2d7c19 Mon Sep 17 00:00:00 2001 From: guslegend <1670547022@qq.com> Date: Tue, 25 Aug 2026 14:39:22 +0800 Subject: [PATCH 05/10] fix(model): harden prompt cache integration --- .../agent/accumulator/ReasoningContext.java | 6 +- .../accumulator/ReasoningContextTest.java | 15 ++++ .../dashscope/dto/DashScopeTokenDetails.java | 49 ++++++++++++- .../dto/DashScopeDtoSerializationTest.java | 28 ++++++++ .../DashScopeResponseParserTest.java | 25 +++++++ .../gemini/formatter/GeminiChatFormatter.java | 6 +- .../formatter/GeminiChatFormatterTest.java | 18 +++++ .../model/openai/OpenAIChatModel.java | 7 +- .../model/openai/dto/OpenAIUsage.java | 66 +++++++++++++++++ .../openai/formatter/OpenAIBaseFormatter.java | 12 ++-- .../formatter/OpenAIResponseParser.java | 11 +-- .../model/openai/OpenAIChatModelTest.java | 8 +++ .../formatter/OpenAICacheControlTest.java | 4 +- .../formatter/OpenAIResponseParserTest.java | 71 +++++++++++++++++++ .../StreamChatResponseAggregator.java | 7 ++ .../StreamChatResponseAggregatorTest.java | 14 +++- docs/v2/en/docs/building-blocks/model.md | 28 ++++++++ docs/v2/zh/docs/building-blocks/model.md | 28 ++++++++ 18 files changed, 386 insertions(+), 17 deletions(-) diff --git a/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java b/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java index 6aa0a96536..7c08a437fb 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java +++ b/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java @@ -294,7 +294,11 @@ public List getAllAccumulatedToolCalls() { * @return ChatUsage with accumulated tokens, or null if no usage data */ public ChatUsage getChatUsage() { - if (inputTokens > 0 || outputTokens > 0 || time > 0) { + if (inputTokens > 0 + || outputTokens > 0 + || cachedTokens > 0 + || cacheCreationInputTokens > 0 + || time > 0) { return ChatUsage.builder() .inputTokens(inputTokens) .outputTokens(outputTokens) diff --git a/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java b/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java index 26f010203d..ce4ef35019 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java @@ -107,6 +107,21 @@ void testCachedTokensPropagation() { assertEquals(15, resultUsage.getCacheCreationInputTokens()); } + @Test + @DisplayName("Should retain cache-only usage") + void testCacheOnlyUsage() { + ChatUsage usage = ChatUsage.builder().cachedTokens(80).cacheCreationInputTokens(15).build(); + ChatResponse chunk = + ChatResponse.builder().id("msg-cache-only").content(List.of()).usage(usage).build(); + + context.processChunk(chunk); + + ChatUsage resultUsage = context.getChatUsage(); + assertNotNull(resultUsage); + assertEquals(80, resultUsage.getCachedTokens()); + assertEquals(15, resultUsage.getCacheCreationInputTokens()); + } + @Test @DisplayName("Should accumulate ChatUsage from multiple chunks") void testMultipleChunksUsageAccumulation() { diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java index 90dcac4240..1713d53df1 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java @@ -28,6 +28,9 @@ public class DashScopeTokenDetails { @JsonProperty("cache_creation_input_tokens") private Integer cacheCreationInputTokens; + @JsonProperty("cache_creation") + private CacheCreationDetails cacheCreation; + public Integer getCachedTokens() { return cachedTokens; } @@ -37,10 +40,54 @@ public void setCachedTokens(Integer cachedTokens) { } public Integer getCacheCreationInputTokens() { - return cacheCreationInputTokens; + if (cacheCreationInputTokens != null) { + return cacheCreationInputTokens; + } + return cacheCreation != null ? cacheCreation.getInputTokens() : null; } public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { this.cacheCreationInputTokens = cacheCreationInputTokens; } + + public CacheCreationDetails getCacheCreation() { + return cacheCreation; + } + + public void setCacheCreation(CacheCreationDetails cacheCreation) { + this.cacheCreation = cacheCreation; + } + + /** Current nested DashScope explicit-cache creation details. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CacheCreationDetails { + + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + @JsonProperty("ephemeral_5m_input_tokens") + private Integer ephemeral5mInputTokens; + + public Integer getCacheCreationInputTokens() { + return cacheCreationInputTokens; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } + + public Integer getEphemeral5mInputTokens() { + return ephemeral5mInputTokens; + } + + public void setEphemeral5mInputTokens(Integer ephemeral5mInputTokens) { + this.ephemeral5mInputTokens = ephemeral5mInputTokens; + } + + Integer getInputTokens() { + return cacheCreationInputTokens != null + ? cacheCreationInputTokens + : ephemeral5mInputTokens; + } + } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java index f133d3a73f..f10ba785aa 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java @@ -262,6 +262,34 @@ void testDashScopeCacheUsageDeserialization() { assertEquals(256, response.getUsage().getCacheCreationInputTokens()); } + @Test + void testDashScopeNestedCacheCreationUsageDeserialization() { + String json = + """ + { + "request_id": "req-cache-nested", + "usage": { + "input_tokens": 2048, + "output_tokens": 128, + "prompt_tokens_details": { + "cached_tokens": 1536, + "cache_creation": { + "cache_creation_input_tokens": 384, + "ephemeral_5m_input_tokens": 999, + "cache_type": "ephemeral" + } + } + } + } + """; + + DashScopeResponse response = jsonCodec.fromJson(json, DashScopeResponse.class); + + assertNotNull(response.getUsage()); + assertEquals(1536, response.getUsage().getCachedTokens()); + assertEquals(384, response.getUsage().getCacheCreationInputTokens()); + } + @Test void testDashScopeResponseWithToolCalls() throws Exception { String json = diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java index 593ea55230..f731e3b0dc 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java @@ -216,6 +216,31 @@ void testParsePromptCacheUsageDetails() { assertEquals(30, chatResponse.getUsage().getCacheCreationInputTokens()); } + @Test + void testParseNestedPromptCacheCreationUsageDetails() { + DashScopeTokenDetails.CacheCreationDetails cacheCreation = + new DashScopeTokenDetails.CacheCreationDetails(); + cacheCreation.setEphemeral5mInputTokens(35); + DashScopeTokenDetails details = new DashScopeTokenDetails(); + details.setCachedTokens(55); + details.setCacheCreation(cacheCreation); + + DashScopeUsage usage = new DashScopeUsage(); + usage.setInputTokens(100); + usage.setOutputTokens(20); + usage.setPromptTokensDetails(details); + + DashScopeResponse response = new DashScopeResponse(); + response.setRequestId("req-nested-cache-usage"); + response.setUsage(usage); + + ChatResponse chatResponse = parser.parseResponse(response, startTime); + + assertNotNull(chatResponse.getUsage()); + assertEquals(55, chatResponse.getUsage().getCachedTokens()); + assertEquals(35, chatResponse.getUsage().getCacheCreationInputTokens()); + } + @Test void testParseLegacyTopLevelCacheUsage() { DashScopeUsage usage = new DashScopeUsage(); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java index cbf84ffabe..f4b0bb634f 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java @@ -144,8 +144,10 @@ public void applyOptions( private String resolveCachedContent(GenerateOptions options, GenerateOptions defaultOptions) { CacheReference requestReference = cacheReference(options); - CacheReference defaultReference = cacheReference(defaultOptions); - Object value = requestReference.present ? requestReference.value : defaultReference.value; + Object value = + requestReference.present + ? requestReference.value + : cacheReference(defaultOptions).value; if (value == null) { return null; diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java index 4caae34784..f00a0f8ebd 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java @@ -140,6 +140,24 @@ void testRequestCachedContentOverridesDefaultAlias() { assertEquals("cachedContents/request", configBuilder.build().cachedContent().orElseThrow()); } + @Test + void testRequestCachedContentShortCircuitsInvalidDefaults() { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + GenerateOptions defaults = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/default-one") + .additionalBodyParam("cached_content", "cachedContents/default-two") + .build(); + GenerateOptions request = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/request") + .build(); + + formatter.applyOptions(configBuilder, request, defaults); + + assertEquals("cachedContents/request", configBuilder.build().cachedContent().orElseThrow()); + } + @Test void testRejectConflictingCachedContentAliases() { GenerateOptions options = diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java index 7f9deef654..ab7e5e55ac 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java @@ -236,7 +236,12 @@ static boolean isOfficialOpenAIBaseUrl(String baseUrl) { } try { String host = URI.create(baseUrl).getHost(); - return host != null && host.equalsIgnoreCase("api.openai.com"); + if (host == null) { + return false; + } + String normalizedHost = host.toLowerCase(Locale.ROOT); + return normalizedHost.equals("api.openai.com") + || normalizedHost.endsWith(".api.openai.com"); } catch (IllegalArgumentException e) { return false; } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java index 13535480d1..2d82d1bbb1 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java @@ -111,6 +111,12 @@ public static class PromptTokensDetails { @JsonProperty("cache_write_tokens") private Integer cacheWriteTokens; + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + @JsonProperty("cache_creation") + private CacheCreationDetails cacheCreation; + public Integer getCachedTokens() { return cachedTokens; } @@ -134,6 +140,66 @@ public Integer getCacheWriteTokens() { public void setCacheWriteTokens(Integer cacheWriteTokens) { this.cacheWriteTokens = cacheWriteTokens; } + + public Integer getCacheCreationInputTokens() { + return cacheCreationInputTokens; + } + + /** Returns prompt-cache creation tokens across OpenAI and compatible response shapes. */ + public Integer resolveCacheCreationInputTokens() { + if (cacheWriteTokens != null) { + return cacheWriteTokens; + } + if (cacheCreationInputTokens != null) { + return cacheCreationInputTokens; + } + return cacheCreation != null ? cacheCreation.getInputTokens() : null; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } + + public CacheCreationDetails getCacheCreation() { + return cacheCreation; + } + + public void setCacheCreation(CacheCreationDetails cacheCreation) { + this.cacheCreation = cacheCreation; + } + + /** Nested explicit-cache creation details returned by compatible providers. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CacheCreationDetails { + + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + @JsonProperty("ephemeral_5m_input_tokens") + private Integer ephemeral5mInputTokens; + + public Integer getCacheCreationInputTokens() { + return cacheCreationInputTokens; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } + + public Integer getEphemeral5mInputTokens() { + return ephemeral5mInputTokens; + } + + public void setEphemeral5mInputTokens(Integer ephemeral5mInputTokens) { + this.ephemeral5mInputTokens = ephemeral5mInputTokens; + } + + Integer getInputTokens() { + return cacheCreationInputTokens != null + ? cacheCreationInputTokens + : ephemeral5mInputTokens; + } + } } /** diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java index 1ead4b43dd..b338570173 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java @@ -174,14 +174,18 @@ public OpenAIRequest buildRequest( } /** - * Apply cache control to OpenAI messages. + * Apply legacy message-level cache control. * - *

    Adds cache_control: {"type": "ephemeral"} to all system messages and the last - * message in the list. Messages that already have cache_control set (e.g., via manual metadata - * marking) will not be overwritten. + *

    This method emits the provider-specific {@code cache_control} field used by older + * OpenAI-compatible integrations. The official OpenAI request path does not call it: OpenAI + * automatic caching is provider-managed, while explicit AgentScope markers are normalized to + * {@code prompt_cache_breakpoint} by {@link #applyOpenAIPromptCache(OpenAIRequest)}. * * @param messages the list of formatted OpenAI messages + * @deprecated use the model request path so the selected endpoint can apply its native cache + * protocol */ + @Deprecated(since = "2.0.3", forRemoval = false) public void applyCacheControl(List messages) { if (messages == null || messages.isEmpty()) { return; diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java index ea2dceeba6..94836c30ad 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java @@ -90,12 +90,13 @@ private long getSafeCachedTokens(OpenAIUsage usage) { return cachedTokens != null ? cachedTokens : 0; } - private long getSafeCacheWriteTokens(OpenAIUsage usage) { + private long getSafeCacheCreationInputTokens(OpenAIUsage usage) { if (usage == null || usage.getPromptTokensDetails() == null) { return 0; } - Integer cacheWriteTokens = usage.getPromptTokensDetails().getCacheWriteTokens(); - return cacheWriteTokens != null ? cacheWriteTokens : 0; + Integer cacheCreationInputTokens = + usage.getPromptTokensDetails().resolveCacheCreationInputTokens(); + return cacheCreationInputTokens != null ? cacheCreationInputTokens : 0; } public OpenAIResponseParser() {} @@ -137,7 +138,7 @@ protected ChatResponse parseCompletionResponse(OpenAIResponse response, Instant .outputTokens((int) getSafeCompletionTokens(openAIUsage)) .cachedTokens((int) getSafeCachedTokens(openAIUsage)) .cacheCreationInputTokens( - (int) getSafeCacheWriteTokens(openAIUsage)) + (int) getSafeCacheCreationInputTokens(openAIUsage)) .time( Duration.between(startTime, Instant.now()).toMillis() / 1000.0) @@ -383,7 +384,7 @@ protected ChatResponse parseChunkResponse(OpenAIResponse response, Instant start : 0) .cachedTokens((int) getSafeCachedTokens(openAIUsage)) .cacheCreationInputTokens( - (int) getSafeCacheWriteTokens(openAIUsage)) + (int) getSafeCacheCreationInputTokens(openAIUsage)) .time( Duration.between(startTime, Instant.now()).toMillis() / 1000.0) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java index 2677fb699a..3b50962b11 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java @@ -135,6 +135,14 @@ void testDashScopeCompatibleBaseUrlDetection() { assertTrue(OpenAIChatModel.isOfficialOpenAIBaseUrl(null)); assertTrue(OpenAIChatModel.isOfficialOpenAIBaseUrl("")); assertTrue(OpenAIChatModel.isOfficialOpenAIBaseUrl("https://api.openai.com/v1")); + for (String region : List.of("us", "eu", "au", "ca", "jp", "in", "sg", "kr", "gb", "ae")) { + assertTrue( + OpenAIChatModel.isOfficialOpenAIBaseUrl( + "https://" + region + ".api.openai.com/v1"), + region); + } + assertFalse( + OpenAIChatModel.isOfficialOpenAIBaseUrl("https://api.openai.com.evil.example/v1")); assertFalse( OpenAIChatModel.isOfficialOpenAIBaseUrl("https://gateway.example.com/openai/v1")); } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java index 0a0f58fc1b..f6ff12ad72 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java @@ -323,14 +323,14 @@ void mergesConfiguredPromptCacheOptions() { .messages(List.of(message)) .extraParam( "prompt_cache_options", - Map.of("mode", "automatic", "ttl", "24h")) + Map.of("mode", "implicit", "ttl", "30m")) .build(); formatter.applyOpenAIPromptCache(request); JsonNode json = toJsonTree(request); assertEquals("explicit", json.path("prompt_cache_options").path("mode").asText()); - assertEquals("24h", json.path("prompt_cache_options").path("ttl").asText()); + assertEquals("30m", json.path("prompt_cache_options").path("ttl").asText()); } @Test diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java index 659662e2b0..3855f0e15a 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java @@ -26,6 +26,8 @@ import io.agentscope.core.message.ThinkingBlock; import io.agentscope.core.message.ToolUseBlock; import io.agentscope.core.model.ChatResponse; +import io.agentscope.core.util.JsonCodec; +import io.agentscope.core.util.JsonUtils; import io.agentscope.extensions.model.openai.dto.OpenAIChoice; import io.agentscope.extensions.model.openai.dto.OpenAIError; import io.agentscope.extensions.model.openai.dto.OpenAIFunction; @@ -55,11 +57,13 @@ class OpenAIResponseParserTest { private OpenAIResponseParser parser; private Instant startTime; + private JsonCodec jsonCodec; @BeforeEach void setUp() { parser = new OpenAIResponseParser(); startTime = Instant.now(); + jsonCodec = JsonUtils.getJsonCodec(); } @Test @@ -1059,6 +1063,73 @@ void testParseChunkResponseWithCachedTokens() { assertEquals(7, result.getUsage().getCacheCreationInputTokens()); } + @Test + @DisplayName("Should parse nested compatible-provider cache creation tokens") + void testParseResponseWithNestedCacheCreationTokens() { + String json = + """ + { + "id": "chatcmpl-cache-nested", + "object": "chat.completion", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "Answer"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 20, + "prompt_tokens_details": { + "cached_tokens": 60, + "cache_creation": { + "ephemeral_5m_input_tokens": 30, + "cache_type": "ephemeral" + } + } + } + } + """; + OpenAIResponse response = jsonCodec.fromJson(json, OpenAIResponse.class); + + ChatResponse result = parser.parseResponse(response, startTime); + + assertNotNull(result.getUsage()); + assertEquals(60, result.getUsage().getCachedTokens()); + assertEquals(30, result.getUsage().getCacheCreationInputTokens()); + } + + @Test + @DisplayName("Should parse direct compatible-provider cache creation tokens from a chunk") + void testParseChunkResponseWithDirectCacheCreationTokens() { + String json = + """ + { + "id": "chatcmpl-cache-direct", + "object": "chat.completion.chunk", + "choices": [{ + "index": 0, + "delta": {"role": "assistant", "content": ""}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 20, + "prompt_tokens_details": { + "cached_tokens": 50, + "cache_creation_input_tokens": 25 + } + } + } + """; + OpenAIResponse response = jsonCodec.fromJson(json, OpenAIResponse.class); + + ChatResponse result = parser.parseResponse(response, startTime); + + assertNotNull(result.getUsage()); + assertEquals(50, result.getUsage().getCachedTokens()); + assertEquals(25, result.getUsage().getCacheCreationInputTokens()); + } + @Test @DisplayName("Should default token counts to 0 when chunk usage fields are null") void testParseChunkResponseWithNullTokenCounts() { diff --git a/agentscope-extensions/agentscope-extensions-studio/src/main/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregator.java b/agentscope-extensions/agentscope-extensions-studio/src/main/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregator.java index a6b7441384..4688064053 100644 --- a/agentscope-extensions/agentscope-extensions-studio/src/main/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregator.java +++ b/agentscope-extensions/agentscope-extensions-studio/src/main/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregator.java @@ -43,6 +43,8 @@ final class StreamChatResponseAggregator { // Usage: take the max value from all chunks, since providers report cumulative totals private int inputTokens; private int outputTokens; + private int cachedTokens; + private int cacheCreationInputTokens; private double time; private String finishReason; @@ -74,6 +76,9 @@ public void append(ChatResponse chunk) { if (usage != null) { inputTokens = Math.max(inputTokens, usage.getInputTokens()); outputTokens = Math.max(outputTokens, usage.getOutputTokens()); + cachedTokens = Math.max(cachedTokens, usage.getCachedTokens()); + cacheCreationInputTokens = + Math.max(cacheCreationInputTokens, usage.getCacheCreationInputTokens()); time = usage.getTime(); } @@ -96,6 +101,8 @@ public ChatResponse getResponse() { ChatUsage.builder() .inputTokens(inputTokens) .outputTokens(outputTokens) + .cachedTokens(cachedTokens) + .cacheCreationInputTokens(cacheCreationInputTokens) .time(time) .build()) .finishReason(finishReason) diff --git a/agentscope-extensions/agentscope-extensions-studio/src/test/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregatorTest.java b/agentscope-extensions/agentscope-extensions-studio/src/test/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregatorTest.java index ad42246bd4..96ffe34be9 100644 --- a/agentscope-extensions/agentscope-extensions-studio/src/test/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregatorTest.java +++ b/agentscope-extensions/agentscope-extensions-studio/src/test/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregatorTest.java @@ -41,6 +41,8 @@ void testCumulativeUsageTakesMax() { ChatUsage.builder() .inputTokens(100) .outputTokens(i * 20) + .cachedTokens(i * 10) + .cacheCreationInputTokens(i * 5) .time(i * 0.5) .build()) .finishReason(i == 5 ? "stop" : null) @@ -51,6 +53,8 @@ void testCumulativeUsageTakesMax() { assertEquals("test-id", response.getId()); assertEquals(100, response.getUsage().getInputTokens()); assertEquals(100, response.getUsage().getOutputTokens()); + assertEquals(50, response.getUsage().getCachedTokens()); + assertEquals(25, response.getUsage().getCacheCreationInputTokens()); assertEquals("stop", response.getFinishReason()); } @@ -70,12 +74,20 @@ void testOnlyLastChunkHasUsage() { agg.append( ChatResponse.builder() .id("openai-id") - .usage(ChatUsage.builder().inputTokens(200).outputTokens(150).build()) + .usage( + ChatUsage.builder() + .inputTokens(200) + .outputTokens(150) + .cachedTokens(120) + .cacheCreationInputTokens(30) + .build()) .finishReason("stop") .build()); ChatResponse response = agg.getResponse(); assertEquals(200, response.getUsage().getInputTokens()); assertEquals(150, response.getUsage().getOutputTokens()); + assertEquals(120, response.getUsage().getCachedTokens()); + assertEquals(30, response.getUsage().getCacheCreationInputTokens()); } } diff --git a/docs/v2/en/docs/building-blocks/model.md b/docs/v2/en/docs/building-blocks/model.md index c34fb54882..c98f718156 100644 --- a/docs/v2/en/docs/building-blocks/model.md +++ b/docs/v2/en/docs/building-blocks/model.md @@ -59,6 +59,34 @@ Other provider artifacts follow the same pattern: `agentscope-extensions-model-o ``` +## Prompt caching + +Prompt caching is provider-specific. `GenerateOptions.cacheControl(true)` asks a supporting adapter to use its automatic strategy, while `MessageMetadataKeys.CACHE_CONTROL` marks a manual cache boundary. A manual boundary remains effective when the automatic strategy is disabled or unset. + +```java +GenerateOptions automaticCache = + GenerateOptions.builder().cacheControl(true).build(); + +Msg stableContext = + Msg.builder() + .role(MsgRole.USER) + .textContent("Stable context to cache") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); +``` + +The same option does not have identical wire semantics for every provider: + +| Provider | Automatic/provider-managed behavior | Explicit behavior | +|----------|-------------------------------------|-------------------| +| OpenAI | Supported models cache eligible prefixes automatically. `cacheControl(true)` does not add an artificial breakpoint. | A marked `Msg` is converted to OpenAI's native explicit breakpoint protocol on official OpenAI endpoints. Internal markers are removed for unknown OpenAI-compatible endpoints. | +| DashScope | `cacheControl(true)` marks the first system message and the last cacheable conversation message. | A marked `Msg` becomes a native content-block cache marker on DashScope native and recognized DashScope-compatible endpoints. | +| Anthropic | `cacheControl(true)` enables Anthropic automatic prompt caching. | A marked `Msg` becomes an Anthropic content-block breakpoint. Automatic caching consumes one of the provider's four available breakpoints. | +| Gemini | Gemini 2.5+ manages implicit caching. The boolean option does not create a cache resource. | Reference an existing explicit resource with `additionalBodyParam("cachedContent", resourceName)`; resource CRUD and TTL remain in the Google SDK. Message markers do not create resources. | +| Ollama | No prompt-cache request or usage protocol is exposed by this adapter. | Not supported. | + +Setting `cacheControl(false)` only disables an AgentScope automatic strategy; it cannot disable provider-managed implicit caching. `ChatUsage.inputTokens` is the total input count. `cachedTokens` and `cacheCreationInputTokens` are subsets of that input count and must not be added again when computing total tokens. + ## Choose a creation path ### String model id diff --git a/docs/v2/zh/docs/building-blocks/model.md b/docs/v2/zh/docs/building-blocks/model.md index af207671d8..499ff9f6b7 100644 --- a/docs/v2/zh/docs/building-blocks/model.md +++ b/docs/v2/zh/docs/building-blocks/model.md @@ -59,6 +59,34 @@ CredentialBase/ ``` +## Prompt 缓存 + +Prompt 缓存是模型提供商专属能力。`GenerateOptions.cacheControl(true)` 会请求支持该能力的适配器使用自动策略；`MessageMetadataKeys.CACHE_CONTROL` 用于手动标记缓存边界。即使自动策略未设置或已关闭，手动边界仍然生效。 + +```java +GenerateOptions automaticCache = + GenerateOptions.builder().cacheControl(true).build(); + +Msg stableContext = + Msg.builder() + .role(MsgRole.USER) + .textContent("需要缓存的稳定上下文") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); +``` + +同一选项在不同提供商的线上协议含义不完全相同： + +| 提供商 | 自动/服务端管理行为 | 显式行为 | +|----------|----------------------|----------| +| OpenAI | 受支持的模型会自动缓存符合条件的前缀；`cacheControl(true)` 不会人为新增断点。 | 官方 OpenAI 端点会把已标记的 `Msg` 转成 OpenAI 原生显式断点协议；未知 OpenAI-compatible 端点会移除内部标记。 | +| DashScope | `cacheControl(true)` 会标记第一条 system 消息和最后一条可缓存对话消息。 | 在 DashScope 原生端点和已识别的 DashScope-compatible 端点上，已标记 `Msg` 会转成内容块缓存标记。 | +| Anthropic | `cacheControl(true)` 启用 Anthropic 自动 prompt 缓存。 | 已标记 `Msg` 会转成 Anthropic 内容块断点；自动缓存会占用提供商最多四个断点中的一个。 | +| Gemini | Gemini 2.5+ 由服务端管理隐式缓存，布尔选项不会创建缓存资源。 | 通过 `additionalBodyParam("cachedContent", resourceName)` 引用已存在的显式资源；资源 CRUD 和 TTL 由 Google SDK 管理，消息标记不会创建资源。 | +| Ollama | 当前适配器未暴露 prompt 缓存请求或 usage 协议。 | 不支持。 | + +设置 `cacheControl(false)` 只会关闭 AgentScope 的自动策略，无法关闭提供商管理的隐式缓存。`ChatUsage.inputTokens` 是输入 token 总数，`cachedTokens` 和 `cacheCreationInputTokens` 是其子集，计算总 token 时不能重复相加。 + ## 选择模型创建方式 ### 字符串 model id From 154d4eb624ac680cffd67678aaa7780623500d1a Mon Sep 17 00:00:00 2001 From: guslegend <1670547022@qq.com> Date: Tue, 25 Aug 2026 15:09:07 +0800 Subject: [PATCH 06/10] fix(ci): add missing license headers --- .../anthropic/formatter/AnthropicPromptCacheSupport.java | 9 +++++++++ .../anthropic/formatter/AnthropicPromptCacheTest.java | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java index 8900b98c47..8b8ebd1226 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java @@ -3,6 +3,15 @@ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.agentscope.extensions.model.anthropic.formatter; diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java index 042a4d69d1..5f9aaacd41 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java @@ -3,6 +3,15 @@ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.agentscope.extensions.model.anthropic.formatter; From 31da45ceb590bc3a4ef3e59838a5b8080e16aad1 Mon Sep 17 00:00:00 2001 From: guslegend <1670547022@qq.com> Date: Tue, 25 Aug 2026 15:21:19 +0800 Subject: [PATCH 07/10] fix(test): close harness integration agents --- .../HarnessAgentIntegrationExampleTest.java | 118 ++++++++++-------- 1 file changed, 63 insertions(+), 55 deletions(-) diff --git a/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentIntegrationExampleTest.java b/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentIntegrationExampleTest.java index 3731ec3984..e77d8f5662 100644 --- a/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentIntegrationExampleTest.java +++ b/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentIntegrationExampleTest.java @@ -117,56 +117,62 @@ void example_fullWorkspace_singleTurn_seesSessionSubagentsAndWorkspaceContext() """); Model model = stubModel("integration-main-reply"); - HarnessAgent agent = + try (HarnessAgent agent = HarnessAgent.builder() .name("integration-main") .description("integration example main agent") .sysPrompt("You are the main agent in an integration test.") .model(model) .workspace(workspace) - .build(); + .build()) { - Msg reply = - agent.call( - userText("Run the integration scenario."), - RuntimeContext.builder().sessionId("integration-session-1").build()) - .block(); + Msg reply = + agent.call( + userText("Run the integration scenario."), + RuntimeContext.builder() + .sessionId("integration-session-1") + .build()) + .block(); - assertTrue(reply.getTextContent().contains("integration-main-reply")); + assertTrue(reply.getTextContent().contains("integration-main-reply")); - @SuppressWarnings("unchecked") - ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); - verify(model, atLeast(1)).stream(captor.capture(), any(), any()); - String combined = - captor.getAllValues().stream() - .map(HarnessAgentIntegrationExampleTest::joinAllText) - .filter(s -> s.contains("## AgentStateStore Context")) - .findFirst() - .orElse(""); + @SuppressWarnings("unchecked") + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(model, atLeast(1)).stream(captor.capture(), any(), any()); + String combined = + captor.getAllValues().stream() + .map(HarnessAgentIntegrationExampleTest::joinAllText) + .filter(s -> s.contains("## AgentStateStore Context")) + .findFirst() + .orElse(""); - assertTrue( - combined.contains("## AgentStateStore Context"), - "AgentStateStore context should be injected; model saw: " - + captor.getAllValues().stream() - .map(HarnessAgentIntegrationExampleTest::joinAllText) - .toList()); - // Current WorkspaceContextHook uses markdown (##) guidance + XML blocks - assertTrue( - combined.contains("## Domain Knowledge") || combined.contains("## Workspace"), - "expected workspace guidance sections"); - assertTrue(combined.contains("`AGENTS.md`") || combined.contains("agents_context")); - assertTrue( - combined.contains(agentsPersona), "AGENTS.md should appear under workspace hook"); - assertTrue(combined.contains("memory_context") || combined.contains("MEMORY.md")); - assertTrue(combined.contains(memoryNote)); - assertTrue( - combined.contains("domain_knowledge_context") || combined.contains("KNOWLEDGE.md")); - assertTrue(combined.contains(knowledgeLine)); - assertTrue( - combined.contains("## Subagents") || combined.contains("Subagents:"), - "subagent list should be injected into the system prompt"); - assertTrue(combined.contains("`" + helperSubId + "`")); - assertTrue(combined.contains("`" + reviewerSubId + "`")); + assertTrue( + combined.contains("## AgentStateStore Context"), + "AgentStateStore context should be injected; model saw: " + + captor.getAllValues().stream() + .map(HarnessAgentIntegrationExampleTest::joinAllText) + .toList()); + // Current WorkspaceContextHook uses markdown (##) guidance + XML + // blocks + assertTrue( + combined.contains("## Domain Knowledge") || combined.contains("## Workspace"), + "expected workspace guidance sections"); + assertTrue(combined.contains("`AGENTS.md`") || combined.contains("agents_context")); + assertTrue( + combined.contains(agentsPersona), + "AGENTS.md should appear under workspace hook"); + assertTrue(combined.contains("memory_context") || combined.contains("MEMORY.md")); + assertTrue(combined.contains(memoryNote)); + assertTrue( + combined.contains("domain_knowledge_context") + || combined.contains("KNOWLEDGE.md")); + assertTrue(combined.contains(knowledgeLine)); + assertTrue( + combined.contains("## Subagents") || combined.contains("Subagents:"), + "subagent list should be injected into the system prompt"); + assertTrue(combined.contains("`" + helperSubId + "`")); + assertTrue(combined.contains("`" + reviewerSubId + "`")); + } } /** @@ -217,23 +223,25 @@ void example_subagentFactory_markdownSpec_runsChildHarnessAgent() throws Excepti .orElseThrow( () -> new AssertionError("missing subagent entry: " + childId)); - Agent sub = child.factory().create(RuntimeContext.empty()); - assertInstanceOf(HarnessAgent.class, sub); - assertEquals(childId, sub.getName()); + try (HarnessAgent sub = + assertInstanceOf( + HarnessAgent.class, child.factory().create(RuntimeContext.empty()))) { + assertEquals(childId, sub.getName()); - Msg subReply = sub.call(List.of(userText("task for child"))).block(); - assertTrue(subReply.getTextContent().contains("integration-child-reply")); + Msg subReply = sub.call(List.of(userText("task for child"))).block(); + assertTrue(subReply.getTextContent().contains("integration-child-reply")); - @SuppressWarnings("unchecked") - ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); - verify(model, atLeast(1)).stream(captor.capture(), any(), any()); - boolean childSysSeen = - captor.getAllValues().stream() - .map(HarnessAgentIntegrationExampleTest::joinAllText) - .anyMatch(s -> s.contains("INTEGRATION_CHILD_SYS")); - assertTrue( - childSysSeen, - "child HarnessAgent should use spec sysPrompt in its system prompt bundle"); + @SuppressWarnings("unchecked") + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(model, atLeast(1)).stream(captor.capture(), any(), any()); + boolean childSysSeen = + captor.getAllValues().stream() + .map(HarnessAgentIntegrationExampleTest::joinAllText) + .anyMatch(s -> s.contains("INTEGRATION_CHILD_SYS")); + assertTrue( + childSysSeen, + "child HarnessAgent should use spec sysPrompt in its system prompt bundle"); + } } private static Msg userText(String text) { From ee710c0fa268d266fefc3a7c94bb61b4eae96acd Mon Sep 17 00:00:00 2001 From: guslegend <1670547022@qq.com> Date: Tue, 25 Aug 2026 15:39:21 +0800 Subject: [PATCH 08/10] fix(test): await protocol task cleanup --- .../AgentProtocolTaskStoreHitlTest.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agent-protocol/src/test/java/io/agentscope/extensions/agentprotocol/AgentProtocolTaskStoreHitlTest.java b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agent-protocol/src/test/java/io/agentscope/extensions/agentprotocol/AgentProtocolTaskStoreHitlTest.java index 84cc34baeb..cbc239c378 100644 --- a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agent-protocol/src/test/java/io/agentscope/extensions/agentprotocol/AgentProtocolTaskStoreHitlTest.java +++ b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agent-protocol/src/test/java/io/agentscope/extensions/agentprotocol/AgentProtocolTaskStoreHitlTest.java @@ -111,7 +111,11 @@ void submit_pausesOnPermissionAsking_andResumeDenyCompletes() throws Exception { store.resume("hitl-1", List.of(new RemoteConfirmDecision("tc-ask", false))); - awaitCondition(() -> "success".equals(store.snapshot("hitl-1").get("status")), 5_000); + awaitCondition( + () -> + "success".equals(store.snapshot("hitl-1").get("status")) + && !store.hasSubmitContext("hitl-1"), + 5_000); Map done = store.snapshot("hitl-1"); assertEquals("success", done.get("status")); @@ -130,7 +134,11 @@ void submitContextClearedOnDirectSuccess() throws Exception { new AgentEndEvent(null))); store.submit("ok-1", "worker", "hello", Map.of("user_id", "u1", "detail", "status")); - awaitCondition(() -> "success".equals(store.snapshot("ok-1").get("status")), 5_000); + awaitCondition( + () -> + "success".equals(store.snapshot("ok-1").get("status")) + && !store.hasSubmitContext("ok-1"), + 5_000); assertFalse(store.hasSubmitContext("ok-1")); } From 51d4979c99ac024de86650a2d526005a863e4fd4 Mon Sep 17 00:00:00 2001 From: guslegend <1670547022@qq.com> Date: Tue, 25 Aug 2026 15:50:20 +0800 Subject: [PATCH 09/10] fix(test): close dynamic hook test agents --- .../HarnessAgentDynamicHookBuilderTest.java | 201 ++++++++++-------- 1 file changed, 115 insertions(+), 86 deletions(-) diff --git a/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentDynamicHookBuilderTest.java b/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentDynamicHookBuilderTest.java index 12f6183a60..6254219827 100644 --- a/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentDynamicHookBuilderTest.java +++ b/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentDynamicHookBuilderTest.java @@ -44,11 +44,13 @@ import io.agentscope.harness.agent.middleware.SubagentsMiddleware; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; @@ -75,17 +77,24 @@ class HarnessAgentDynamicHookBuilderTest { @TempDir Path workspace; + private final List agents = new ArrayList<>(); + + @AfterEach + void closeAgents() { + agents.forEach(HarnessAgent::close); + } @Test void defaultBuild_registersDynamicSkillAndSubagentMiddlewares() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .build()); List mws = agent.getDelegate().getMiddlewares(); assertTrue( @@ -108,13 +117,14 @@ void customSkillRepository_composesWithDynamicMiddleware() throws Exception { AgentSkillRepository emptyRepo = new EmptySkillRepository(); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(emptyRepo) - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(emptyRepo) + .build()); List mws = agent.getDelegate().getMiddlewares(); assertTrue( @@ -126,13 +136,14 @@ void customSkillRepository_composesWithDynamicMiddleware() throws Exception { void disableDynamicSkills_skipsDynamicSkillMiddleware() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .disableDynamicSkills() + .build()); List mws = agent.getDelegate().getMiddlewares(); assertFalse( @@ -154,14 +165,15 @@ void disableDynamicSkills_freezesRepositoriesIntoStaticMiddleware() throws Excep null))); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(model) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(repository) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(model) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(repository) + .disableDynamicSkills() + .build()); assertEquals( 1, @@ -223,21 +235,27 @@ void disableDynamicSkills_appliesBuilderAndVisibilityFiltersToPromptAndLoader() skill("gamma", "gamma description"))); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(model) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(repository) - .enableSkills("alpha", "beta") - .enableSkillPromotionGate( - null, - (skills, ctx) -> - skills.stream() - .filter(skill -> !"alpha".equals(skill.getName())) - .toList()) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(model) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(repository) + .enableSkills("alpha", "beta") + .enableSkillPromotionGate( + null, + (skills, ctx) -> + skills.stream() + .filter( + skill -> + !"alpha" + .equals( + skill + .getName())) + .toList()) + .disableDynamicSkills() + .build()); RuntimeContext ctx = RuntimeContext.builder().sessionId("filtered-static").build(); agent.call("hello", ctx).block(); @@ -266,15 +284,16 @@ void disableDynamicSkills_skillsEnabledFalseLeavesCatalogEmpty() throws Exceptio new CountingSkillRepository(List.of(skill("disabled", "must stay hidden"))); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(model) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(repository) - .skillsEnabled(false) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(model) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(repository) + .skillsEnabled(false) + .disableDynamicSkills() + .build()); RuntimeContext ctx = RuntimeContext.builder().sessionId("no-static-skills").build(); agent.call("hello", ctx).block(); @@ -294,13 +313,14 @@ void disableDynamicSkills_keepsWorkspaceLazyResourcesLoadable() throws Exception Files.writeString(skillDir.resolve("references/guide.md"), "lazy reference body"); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .disableDynamicSkills() + .build()); RuntimeContext ctx = RuntimeContext.builder().sessionId("lazy-static").build(); agent.call("hello", ctx).block(); @@ -332,13 +352,14 @@ void getSkillRepositories_exposesComposedListInOrder() throws Exception { AgentSkillRepository custom = new EmptySkillRepository(); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(custom) - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(custom) + .build()); List repos = agent.getSkillRepositories(); assertNotNull(repos, "getSkillRepositories() must never return null"); @@ -357,13 +378,14 @@ void getSkillRepositories_exposesComposedListInOrder() throws Exception { void getSkillRepositories_isEmptyWhenNothingComposed() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .disableDynamicSkills() + .build()); assertNotNull(agent.getSkillRepositories()); } @@ -372,12 +394,13 @@ void getSkillRepositories_isEmptyWhenNothingComposed() throws Exception { void getSkillRepositories_returnsImmutableList() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .build()); List first = agent.getSkillRepositories(); List second = agent.getSkillRepositories(); @@ -395,13 +418,14 @@ void getSkillRepositories_returnsImmutableList() throws Exception { void disableDynamicSubagents_fallsBackToStaticSubagentsMiddleware() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .disableDynamicSubagents() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .disableDynamicSubagents() + .build()); List mws = agent.getDelegate().getMiddlewares(); assertFalse( @@ -424,6 +448,11 @@ void disableDynamicSubagents_fallsBackToStaticSubagentsMiddleware() throws Excep .orElseThrow(); } + private HarnessAgent track(HarnessAgent agent) { + agents.add(agent); + return agent; + } + private static AgentSkill skill(String name, String description) { return new AgentSkill(name, description, "# " + name, null); } From 2764954f2e052e473d863cc7c7cbdb8446c436d7 Mon Sep 17 00:00:00 2001 From: guslegend0510 Date: Tue, 25 Aug 2026 20:28:23 +0800 Subject: [PATCH 10/10] style(anthropic): remove trailing blank lines --- .../model/anthropic/formatter/AnthropicBaseFormatter.java | 1 - .../extensions/model/anthropic/AnthropicChatModelTest.java | 1 - .../model/anthropic/formatter/AnthropicResponseParserTest.java | 1 - 3 files changed, 3 deletions(-) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java index 4e14daffa2..0331652bde 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java @@ -326,4 +326,3 @@ public PromptCachePlan applyPromptCache( /** Provider-ready messages and tool cache decision for one request. */ public record PromptCachePlan(List messages) {} } - diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/AnthropicChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/AnthropicChatModelTest.java index cff13b7125..815e22b631 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/AnthropicChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/AnthropicChatModelTest.java @@ -505,4 +505,3 @@ void testCallWithCacheControlDisabled() { .verify(Duration.ofSeconds(60)); } } - diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java index 3679b21e39..0dd8bbcd93 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java @@ -596,4 +596,3 @@ void testParseStreamEventsDeltaCarryingOwnPromptUsage() { .verifyComplete(); } } -