diff --git a/agentscope-core/src/main/java/io/agentscope/core/ReActAgent.java b/agentscope-core/src/main/java/io/agentscope/core/ReActAgent.java index 86879c1172..69ccd41277 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/ReActAgent.java +++ b/agentscope-core/src/main/java/io/agentscope/core/ReActAgent.java @@ -1449,6 +1449,7 @@ private ChatUsage collectAggregatedUsage(AgentState agentState) { int totalInput = 0; int totalOutput = 0; int totalCached = 0; + int totalCacheCreation = 0; double totalTime = 0; boolean hasUsage = false; for (Msg msg : agentState.getContext()) { @@ -1459,6 +1460,7 @@ private ChatUsage collectAggregatedUsage(AgentState agentState) { totalInput += usage.getInputTokens(); totalOutput += usage.getOutputTokens(); totalCached += usage.getCachedTokens(); + totalCacheCreation += usage.getCacheCreationInputTokens(); totalTime += usage.getTime(); } } @@ -1468,6 +1470,7 @@ private ChatUsage collectAggregatedUsage(AgentState agentState) { .inputTokens(totalInput) .outputTokens(totalOutput) .cachedTokens(totalCached) + .cacheCreationInputTokens(totalCacheCreation) .time(totalTime) .build() : null; diff --git a/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java b/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java index bf08581b4c..7c08a437fb 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java +++ b/agentscope-core/src/main/java/io/agentscope/core/agent/accumulator/ReasoningContext.java @@ -56,6 +56,7 @@ public class ReasoningContext { private int inputTokens = 0; private int outputTokens = 0; private int cachedTokens = 0; + private int cacheCreationInputTokens = 0; private double time = 0; public ReasoningContext(String agentName) { @@ -85,6 +86,7 @@ public List processChunk(ChatResponse chunk) { inputTokens = usage.getInputTokens(); outputTokens = usage.getOutputTokens(); cachedTokens = usage.getCachedTokens(); + cacheCreationInputTokens = usage.getCacheCreationInputTokens(); time = usage.getTime(); } @@ -169,12 +171,17 @@ public Msg buildFinalMessage() { // Build metadata with accumulated ChatUsage Map metadata = new HashMap<>(); ChatUsage chatUsage = null; - if (inputTokens > 0 || outputTokens > 0 || time > 0) { + if (inputTokens > 0 + || outputTokens > 0 + || cachedTokens > 0 + || cacheCreationInputTokens > 0 + || time > 0) { chatUsage = ChatUsage.builder() .inputTokens(inputTokens) .outputTokens(outputTokens) .cachedTokens(cachedTokens) + .cacheCreationInputTokens(cacheCreationInputTokens) .time(time) .build(); metadata.put(MessageMetadataKeys.CHAT_USAGE, chatUsage); @@ -287,11 +294,16 @@ public List getAllAccumulatedToolCalls() { * @return ChatUsage with accumulated tokens, or null if no usage data */ public ChatUsage getChatUsage() { - if (inputTokens > 0 || outputTokens > 0 || time > 0) { + if (inputTokens > 0 + || outputTokens > 0 + || cachedTokens > 0 + || cacheCreationInputTokens > 0 + || time > 0) { return ChatUsage.builder() .inputTokens(inputTokens) .outputTokens(outputTokens) .cachedTokens(cachedTokens) + .cacheCreationInputTokens(cacheCreationInputTokens) .time(time) .build(); } diff --git a/agentscope-core/src/main/java/io/agentscope/core/formatter/AbstractBaseFormatter.java b/agentscope-core/src/main/java/io/agentscope/core/formatter/AbstractBaseFormatter.java index 16bab108e7..5743a0b0c1 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/formatter/AbstractBaseFormatter.java +++ b/agentscope-core/src/main/java/io/agentscope/core/formatter/AbstractBaseFormatter.java @@ -37,8 +37,10 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Base64; +import java.util.Collections; import java.util.List; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.slf4j.Logger; @@ -62,6 +64,8 @@ public abstract class AbstractBaseFormatter implements Formatter { + protected static final int MAX_PROMPT_CACHE_BREAKPOINTS = 4; + private static final Logger log = LoggerFactory.getLogger(AbstractBaseFormatter.class); /** @@ -160,8 +164,11 @@ protected String formatRoleLabel(MsgRole role) { /** * Check if a message should bypass history merging in multiagent formatters. - * Messages with the {@link MessageMetadataKeys#BYPASS_MULTIAGENT_HISTORY_MERGE} flag set to {@code true} - * should be kept as separate messages rather than merged into the conversation history. + * Messages with the {@link MessageMetadataKeys#BYPASS_MULTIAGENT_HISTORY_MERGE} or {@link + * MessageMetadataKeys#CACHE_CONTROL} flag set to either {@code true} or {@code false} should be + * kept as separate messages rather than merged into the conversation history. Preserving + * explicitly marked cache boundaries prevents formatter-level history merging from discarding + * the marker. * * @param msg The message to check * @return true if message should bypass history merging @@ -172,7 +179,80 @@ protected boolean shouldBypassHistory(Msg msg) { } Object bypassFlag = msg.getMetadata().get(MessageMetadataKeys.BYPASS_MULTIAGENT_HISTORY_MERGE); - return Boolean.TRUE.equals(bypassFlag); + Object cacheControlFlag = msg.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL); + return Boolean.TRUE.equals(bypassFlag) || cacheControlFlag instanceof Boolean; + } + + /** + * Select prompt cache breakpoints while respecting the provider limit. + * + *

Explicitly marked items always take priority. When automatic selection is enabled, the + * first cacheable system item and the last cacheable non-system item fill any remaining slots. + * The returned items preserve their original request order. + * + * @param items provider request items to inspect + * @param automatic whether automatic cache breakpoint selection is enabled + * @param explicitlyMarked predicate identifying explicitly marked items + * @param systemItem predicate identifying system items + * @param cacheableItem predicate identifying items eligible for automatic caching + * @param provider request item type + * @return selected cache breakpoint items in request order + * @throws IllegalArgumentException when more than four items are explicitly marked + */ + protected List selectPromptCacheBreakpoints( + List items, + boolean automatic, + Predicate explicitlyMarked, + Predicate systemItem, + Predicate cacheableItem) { + if (items == null || items.isEmpty()) { + return List.of(); + } + + List selectedIndices = new ArrayList<>(); + for (int i = 0; i < items.size(); i++) { + if (explicitlyMarked.test(items.get(i))) { + selectedIndices.add(i); + } + } + + if (selectedIndices.size() > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "Prompt cache supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " explicit breakpoints, but got " + + selectedIndices.size()); + } + + if (automatic && selectedIndices.size() < MAX_PROMPT_CACHE_BREAKPOINTS) { + for (int i = 0; i < items.size(); i++) { + T item = items.get(i); + if (systemItem.test(item) && cacheableItem.test(item)) { + addBreakpointIfAvailable(selectedIndices, i); + break; + } + } + + for (int i = items.size() - 1; + i >= 0 && selectedIndices.size() < MAX_PROMPT_CACHE_BREAKPOINTS; + i--) { + T item = items.get(i); + if (!systemItem.test(item) && cacheableItem.test(item)) { + addBreakpointIfAvailable(selectedIndices, i); + break; + } + } + } + + Collections.sort(selectedIndices); + return selectedIndices.stream().map(items::get).toList(); + } + + private void addBreakpointIfAvailable(List selectedIndices, int index) { + if (selectedIndices.size() < MAX_PROMPT_CACHE_BREAKPOINTS + && !selectedIndices.contains(index)) { + selectedIndices.add(index); + } } /** diff --git a/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java b/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java index 55243785ad..9c1e36be8f 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java +++ b/agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java @@ -109,16 +109,16 @@ private MessageMetadataKeys() { /** * Metadata key to explicitly mark a message for prompt caching or non-caching. * - *

When set to {@code true}, the formatter adds cache_control: - * {"type": "ephemeral"} to this message during formatting, unless the message already - * carries a custom cache_control value (e.g. with additional attributes such as - * ttl), in which case it is left untouched. When set to {@code false}, the - * message is explicitly excluded from caching: no cache_control is emitted for it, - * and the automatic cache control strategy configured via - * {@link io.agentscope.core.model.GenerateOptions#getCacheControl()} skips it. + *

When set to {@code true}, a supporting formatter encodes a provider-specific cache + * breakpoint at this message, unless an existing custom {@code cache_control} value is present. + * Explicit cache markers remain effective when the automatic strategy configured through + * {@link io.agentscope.core.model.GenerateOptions#getCacheControl()} is disabled or unset. When + * set to {@code false}, the message is explicitly excluded from caching: no provider cache + * marker is emitted for it, and the automatic strategy skips it. * *

Explicitly marked messages (either {@code true} or {@code false}) take priority over the - * automatic strategy — they will not be overwritten. + * automatic strategy — they will not be overwritten. A request may contain at most four cache + * breakpoints explicitly enabled with {@code true}. * *

Type: Boolean *

Example: diff --git a/agentscope-core/src/main/java/io/agentscope/core/message/Msg.java b/agentscope-core/src/main/java/io/agentscope/core/message/Msg.java index 82df1e01cf..1cded18b74 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/message/Msg.java +++ b/agentscope-core/src/main/java/io/agentscope/core/message/Msg.java @@ -590,6 +590,8 @@ public ChatUsage getChatUsage() { ChatUsage.builder() .inputTokens(toInt(map.get("inputTokens"))) .outputTokens(toInt(map.get("outputTokens"))) + .cachedTokens(toInt(map.get("cachedTokens"))) + .cacheCreationInputTokens(toInt(map.get("cacheCreationInputTokens"))) .time(toDouble(map.get("time"))) .build(); metadata.put(MessageMetadataKeys.CHAT_USAGE, chatUsage); diff --git a/agentscope-core/src/main/java/io/agentscope/core/model/ChatUsage.java b/agentscope-core/src/main/java/io/agentscope/core/model/ChatUsage.java index 1e22b8ddeb..2a9a13cf27 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/model/ChatUsage.java +++ b/agentscope-core/src/main/java/io/agentscope/core/model/ChatUsage.java @@ -22,14 +22,15 @@ * Represents token usage information for chat completion responses. * *

This immutable data class tracks the number of tokens used during a chat completion, - * including input tokens (prompt), output tokens (generated response), cached input tokens, and - * execution time. + * including input tokens (prompt), output tokens (generated response), prompt cache reads and + * writes, and execution time. */ public class ChatUsage { private final int inputTokens; private final int outputTokens; private final int cachedTokens; + private final int cacheCreationInputTokens; private final double time; /** @@ -55,15 +56,33 @@ public ChatUsage(int inputTokens, int outputTokens, double time) { * {@code inputTokens}); {@code 0} when the provider does not report cache information * @param time the execution time in seconds */ + public ChatUsage(int inputTokens, int outputTokens, int cachedTokens, double time) { + this(inputTokens, outputTokens, cachedTokens, 0, time); + } + + /** + * Creates a new ChatUsage instance with prompt cache read and creation information. + * + * @param inputTokens the total number of tokens used for the input/prompt + * @param outputTokens the number of tokens used for the output/generated response + * @param cachedTokens the number of input tokens served from the prompt cache (a subset of + * {@code inputTokens}); {@code 0} when the provider does not report cache information + * @param cacheCreationInputTokens the number of input tokens written to the prompt cache (a + * subset of {@code inputTokens}); {@code 0} when the provider does not report cache + * creation information + * @param time the execution time in seconds + */ @JsonCreator public ChatUsage( @JsonProperty("inputTokens") int inputTokens, @JsonProperty("outputTokens") int outputTokens, @JsonProperty("cachedTokens") int cachedTokens, + @JsonProperty("cacheCreationInputTokens") int cacheCreationInputTokens, @JsonProperty("time") double time) { this.inputTokens = inputTokens; this.outputTokens = outputTokens; this.cachedTokens = cachedTokens; + this.cacheCreationInputTokens = cacheCreationInputTokens; this.time = time; } @@ -98,6 +117,18 @@ public int getCachedTokens() { return cachedTokens; } + /** + * Gets the number of input tokens written to the prompt cache. + * + *

Cache creation tokens are a subset of {@link #getInputTokens()}, not an additional amount. + * Returns {@code 0} when the provider does not report cache creation information. + * + * @return the number of input tokens written to the prompt cache + */ + public int getCacheCreationInputTokens() { + return cacheCreationInputTokens; + } + /** * Gets the total number of tokens used. * @@ -132,6 +163,7 @@ public static class Builder { private int inputTokens; private int outputTokens; private int cachedTokens; + private int cacheCreationInputTokens; private double time; /** @@ -168,6 +200,18 @@ public Builder cachedTokens(int cachedTokens) { return this; } + /** + * Sets the number of input tokens written to the prompt cache. + * + * @param cacheCreationInputTokens the number of input tokens written to the prompt cache + * (a subset of {@code inputTokens}) + * @return this builder instance + */ + public Builder cacheCreationInputTokens(int cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + return this; + } + /** * Sets the execution time. * @@ -185,7 +229,8 @@ public Builder time(double time) { * @return a new ChatUsage instance */ public ChatUsage build() { - return new ChatUsage(inputTokens, outputTokens, cachedTokens, time); + return new ChatUsage( + inputTokens, outputTokens, cachedTokens, cacheCreationInputTokens, time); } } } diff --git a/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java b/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java index 87fc0243ec..1970936493 100644 --- a/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java +++ b/agentscope-core/src/main/java/io/agentscope/core/model/GenerateOptions.java @@ -309,15 +309,14 @@ public Long getSeed() { /** * Gets whether cache control is enabled for prompt caching. * - *

When true, the formatter will automatically add cache_control: - * {"type": "ephemeral"} to system messages and the last message in the request. This - * enables prompt - * caching on supported providers (e.g., Anthropic, DashScope, OpenAI-compatible APIs) to reduce - * latency and cost. + *

When true, a supporting provider enables its automatic prompt-caching strategy. Some + * providers select request breakpoints in the formatter, while others expose a provider-native + * automatic caching mode. * *

Users can also manually mark individual messages for caching via {@link * io.agentscope.core.message.MessageMetadataKeys#CACHE_CONTROL} metadata. Manually marked - * messages take priority over the automatic strategy. + * messages take priority over the automatic strategy and remain effective when this option is + * false or unset. Providers currently support at most four explicit breakpoints per request. * * @return true if cache control is enabled, false or null if not set */ @@ -780,8 +779,8 @@ public Builder seed(Long seed) { /** * Sets whether cache control is enabled for prompt caching. * - *

When true, the formatter will automatically add cache_control: - * {"type": "ephemeral"} to system messages and the last message in the request. + *

When true, a supporting provider enables its automatic prompt-caching strategy using + * the request shape defined by that provider. * * @param cacheControl true to enable cache control, false to disable * @return this builder instance diff --git a/agentscope-core/src/test/java/io/agentscope/core/agent/ReActAgentStructuredOutputTest.java b/agentscope-core/src/test/java/io/agentscope/core/agent/ReActAgentStructuredOutputTest.java index e6a1a49957..a537d6a1e5 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/agent/ReActAgentStructuredOutputTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/agent/ReActAgentStructuredOutputTest.java @@ -388,7 +388,14 @@ void testStructuredOutputPreservesChatUsage() { .toJson( toolInput)) .build())) - .usage(new ChatUsage(100, 50, 1.5)) + .usage( + ChatUsage.builder() + .inputTokens(100) + .outputTokens(50) + .cachedTokens(40) + .cacheCreationInputTokens(25) + .time(1.5) + .build()) .build()); } else { return List.of( @@ -435,6 +442,11 @@ void testStructuredOutputPreservesChatUsage() { assertNotNull(usage, "ChatUsage should be preserved after structured output compression"); assertEquals(100, usage.getInputTokens(), "Input tokens should be preserved"); assertEquals(50, usage.getOutputTokens(), "Output tokens should be preserved"); + assertEquals(40, usage.getCachedTokens(), "Cached tokens should be preserved"); + assertEquals( + 25, + usage.getCacheCreationInputTokens(), + "Cache creation tokens should be preserved"); assertEquals(1.5, usage.getTime(), 0.01, "Time should be preserved"); } diff --git a/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java b/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java index 83b00def5e..ce4ef35019 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/agent/accumulator/ReasoningContextTest.java @@ -82,6 +82,7 @@ void testCachedTokensPropagation() { .inputTokens(100) .outputTokens(50) .cachedTokens(80) + .cacheCreationInputTokens(15) .time(1.5) .build(); @@ -98,10 +99,27 @@ void testCachedTokensPropagation() { assertNotNull(msg); assertNotNull(msg.getChatUsage()); assertEquals(80, msg.getChatUsage().getCachedTokens()); + assertEquals(15, msg.getChatUsage().getCacheCreationInputTokens()); ChatUsage resultUsage = context.getChatUsage(); assertNotNull(resultUsage); assertEquals(80, resultUsage.getCachedTokens()); + assertEquals(15, resultUsage.getCacheCreationInputTokens()); + } + + @Test + @DisplayName("Should retain cache-only usage") + void testCacheOnlyUsage() { + ChatUsage usage = ChatUsage.builder().cachedTokens(80).cacheCreationInputTokens(15).build(); + ChatResponse chunk = + ChatResponse.builder().id("msg-cache-only").content(List.of()).usage(usage).build(); + + context.processChunk(chunk); + + ChatUsage resultUsage = context.getChatUsage(); + assertNotNull(resultUsage); + assertEquals(80, resultUsage.getCachedTokens()); + assertEquals(15, resultUsage.getCacheCreationInputTokens()); } @Test diff --git a/agentscope-core/src/test/java/io/agentscope/core/formatter/AbstractBaseFormatterPromptCacheTest.java b/agentscope-core/src/test/java/io/agentscope/core/formatter/AbstractBaseFormatterPromptCacheTest.java new file mode 100644 index 0000000000..4e6b843d72 --- /dev/null +++ b/agentscope-core/src/test/java/io/agentscope/core/formatter/AbstractBaseFormatterPromptCacheTest.java @@ -0,0 +1,162 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.formatter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.agentscope.core.message.MessageMetadataKeys; +import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; +import io.agentscope.core.model.ChatResponse; +import io.agentscope.core.model.GenerateOptions; +import io.agentscope.core.model.ToolSchema; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("AbstractBaseFormatter prompt cache tests") +class AbstractBaseFormatterPromptCacheTest { + + private final TestFormatter formatter = new TestFormatter(); + + @Test + void selectsFirstSystemAndLastCacheableConversationItemAutomatically() { + CacheItem firstSystem = new CacheItem("system-1", false, true, true); + CacheItem secondSystem = new CacheItem("system-2", false, true, true); + CacheItem user = new CacheItem("user", false, false, true); + CacheItem emptyAssistant = new CacheItem("assistant-empty", false, false, false); + + List selected = + formatter.select(List.of(firstSystem, secondSystem, user, emptyAssistant), true); + + assertEquals(List.of(firstSystem, user), selected); + } + + @Test + void keepsExplicitBreakpointsAndPreservesRequestOrder() { + CacheItem system = new CacheItem("system", false, true, true); + CacheItem explicitUser = new CacheItem("user", true, false, true); + CacheItem assistant = new CacheItem("assistant", false, false, true); + CacheItem explicitTool = new CacheItem("tool", true, false, true); + + List selected = + formatter.select(List.of(system, explicitUser, assistant, explicitTool), true); + + assertEquals(List.of(system, explicitUser, explicitTool), selected); + } + + @Test + void explicitBreakpointsConsumeCapacityBeforeAutomaticBreakpoints() { + CacheItem system = new CacheItem("system", false, true, true); + CacheItem explicitOne = new CacheItem("one", true, false, true); + CacheItem explicitTwo = new CacheItem("two", true, false, true); + CacheItem explicitThree = new CacheItem("three", true, false, true); + CacheItem last = new CacheItem("last", false, false, true); + + List selected = + formatter.select( + List.of(system, explicitOne, explicitTwo, explicitThree, last), true); + + assertEquals(List.of(system, explicitOne, explicitTwo, explicitThree), selected); + } + + @Test + void disablingAutomaticSelectionKeepsOnlyExplicitBreakpoints() { + CacheItem system = new CacheItem("system", false, true, true); + CacheItem explicit = new CacheItem("explicit", true, false, true); + CacheItem last = new CacheItem("last", false, false, true); + + assertEquals(List.of(explicit), formatter.select(List.of(system, explicit, last), false)); + } + + @Test + void rejectsMoreThanFourExplicitBreakpoints() { + List items = + List.of( + new CacheItem("one", true, false, true), + new CacheItem("two", true, false, true), + new CacheItem("three", true, false, true), + new CacheItem("four", true, false, true), + new CacheItem("five", true, false, true)); + + IllegalArgumentException error = + assertThrows(IllegalArgumentException.class, () -> formatter.select(items, true)); + + assertTrue(error.getMessage().contains("at most 4 explicit breakpoints")); + } + + @Test + void returnsEmptySelectionForNullOrEmptyInput() { + assertTrue(formatter.select(null, true).isEmpty()); + assertTrue(formatter.select(List.of(), true).isEmpty()); + } + + @Test + void explicitCacheControlPreservesMultiAgentMessageBoundary() { + Msg cached = + Msg.builder() + .name("user") + .role(MsgRole.USER) + .textContent("cache me") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + Msg notCached = + Msg.builder() + .name("user") + .role(MsgRole.USER) + .textContent("merge me") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, false)) + .build(); + + assertTrue(formatter.bypassesHistory(cached)); + assertTrue(formatter.bypassesHistory(notCached)); + } + + private record CacheItem(String name, boolean explicit, boolean system, boolean cacheable) {} + + private static final class TestFormatter extends AbstractBaseFormatter { + + List select(List items, boolean automatic) { + return selectPromptCacheBreakpoints( + items, automatic, CacheItem::explicit, CacheItem::system, CacheItem::cacheable); + } + + boolean bypassesHistory(Msg msg) { + return shouldBypassHistory(msg); + } + + @Override + protected List doFormat(List msgs) { + return List.of(); + } + + @Override + public ChatResponse parseResponse(String response, Instant startTime) { + return null; + } + + @Override + public void applyOptions( + String paramsBuilder, GenerateOptions options, GenerateOptions defaultOptions) {} + + @Override + public void applyTools(String paramsBuilder, List tools) {} + } +} diff --git a/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java b/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java index 1fafc0233e..e68d7dbba2 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/message/MsgUsageSerializationTest.java @@ -50,7 +50,12 @@ void usageFieldRoundTripsViaJson() { @Test void cachedTokensRoundTripViaJson() { ChatUsage usage = - ChatUsage.builder().inputTokens(100).outputTokens(50).cachedTokens(30).build(); + ChatUsage.builder() + .inputTokens(100) + .outputTokens(50) + .cachedTokens(30) + .cacheCreationInputTokens(20) + .build(); Msg msg = Msg.builder() .name("assistant") @@ -68,12 +73,34 @@ void cachedTokensRoundTripViaJson() { assertEquals(100, deserialized.getUsage().getInputTokens()); assertEquals(50, deserialized.getUsage().getOutputTokens()); assertEquals(30, deserialized.getUsage().getCachedTokens()); + assertEquals(20, deserialized.getUsage().getCacheCreationInputTokens()); } @Test void cachedTokensDefaultsToZeroForLegacyConstructor() { ChatUsage usage = new ChatUsage(100, 50, 1.5); assertEquals(0, usage.getCachedTokens()); + assertEquals(0, usage.getCacheCreationInputTokens()); + } + + @Test + void cacheCreationTokensDefaultToZeroForFourArgumentConstructor() { + ChatUsage usage = new ChatUsage(100, 50, 30, 1.5); + + assertEquals(30, usage.getCachedTokens()); + assertEquals(0, usage.getCacheCreationInputTokens()); + } + + @Test + void legacyUsageJsonDefaultsCacheCreationTokensToZero() { + String json = "{\"inputTokens\":100,\"outputTokens\":50,\"cachedTokens\":30,\"time\":1.5}"; + + ChatUsage usage = JsonUtils.getJsonCodec().fromJson(json, ChatUsage.class); + + assertEquals(100, usage.getInputTokens()); + assertEquals(50, usage.getOutputTokens()); + assertEquals(30, usage.getCachedTokens()); + assertEquals(0, usage.getCacheCreationInputTokens()); } @Test @@ -117,6 +144,34 @@ void getChatUsageFallsBackToMetadata() { assertEquals(300, retrieved.getInputTokens()); } + @Test + void getChatUsageMapFallbackPreservesPromptCacheUsage() { + Msg msg = + Msg.builder() + .name("assistant") + .role(MsgRole.ASSISTANT) + .textContent("reply") + .metadata( + java.util.Map.of( + MessageMetadataKeys.CHAT_USAGE, + java.util.Map.of( + "inputTokens", 300, + "outputTokens", 150, + "cachedTokens", 120, + "cacheCreationInputTokens", 30, + "time", 3.0))) + .build(); + + ChatUsage retrieved = msg.getChatUsage(); + + assertNotNull(retrieved); + assertEquals(300, retrieved.getInputTokens()); + assertEquals(150, retrieved.getOutputTokens()); + assertEquals(120, retrieved.getCachedTokens()); + assertEquals(30, retrieved.getCacheCreationInputTokens()); + assertEquals(3.0, retrieved.getTime()); + } + @Test void usageDeserializesFromJsonWithoutField() { String json = diff --git a/agentscope-core/src/test/java/io/agentscope/core/model/GenerateOptionsTest.java b/agentscope-core/src/test/java/io/agentscope/core/model/GenerateOptionsTest.java index 9db349d158..01d44c5166 100644 --- a/agentscope-core/src/test/java/io/agentscope/core/model/GenerateOptionsTest.java +++ b/agentscope-core/src/test/java/io/agentscope/core/model/GenerateOptionsTest.java @@ -356,6 +356,28 @@ void testMergeOptionsWithAdditionalParams() { assertEquals("fallback", merged.getAdditionalBodyParams().get("fallback_param")); } + @Test + @DisplayName("Should prefer primary cache control when merging options") + void testMergeOptionsPrefersPrimaryCacheControl() { + GenerateOptions primary = GenerateOptions.builder().cacheControl(false).build(); + GenerateOptions fallback = GenerateOptions.builder().cacheControl(true).build(); + + GenerateOptions merged = GenerateOptions.mergeOptions(primary, fallback); + + assertEquals(Boolean.FALSE, merged.getCacheControl()); + } + + @Test + @DisplayName("Should inherit fallback cache control when primary is unset") + void testMergeOptionsInheritsFallbackCacheControl() { + GenerateOptions primary = GenerateOptions.builder().temperature(0.8).build(); + GenerateOptions fallback = GenerateOptions.builder().cacheControl(true).build(); + + GenerateOptions merged = GenerateOptions.mergeOptions(primary, fallback); + + assertEquals(Boolean.TRUE, merged.getCacheControl()); + } + @Test @DisplayName("Should set additional headers using map") void testSetAdditionalHeadersMap() { diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/AnthropicChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/AnthropicChatModel.java index c073acdbc3..726fa9c4bf 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/AnthropicChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/AnthropicChatModel.java @@ -162,42 +162,28 @@ protected Flux doStream( .model(modelName) .maxTokens(4096); - GenerateOptions effectiveOptions = - GenerateOptions.mergeOptions(options, defaultOptions); - boolean cacheControlEnabled = - effectiveOptions != null - && Boolean.TRUE.equals( - effectiveOptions.getCacheControl()); - - // Extract and apply system message - // (Anthropic-specific requirement); - // adds cache_control when prompt caching is enabled - formatter.applySystemMessage( - paramsBuilder, messages, cacheControlEnabled); - - // The leading system message has been applied to the `system` - // field above. Exclude it from the message body so the system - // prompt isn't sent twice. - List conversationMessages = messages; - if (messages != null - && !messages.isEmpty() - && messages.get(0).getRole() == MsgRole.SYSTEM) { - conversationMessages = messages.subList(1, messages.size()); - } - // Use formatter to convert Msg to Anthropic // MessageParam + List providerMessages = + messages != null + && !messages.isEmpty() + && messages.get(0).getRole() + == MsgRole.SYSTEM + ? messages.subList(1, messages.size()) + : messages; List formattedMessages = - formatter.format(conversationMessages); - - // Apply automatic cache control strategy - // (marks the last message to cache the conversation prefix) - if (cacheControlEnabled) { - formattedMessages = - formatter.applyCacheControl(formattedMessages); - } - - for (MessageParam param : formattedMessages) { + formatter.format(providerMessages); + GenerateOptions effectiveOptions = + GenerateOptions.mergeOptions(options, defaultOptions); + boolean automaticCache = + Boolean.TRUE.equals(effectiveOptions.getCacheControl()); + AnthropicBaseFormatter.PromptCachePlan cachePlan = + formatter.applyPromptCache( + paramsBuilder, + messages, + formattedMessages, + automaticCache); + for (MessageParam param : cachePlan.messages()) { paramsBuilder.addMessage(param); } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java index d3537f4788..0331652bde 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicBaseFormatter.java @@ -15,17 +15,21 @@ */ package io.agentscope.extensions.model.anthropic.formatter; +import com.anthropic.core.JsonValue; import com.anthropic.models.messages.CacheControlEphemeral; import com.anthropic.models.messages.ContentBlockParam; import com.anthropic.models.messages.MessageCreateParams; import com.anthropic.models.messages.MessageParam; import com.anthropic.models.messages.TextBlockParam; import io.agentscope.core.formatter.AbstractBaseFormatter; +import io.agentscope.core.message.MessageMetadataKeys; import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; import io.agentscope.core.model.GenerateOptions; import io.agentscope.core.model.ToolSchema; import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Abstract base formatter for Anthropic API with shared logic for handling Anthropic-specific @@ -251,4 +255,74 @@ private static ContentBlockParam withCacheControl( } return null; } + + /** + * Encode explicit and automatic prompt-cache breakpoints into an Anthropic request. + * Explicit message metadata always remains effective. Automatic selection uses Anthropic's + * provider-native top-level cache control, which moves the final breakpoint as a conversation + * grows. + */ + public PromptCachePlan applyPromptCache( + MessageCreateParams.Builder paramsBuilder, + List originalMessages, + List formattedMessages, + boolean automatic) { + List plannedMessages = new ArrayList<>(formattedMessages); + Msg system = + originalMessages != null + && !originalMessages.isEmpty() + && originalMessages.get(0).getRole() == MsgRole.SYSTEM + ? originalMessages.get(0) + : null; + + boolean explicitSystem = + system != null + && system.getMetadata() != null + && Boolean.TRUE.equals( + system.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL)); + int explicitCount = explicitSystem ? 1 : 0; + explicitCount += + (int) + plannedMessages.stream() + .filter(AnthropicPromptCacheSupport::hasCacheControl) + .count(); + if (explicitCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "Prompt cache supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " explicit breakpoints, but got " + + explicitCount); + } + if (automatic && explicitCount == MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "Anthropic automatic prompt caching requires one of the four breakpoint slots"); + } + if (automatic) { + paramsBuilder.putAdditionalBodyProperty( + "cache_control", + JsonValue.from( + cacheTtl == null || cacheTtl.isEmpty() + ? Map.of("type", "ephemeral") + : Map.of("type", "ephemeral", "ttl", cacheTtl))); + } + + if (system != null) { + String systemText = messageConverter.extractSystemMessage(originalMessages); + if (systemText != null && !systemText.isEmpty()) { + TextBlockParam.Builder block = TextBlockParam.builder().text(systemText); + if (explicitSystem) { + block.cacheControl(buildCacheControl(cacheTtl)); + } + paramsBuilder.systemOfTextBlockParams(List.of(block.build())); + } else if (explicitSystem) { + throw new IllegalArgumentException( + "Explicit Anthropic system cache breakpoint has no text content"); + } + } + + return new PromptCachePlan(List.copyOf(plannedMessages)); + } + + /** Provider-ready messages and tool cache decision for one request. */ + public record PromptCachePlan(List messages) {} } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMessageConverter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMessageConverter.java index 60ec2f1035..5f7538bbac 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMessageConverter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMessageConverter.java @@ -80,12 +80,15 @@ public List convert(List messages) { for (int i = 0; i < messages.size(); i++) { Msg msg = messages.get(i); boolean isFirstMessage = (i == 0); + int resultStart = result.size(); if (msg.getRole() == MsgRole.ASSISTANT && msg.getContentBlocks(ToolUseBlock.class).size() > 1) { SplitToolResultSequence splitResults = collectSplitToolResults(messages, i + 1); if (shouldSplitParallelToolCalls(msg, splitResults)) { result.addAll(convertParallelToolCalls(msg, splitResults)); + applyExplicitBreakpoints( + result, resultStart, msg, messages, i + 1, splitResults); i += splitResults.consumedMessages(); continue; } @@ -123,11 +126,73 @@ public List convert(List messages) { result.add(param); } } + + markLastProducedMessage(result, resultStart, msg); } return result; } + private void markLastProducedMessage(List result, int start, Msg source) { + if (!AnthropicPromptCacheSupport.isExplicitlyMarked(source)) { + return; + } + if (result.size() == start) { + throw new IllegalArgumentException( + "Explicit Anthropic prompt cache breakpoint has no provider message"); + } + int last = result.size() - 1; + result.set(last, AnthropicPromptCacheSupport.markLastCacheableBlock(result.get(last))); + } + + private void applyExplicitBreakpoints( + List result, + int start, + Msg assistant, + List messages, + int toolResultStart, + SplitToolResultSequence splitResults) { + if (AnthropicPromptCacheSupport.isExplicitlyMarked(assistant)) { + int assistantIndex = start + (splitResults.resultsById().size() - 1) * 2; + result.set( + assistantIndex, + AnthropicPromptCacheSupport.markLastCacheableBlock(result.get(assistantIndex))); + } + + for (int i = toolResultStart; i < toolResultStart + splitResults.consumedMessages(); i++) { + Msg toolResultMsg = messages.get(i); + if (AnthropicPromptCacheSupport.isExplicitlyMarked(toolResultMsg)) { + List sourceResults = + toolResultMsg.getContentBlocks(ToolResultBlock.class); + String lastToolUseId = sourceResults.get(sourceResults.size() - 1).getId(); + int providerIndex = findToolResultMessage(result, start, lastToolUseId); + result.set( + providerIndex, + AnthropicPromptCacheSupport.markLastCacheableBlock( + result.get(providerIndex))); + } + } + } + + private int findToolResultMessage( + List result, int start, String expectedToolUseId) { + for (int i = result.size() - 1; i >= start; i--) { + MessageParam param = result.get(i); + if (!param.content().isBlockParams()) { + continue; + } + for (ContentBlockParam block : param.content().asBlockParams()) { + if (block.isToolResult() + && java.util.Objects.equals( + block.asToolResult().toolUseId(), expectedToolUseId)) { + return i; + } + } + } + throw new IllegalArgumentException( + "Explicit Anthropic tool-result cache breakpoint lost during conversion"); + } + /** * Decide whether the current assistant message should be expanded into Anthropic's required * tool-use/tool-result alternation. diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMultiAgentFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMultiAgentFormatter.java index 7ff7a644ba..e5cb8168d0 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMultiAgentFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicMultiAgentFormatter.java @@ -89,6 +89,7 @@ public List doFormat(List msgs) { } case TOOL_SEQUENCE -> result.addAll(formatToolSequence(group.messages)); case AGENT_CONVERSATION -> result.addAll(formatAgentConversation(group.messages)); + case BYPASS -> result.addAll(messageConverter.convert(group.messages)); } } @@ -109,7 +110,8 @@ public ChatResponse parseResponse(Object response, Instant startTime) { private enum GroupType { SYSTEM, TOOL_SEQUENCE, - AGENT_CONVERSATION + AGENT_CONVERSATION, + BYPASS } private static class MessageGroup { @@ -158,6 +160,9 @@ private List groupMessages(List msgs) { /** Determine message type for grouping. */ private GroupType determineMessageType(Msg msg) { + if (shouldBypassHistory(msg)) { + return GroupType.BYPASS; + } if (msg.getRole() == MsgRole.SYSTEM && !msg.getContent().isEmpty() && !(msg.getContent().get(0) instanceof ToolResultBlock)) { diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java new file mode 100644 index 0000000000..8b8ebd1226 --- /dev/null +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheSupport.java @@ -0,0 +1,106 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.extensions.model.anthropic.formatter; + +import com.anthropic.models.messages.CacheControlEphemeral; +import com.anthropic.models.messages.ContentBlockParam; +import com.anthropic.models.messages.MessageParam; +import io.agentscope.core.message.MessageMetadataKeys; +import io.agentscope.core.message.Msg; +import java.util.ArrayList; +import java.util.List; + +/** Utilities for encoding AgentScope prompt-cache intent into Anthropic request blocks. */ +final class AnthropicPromptCacheSupport { + + private static final CacheControlEphemeral EPHEMERAL = CacheControlEphemeral.builder().build(); + + private AnthropicPromptCacheSupport() {} + + static CacheControlEphemeral ephemeral() { + return EPHEMERAL; + } + + static boolean isExplicitlyMarked(Msg msg) { + return msg != null + && msg.getMetadata() != null + && Boolean.TRUE.equals(msg.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL)); + } + + static boolean hasCacheControl(MessageParam message) { + if (!message.content().isBlockParams()) { + return false; + } + return message.content().asBlockParams().stream() + .anyMatch(AnthropicPromptCacheSupport::hasCacheControl); + } + + static MessageParam markLastCacheableBlock(MessageParam message) { + if (!message.content().isBlockParams()) { + throw new IllegalArgumentException( + "Anthropic prompt cache breakpoint requires block content"); + } + + List blocks = new ArrayList<>(message.content().asBlockParams()); + for (int i = blocks.size() - 1; i >= 0; i--) { + ContentBlockParam marked = mark(blocks.get(i)); + if (marked != null) { + blocks.set(i, marked); + return message.toBuilder() + .content(MessageParam.Content.ofBlockParams(blocks)) + .build(); + } + } + throw new IllegalArgumentException( + "Anthropic prompt cache breakpoint has no cacheable content block"); + } + + private static boolean hasCacheControl(ContentBlockParam block) { + if (block.isText()) { + return block.asText().cacheControl().isPresent(); + } + if (block.isImage()) { + return block.asImage().cacheControl().isPresent(); + } + if (block.isToolUse()) { + return block.asToolUse().cacheControl().isPresent(); + } + if (block.isToolResult()) { + return block.asToolResult().cacheControl().isPresent(); + } + return false; + } + + private static ContentBlockParam mark(ContentBlockParam block) { + if (block.isText()) { + return ContentBlockParam.ofText( + block.asText().toBuilder().cacheControl(EPHEMERAL).build()); + } + if (block.isImage()) { + return ContentBlockParam.ofImage( + block.asImage().toBuilder().cacheControl(EPHEMERAL).build()); + } + if (block.isToolUse()) { + return ContentBlockParam.ofToolUse( + block.asToolUse().toBuilder().cacheControl(EPHEMERAL).build()); + } + if (block.isToolResult()) { + return ContentBlockParam.ofToolResult( + block.asToolResult().toBuilder().cacheControl(EPHEMERAL).build()); + } + return null; + } +} diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParser.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParser.java index 582cf1be90..deabf936c4 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParser.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/main/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParser.java @@ -19,6 +19,7 @@ import com.anthropic.core.ObjectMappers; import com.anthropic.models.messages.Message; import com.anthropic.models.messages.RawMessageStreamEvent; +import com.anthropic.models.messages.Usage; import io.agentscope.core.message.ContentBlock; import io.agentscope.core.message.TextBlock; import io.agentscope.core.message.ThinkingBlock; @@ -87,14 +88,17 @@ public static ChatResponse parseMessage(Message message, Instant startTime) { } // Parse usage - long baseInput = message.usage().inputTokens(); - long cacheRead = message.usage().cacheReadInputTokens().orElse(0L); - long cacheCreate = message.usage().cacheCreationInputTokens().orElse(0L); + Usage anthropicUsage = message.usage(); + long cacheRead = anthropicUsage.cacheReadInputTokens().orElse(0L); + long cacheCreation = anthropicUsage.cacheCreationInputTokens().orElse(0L); ChatUsage usage = ChatUsage.builder() - .inputTokens((int) (baseInput + cacheRead + cacheCreate)) - .outputTokens((int) message.usage().outputTokens()) - .cachedTokens((int) cacheRead) + .inputTokens( + Math.toIntExact( + anthropicUsage.inputTokens() + cacheRead + cacheCreation)) + .outputTokens(Math.toIntExact(anthropicUsage.outputTokens())) + .cachedTokens(Math.toIntExact(cacheRead)) + .cacheCreationInputTokens(Math.toIntExact(cacheCreation)) .time(Duration.between(startTime, Instant.now()).toMillis() / 1000.0) .build(); @@ -117,13 +121,13 @@ public static Flux parseStreamEvents( Flux eventFlux, Instant startTime) { return Flux.defer( () -> { - StreamUsageState usageState = new StreamUsageState(); + StreamState state = new StreamState(); return eventFlux - .flatMap( + .concatMap( event -> { try { return Flux.just( - parseStreamEvent(event, startTime, usageState)); + parseStreamEvent(event, startTime, state)); } catch (Exception e) { log.warn( "Error parsing stream event: {}", @@ -142,26 +146,26 @@ public static Flux parseStreamEvents( /** * Parse single stream event. */ + private static ChatResponse parseStreamEvent(RawMessageStreamEvent event, Instant startTime) { + return parseStreamEvent(event, startTime, new StreamState()); + } + private static ChatResponse parseStreamEvent( - RawMessageStreamEvent event, Instant startTime, StreamUsageState usageState) { + RawMessageStreamEvent event, Instant startTime, StreamState state) { List contentBlocks = new ArrayList<>(); ChatUsage usage = null; - String messageId = null; // Message start - record prompt usage (input tokens and cache read/creation tokens) so // the final usage emitted on message_delta can include it if (event.isMessageStart()) { - var startMessage = event.asMessageStart().message(); - messageId = startMessage.id(); - - var startUsage = startMessage.usage(); - long cacheReadTokens = startUsage.cacheReadInputTokens().orElse(0L); - long cacheCreationTokens = startUsage.cacheCreationInputTokens().orElse(0L); - // Anthropic reports input_tokens excluding cached tokens; add them back so - // cachedTokens stays a subset of inputTokens (ChatUsage invariant). - usageState.inputTokens = - (int) (startUsage.inputTokens() + cacheReadTokens + cacheCreationTokens); - usageState.cachedTokens = (int) cacheReadTokens; + Message message = event.asMessageStart().message(); + state.messageId = message.id(); + Usage startUsage = message.usage(); + if (startUsage != null) { + state.inputTokens = startUsage.inputTokens(); + state.cachedTokens = startUsage.cacheReadInputTokens().orElse(0L); + state.cacheCreationInputTokens = startUsage.cacheCreationInputTokens().orElse(0L); + } } // Content block delta - text @@ -225,25 +229,45 @@ private static ChatResponse parseStreamEvent( // Message delta - final usage information; combine the cumulative output tokens with // the prompt usage captured on message_start if (event.isMessageDelta()) { - var deltaUsage = event.asMessageDelta().usage(); - long cacheReadTokens = - deltaUsage.cacheReadInputTokens().orElse((long) usageState.cachedTokens); - long inputTokens = - deltaUsage.inputTokens().isPresent() - ? deltaUsage.inputTokens().get() - + cacheReadTokens - + deltaUsage.cacheCreationInputTokens().orElse(0L) - : usageState.inputTokens; + var messageDelta = event.asMessageDelta(); + messageDelta.usage().inputTokens().ifPresent(value -> state.inputTokens = value); + messageDelta + .usage() + .cacheReadInputTokens() + .ifPresent(value -> state.cachedTokens = value); + messageDelta + .usage() + .cacheCreationInputTokens() + .ifPresent(value -> state.cacheCreationInputTokens = value); + state.outputTokens = messageDelta.usage().outputTokens(); usage = ChatUsage.builder() - .inputTokens((int) inputTokens) - .cachedTokens((int) cacheReadTokens) - .outputTokens((int) deltaUsage.outputTokens()) + .inputTokens( + Math.toIntExact( + state.inputTokens + + state.cachedTokens + + state.cacheCreationInputTokens)) + .outputTokens(Math.toIntExact(state.outputTokens)) + .cachedTokens(Math.toIntExact(state.cachedTokens)) + .cacheCreationInputTokens( + Math.toIntExact(state.cacheCreationInputTokens)) .time(Duration.between(startTime, Instant.now()).toMillis() / 1000.0) .build(); } - return ChatResponse.builder().id(messageId).content(contentBlocks).usage(usage).build(); + return ChatResponse.builder() + .id(state.messageId) + .content(contentBlocks) + .usage(usage) + .build(); + } + + private static final class StreamState { + private String messageId; + private long inputTokens; + private long outputTokens; + private long cachedTokens; + private long cacheCreationInputTokens; } /** diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/AnthropicChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/AnthropicChatModelTest.java index d212b1b43a..815e22b631 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/AnthropicChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/AnthropicChatModelTest.java @@ -461,9 +461,9 @@ void testProxyWithBaseUrl() { @Test @DisplayName("Should build streaming request with cacheControl enabled before transport error") void testStreamWithCacheControlEnabled() { - // Unreachable local endpoint: the request-building path (system message with - // cache_control, message formatting, applyCacheControl) executes fully and - // only the transport call fails. + // Unreachable local endpoint: the request-building path (system extraction, + // message formatting, and prompt-cache planning) executes fully and only the + // transport call fails. AnthropicChatModel model = AnthropicChatModel.builder() .apiKey(mockApiKey) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java new file mode 100644 index 0000000000..5f9aaacd41 --- /dev/null +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicPromptCacheTest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.extensions.model.anthropic.formatter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.anthropic.models.messages.MessageCreateParams; +import com.anthropic.models.messages.MessageParam; +import io.agentscope.core.message.MessageMetadataKeys; +import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; +import io.agentscope.core.message.TextBlock; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class AnthropicPromptCacheTest { + + private final AnthropicChatFormatter formatter = new AnthropicChatFormatter(); + + @Test + void automaticCachingUsesTopLevelProviderField() { + Msg user = message(MsgRole.USER, "hello", false); + List formatted = formatter.format(List.of(user)); + MessageCreateParams.Builder builder = baseBuilder(); + + AnthropicBaseFormatter.PromptCachePlan plan = + formatter.applyPromptCache(builder, List.of(user), formatted, true); + plan.messages().forEach(builder::addMessage); + MessageCreateParams params = builder.build(); + + @SuppressWarnings("unchecked") + Map cacheControl = + params._additionalBodyProperties().get("cache_control").convert(Map.class); + assertEquals(Map.of("type", "ephemeral"), cacheControl); + assertFalse(AnthropicPromptCacheSupport.hasCacheControl(plan.messages().get(0))); + } + + @Test + void explicitMessageMarkerWorksWhenAutomaticCachingIsDisabled() { + Msg user = message(MsgRole.USER, "stable context", true); + List formatted = formatter.format(List.of(user)); + MessageCreateParams.Builder builder = baseBuilder(); + + AnthropicBaseFormatter.PromptCachePlan plan = + formatter.applyPromptCache(builder, List.of(user), formatted, false); + plan.messages().forEach(builder::addMessage); + MessageCreateParams params = builder.build(); + + assertTrue(AnthropicPromptCacheSupport.hasCacheControl(plan.messages().get(0))); + assertFalse(params._additionalBodyProperties().containsKey("cache_control")); + } + + @Test + void explicitSystemMarkerUsesTypedSystemBlockWithoutDuplicatingMessage() { + Msg system = message(MsgRole.SYSTEM, "You are helpful", true); + Msg user = message(MsgRole.USER, "hello", false); + List formatted = formatter.format(List.of(user)); + MessageCreateParams.Builder builder = baseBuilder(); + + AnthropicBaseFormatter.PromptCachePlan plan = + formatter.applyPromptCache(builder, List.of(system, user), formatted, false); + plan.messages().forEach(builder::addMessage); + MessageCreateParams params = builder.build(); + + assertEquals(1, params.messages().size()); + assertTrue(params.system().orElseThrow().isTextBlockParams()); + assertTrue( + params.system() + .orElseThrow() + .asTextBlockParams() + .get(0) + .cacheControl() + .isPresent()); + } + + @Test + void multiAgentFormatterPreservesExplicitBoundary() { + AnthropicMultiAgentFormatter multiAgentFormatter = new AnthropicMultiAgentFormatter(); + List formatted = + multiAgentFormatter.format( + List.of( + message(MsgRole.USER, "first", false), + message(MsgRole.ASSISTANT, "stable", true), + message(MsgRole.USER, "last", false))); + + assertEquals(3, formatted.size()); + assertTrue(AnthropicPromptCacheSupport.hasCacheControl(formatted.get(1))); + } + + @Test + void rejectsMoreThanFourExplicitBreakpoints() { + List messages = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + messages.add(message(MsgRole.USER, "context-" + i, true)); + } + List formatted = formatter.format(messages); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyPromptCache(baseBuilder(), messages, formatted, false)); + } + + @Test + void automaticCachingRejectsFourExplicitBreakpointsBecauseItUsesOneSlot() { + List messages = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + messages.add(message(MsgRole.USER, "context-" + i, true)); + } + List formatted = formatter.format(messages); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyPromptCache(baseBuilder(), messages, formatted, true)); + } + + private static MessageCreateParams.Builder baseBuilder() { + return MessageCreateParams.builder().model("claude-test").maxTokens(32); + } + + private static Msg message(MsgRole role, String text, boolean cache) { + Msg.Builder builder = + Msg.builder() + .name(role.name()) + .role(role) + .content(TextBlock.builder().text(text).build()); + if (cache) { + builder.metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)); + } + return builder.build(); + } +} diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java index 9dde5adbf0..0dd8bbcd93 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-anthropic/src/test/java/io/agentscope/extensions/model/anthropic/formatter/AnthropicResponseParserTest.java @@ -66,17 +66,11 @@ private static com.anthropic.models.messages.ToolUseBlock mockToolUseBlock() { */ private ChatResponse invokeParseStreamEvent(RawMessageStreamEvent event, Instant startTime) throws Exception { - Class stateClass = - Class.forName(AnthropicResponseParser.class.getName() + "$StreamUsageState"); - var constructor = stateClass.getDeclaredConstructor(); - constructor.setAccessible(true); - Object state = constructor.newInstance(); - Method method = AnthropicResponseParser.class.getDeclaredMethod( - "parseStreamEvent", RawMessageStreamEvent.class, Instant.class, stateClass); + "parseStreamEvent", RawMessageStreamEvent.class, Instant.class); method.setAccessible(true); - return (ChatResponse) method.invoke(null, event, startTime, state); + return (ChatResponse) method.invoke(null, event, startTime); } @Test @@ -92,8 +86,8 @@ void testParseMessageWithTextBlock() { when(message.usage()).thenReturn(usage); when(usage.inputTokens()).thenReturn(100L); when(usage.outputTokens()).thenReturn(50L); - when(usage.cacheReadInputTokens()).thenReturn(Optional.empty()); - when(usage.cacheCreationInputTokens()).thenReturn(Optional.empty()); + when(usage.cacheReadInputTokens()).thenReturn(Optional.of(30L)); + when(usage.cacheCreationInputTokens()).thenReturn(Optional.of(20L)); when(contentBlock.text()).thenReturn(Optional.of(textBlock)); when(contentBlock.toolUse()).thenReturn(Optional.empty()); @@ -111,8 +105,10 @@ void testParseMessageWithTextBlock() { ChatUsage responseUsage = response.getUsage(); assertNotNull(responseUsage); - assertEquals(100, responseUsage.getInputTokens()); + assertEquals(150, responseUsage.getInputTokens()); assertEquals(50, responseUsage.getOutputTokens()); + assertEquals(30, responseUsage.getCachedTokens()); + assertEquals(20, responseUsage.getCacheCreationInputTokens()); } @Test @@ -368,6 +364,47 @@ void testParseStreamEventMessageStart() throws Exception { assertTrue(response.getContent().isEmpty()); // MessageStart has no content } + @Test + void testStreamUsageRetainsCacheTokensAndMessageId() { + RawMessageStreamEvent startEvent = mock(RawMessageStreamEvent.class); + RawMessageStartEvent messageStart = mock(RawMessageStartEvent.class); + Message message = mock(Message.class); + Usage startUsage = mock(Usage.class); + when(startEvent.isMessageStart()).thenReturn(true); + when(startEvent.asMessageStart()).thenReturn(messageStart); + when(messageStart.message()).thenReturn(message); + when(message.id()).thenReturn("msg-cache"); + when(message.usage()).thenReturn(startUsage); + when(startUsage.inputTokens()).thenReturn(10L); + when(startUsage.cacheReadInputTokens()).thenReturn(Optional.of(70L)); + when(startUsage.cacheCreationInputTokens()).thenReturn(Optional.of(20L)); + + RawMessageStreamEvent deltaEvent = mock(RawMessageStreamEvent.class); + RawMessageDeltaEvent messageDelta = mock(RawMessageDeltaEvent.class); + MessageDeltaUsage deltaUsage = mock(MessageDeltaUsage.class); + when(deltaEvent.isMessageDelta()).thenReturn(true); + when(deltaEvent.asMessageDelta()).thenReturn(messageDelta); + when(messageDelta.usage()).thenReturn(deltaUsage); + when(deltaUsage.inputTokens()).thenReturn(Optional.empty()); + when(deltaUsage.cacheReadInputTokens()).thenReturn(Optional.empty()); + when(deltaUsage.cacheCreationInputTokens()).thenReturn(Optional.empty()); + when(deltaUsage.outputTokens()).thenReturn(5L); + + StepVerifier.create( + AnthropicResponseParser.parseStreamEvents( + Flux.just(startEvent, deltaEvent), Instant.now())) + .assertNext( + response -> { + assertEquals("msg-cache", response.getId()); + assertTrue(response.getContent().isEmpty()); + assertEquals(100, response.getUsage().getInputTokens()); + assertEquals(5, response.getUsage().getOutputTokens()); + assertEquals(70, response.getUsage().getCachedTokens()); + assertEquals(20, response.getUsage().getCacheCreationInputTokens()); + }) + .verifyComplete(); + } + @Test void testParseStreamEventThinkingDelta() throws Exception { RawContentBlockDeltaEvent deltaEvent = diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/DashScopeChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/DashScopeChatModel.java index cb335dbf47..3a4b823b1a 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/DashScopeChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/DashScopeChatModel.java @@ -280,13 +280,12 @@ private Flux streamWithHttpClient( // Apply thinking mode if enabled applyThinkingMode(request, effectiveOptions); - // Apply cache control if enabled (adds cache_control to system msgs + last msg) - if (Boolean.TRUE.equals(effectiveOptions.getCacheControl())) { - if (formatter instanceof DashScopeChatFormatter chatFmt) { - chatFmt.applyCacheControl(request.getInput().getMessages()); - } else if (formatter instanceof DashScopeMultiAgentFormatter multiFmt) { - multiFmt.applyCacheControl(request.getInput().getMessages()); - } + // Always normalize explicit markers; automatic breakpoints remain opt-in. + boolean automaticCacheControl = Boolean.TRUE.equals(effectiveOptions.getCacheControl()); + if (formatter instanceof DashScopeChatFormatter chatFmt) { + chatFmt.applyCacheControl(request.getInput().getMessages(), automaticCacheControl); + } else if (formatter instanceof DashScopeMultiAgentFormatter multiFmt) { + multiFmt.applyCacheControl(request.getInput().getMessages(), automaticCacheControl); } // Set endpoint type for endpoint selection diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeContentPart.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeContentPart.java index ba7d3be21f..5abf031054 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeContentPart.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeContentPart.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; +import java.util.Map; /** * DashScope content part DTO for multimodal messages. @@ -43,6 +44,10 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class DashScopeContentPart { + /** Content type. Explicitly cached text blocks use {@code text}. */ + @JsonProperty("type") + private String type; + /** Text content. */ @JsonProperty("text") private String text; @@ -79,8 +84,20 @@ public class DashScopeContentPart { @JsonProperty("total_pixels") private Integer totalPixels; + /** Prompt cache control marker. */ + @JsonProperty("cache_control") + private Map cacheControl; + public DashScopeContentPart() {} + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + public String getText() { return text; } @@ -153,6 +170,14 @@ public void setTotalPixels(Integer totalPixels) { this.totalPixels = totalPixels; } + public Map getCacheControl() { + return cacheControl; + } + + public void setCacheControl(Map cacheControl) { + this.cacheControl = cacheControl; + } + /** * Get video as URL string. * @@ -245,6 +270,11 @@ public static Builder builder() { public static class Builder { private final DashScopeContentPart part = new DashScopeContentPart(); + public Builder type(String type) { + part.setType(type); + return this; + } + public Builder text(String text) { part.setText(text); return this; @@ -290,6 +320,11 @@ public Builder totalPixels(Integer totalPixels) { return this; } + public Builder cacheControl(Map cacheControl) { + part.setCacheControl(cacheControl); + return this; + } + public DashScopeContentPart build() { return part; } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java new file mode 100644 index 0000000000..1713d53df1 --- /dev/null +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeTokenDetails.java @@ -0,0 +1,93 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.extensions.model.dashscope.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Fine-grained DashScope input token usage details. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DashScopeTokenDetails { + + @JsonProperty("cached_tokens") + private Integer cachedTokens; + + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + @JsonProperty("cache_creation") + private CacheCreationDetails cacheCreation; + + public Integer getCachedTokens() { + return cachedTokens; + } + + public void setCachedTokens(Integer cachedTokens) { + this.cachedTokens = cachedTokens; + } + + public Integer getCacheCreationInputTokens() { + if (cacheCreationInputTokens != null) { + return cacheCreationInputTokens; + } + return cacheCreation != null ? cacheCreation.getInputTokens() : null; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } + + public CacheCreationDetails getCacheCreation() { + return cacheCreation; + } + + public void setCacheCreation(CacheCreationDetails cacheCreation) { + this.cacheCreation = cacheCreation; + } + + /** Current nested DashScope explicit-cache creation details. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CacheCreationDetails { + + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + @JsonProperty("ephemeral_5m_input_tokens") + private Integer ephemeral5mInputTokens; + + public Integer getCacheCreationInputTokens() { + return cacheCreationInputTokens; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } + + public Integer getEphemeral5mInputTokens() { + return ephemeral5mInputTokens; + } + + public void setEphemeral5mInputTokens(Integer ephemeral5mInputTokens) { + this.ephemeral5mInputTokens = ephemeral5mInputTokens; + } + + Integer getInputTokens() { + return cacheCreationInputTokens != null + ? cacheCreationInputTokens + : ephemeral5mInputTokens; + } + } +} diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeUsage.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeUsage.java index d80eec1877..0c4fb9c07b 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeUsage.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/dto/DashScopeUsage.java @@ -59,6 +59,22 @@ public class DashScopeUsage { @JsonProperty("audio_tokens") private Integer audioTokens; + /** Legacy regional field for cache hits. */ + @JsonProperty("cached_tokens") + private Integer cachedTokens; + + /** Legacy/top-level field for explicit cache creation. */ + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + /** Current DashScope prompt token details. */ + @JsonProperty("prompt_tokens_details") + private DashScopeTokenDetails promptTokensDetails; + + /** Alternate token detail shape returned by some endpoints. */ + @JsonProperty("input_tokens_details") + private DashScopeTokenDetails inputTokensDetails; + public DashScopeUsage() {} public Integer getInputTokens() { @@ -108,4 +124,52 @@ public Integer getAudioTokens() { public void setAudioTokens(Integer audioTokens) { this.audioTokens = audioTokens; } + + /** Returns cache hits across current and legacy DashScope response shapes. */ + public Integer getCachedTokens() { + if (promptTokensDetails != null && promptTokensDetails.getCachedTokens() != null) { + return promptTokensDetails.getCachedTokens(); + } + if (inputTokensDetails != null && inputTokensDetails.getCachedTokens() != null) { + return inputTokensDetails.getCachedTokens(); + } + return cachedTokens; + } + + public void setCachedTokens(Integer cachedTokens) { + this.cachedTokens = cachedTokens; + } + + /** Returns explicit cache creation tokens across current and legacy response shapes. */ + public Integer getCacheCreationInputTokens() { + if (promptTokensDetails != null + && promptTokensDetails.getCacheCreationInputTokens() != null) { + return promptTokensDetails.getCacheCreationInputTokens(); + } + if (inputTokensDetails != null + && inputTokensDetails.getCacheCreationInputTokens() != null) { + return inputTokensDetails.getCacheCreationInputTokens(); + } + return cacheCreationInputTokens; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } + + public DashScopeTokenDetails getPromptTokensDetails() { + return promptTokensDetails; + } + + public void setPromptTokensDetails(DashScopeTokenDetails promptTokensDetails) { + this.promptTokensDetails = promptTokensDetails; + } + + public DashScopeTokenDetails getInputTokensDetails() { + return inputTokensDetails; + } + + public void setInputTokensDetails(DashScopeTokenDetails inputTokensDetails) { + this.inputTokensDetails = inputTokensDetails; + } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeChatFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeChatFormatter.java index 8f216885d8..74fbdb45be 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeChatFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeChatFormatter.java @@ -21,6 +21,7 @@ import io.agentscope.core.model.GenerateOptions; import io.agentscope.core.model.ToolChoice; import io.agentscope.core.model.ToolSchema; +import io.agentscope.extensions.model.dashscope.dto.DashScopeContentPart; import io.agentscope.extensions.model.dashscope.dto.DashScopeInput; import io.agentscope.extensions.model.dashscope.dto.DashScopeMessage; import io.agentscope.extensions.model.dashscope.dto.DashScopeParameters; @@ -181,27 +182,140 @@ public DashScopeRequest buildRequest( } /** - * Apply cache control to DashScope messages. - * - *

Adds cache_control: {"type": "ephemeral"} to all system messages and the last - * message in the list. Messages that already carry a cache_control value (including the empty - * "no cache" sentinel) are left untouched. + * Apply automatic prompt cache control to DashScope messages. * * @param messages the list of formatted DashScope messages */ public void applyCacheControl(List messages) { + applyCacheControl(messages, true); + } + + /** + * Normalize explicit prompt cache markers and optionally apply the automatic strategy. + * + *

DashScope requires {@code cache_control} inside a content block. Legacy message-level + * markers are migrated to the last content block and removed from the message before + * serialization. + * + * @param messages formatted DashScope messages + * @param automatic whether to add automatic breakpoints + */ + public void applyCacheControl(List messages, boolean automatic) { if (messages == null || messages.isEmpty()) { return; } - for (DashScopeMessage msg : messages) { - if ("system".equals(msg.getRole()) && shouldAutoCache(msg)) { - msg.setCacheControl(EPHEMERAL_CACHE_CONTROL); - } + List selected = + selectPromptCacheBreakpoints( + messages, + automatic, + DashScopeChatFormatter::hasExplicitCacheControl, + message -> "system".equals(message.getRole()), + DashScopeChatFormatter::isCacheable); + + for (DashScopeMessage message : selected) { + applyCacheControlToContentBlock(message); + } + for (DashScopeMessage message : messages) { + message.setCacheControl(null); + } + + int markerCount = countCacheControlMarkers(messages); + if (markerCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "DashScope supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " cache_control markers, but got " + + markerCount); + } + } + + static void applyCacheControlToContentBlock(DashScopeMessage message) { + if (isExplicitNoCache(message)) { + message.setCacheControl(null); + return; + } + if (hasContentBlockCacheControl(message)) { + message.setCacheControl(null); + return; + } + + List parts = ensureContentArray(message); + if (parts.isEmpty()) { + throw new IllegalArgumentException( + "Cannot apply cache_control to a message without cacheable content"); } - DashScopeMessage lastMsg = messages.get(messages.size() - 1); - if (shouldAutoCache(lastMsg)) { - lastMsg.setCacheControl(EPHEMERAL_CACHE_CONTROL); + + DashScopeContentPart lastPart = parts.get(parts.size() - 1); + if (lastPart.getType() == null && lastPart.getText() != null) { + lastPart.setType("text"); + } + Map cacheControl = + message.getCacheControl() != null + ? message.getCacheControl() + : EPHEMERAL_CACHE_CONTROL; + lastPart.setCacheControl(cacheControl); + message.setCacheControl(null); + } + + @SuppressWarnings("unchecked") + static List ensureContentArray(DashScopeMessage message) { + Object content = message.getContent(); + if (content instanceof List) { + return (List) content; + } + if (content instanceof String text) { + List parts = + new ArrayList<>( + List.of( + DashScopeContentPart.builder() + .type("text") + .text(text) + .build())); + message.setContent(parts); + return parts; + } + return List.of(); + } + + static boolean isCacheable(DashScopeMessage message) { + if (isExplicitNoCache(message)) { + return false; + } + if (!("system".equals(message.getRole()) + || "user".equals(message.getRole()) + || "assistant".equals(message.getRole()) + || "tool".equals(message.getRole()))) { + return false; + } + Object content = message.getContent(); + return (content instanceof String text && !text.isEmpty()) + || (content instanceof List parts && !parts.isEmpty()); + } + + static boolean hasExplicitCacheControl(DashScopeMessage message) { + return (message.getCacheControl() != null && !message.getCacheControl().isEmpty()) + || hasContentBlockCacheControl(message); + } + + static boolean isExplicitNoCache(DashScopeMessage message) { + return message.getCacheControl() != null && message.getCacheControl().isEmpty(); + } + + static boolean hasContentBlockCacheControl(DashScopeMessage message) { + List parts = message.getContentAsList(); + return parts != null && parts.stream().anyMatch(part -> part.getCacheControl() != null); + } + + static int countCacheControlMarkers(List messages) { + int count = 0; + for (DashScopeMessage message : messages) { + List parts = message.getContentAsList(); + if (parts != null) { + count += + (int) parts.stream().filter(part -> part.getCacheControl() != null).count(); + } } + return count; } /** diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMessageConverter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMessageConverter.java index a6a069d4b8..028f0fd6c6 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMessageConverter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMessageConverter.java @@ -279,9 +279,7 @@ private void applyCacheControlFromMetadata(Msg msg, DashScopeMessage result) { } Object cacheFlag = msg.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL); if (Boolean.TRUE.equals(cacheFlag)) { - if (result.getCacheControl() == null || result.getCacheControl().isEmpty()) { - result.setCacheControl(DashScopeChatFormatter.getEphemeralCacheControl()); - } + DashScopeChatFormatter.applyCacheControlToContentBlock(result); } else if (Boolean.FALSE.equals(cacheFlag)) { result.setCacheControl(DashScopeChatFormatter.getNoCacheControl()); } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMultiAgentFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMultiAgentFormatter.java index d038362986..9eb6a1b9d1 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMultiAgentFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeMultiAgentFormatter.java @@ -24,7 +24,6 @@ import io.agentscope.core.model.GenerateOptions; import io.agentscope.core.model.ToolChoice; import io.agentscope.core.model.ToolSchema; -import io.agentscope.extensions.model.dashscope.dto.DashScopeContentPart; import io.agentscope.extensions.model.dashscope.dto.DashScopeInput; import io.agentscope.extensions.model.dashscope.dto.DashScopeMessage; import io.agentscope.extensions.model.dashscope.dto.DashScopeParameters; @@ -33,7 +32,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.Map; /** * DashScope formatter for multi-agent conversations. @@ -84,11 +82,7 @@ protected List doFormat(List msgs) { // Process system message first (if any) - output separately if (!msgs.isEmpty() && msgs.get(0).getRole() == MsgRole.SYSTEM) { - result.add( - DashScopeMessage.builder() - .role("system") - .content(extractTextContent(msgs.get(0))) - .build()); + result.add(messageConverter.convertToMessage(msgs.get(0), false)); startIndex = 1; } @@ -111,6 +105,8 @@ protected List doFormat(List msgs) { } else if (group.type == GroupType.TOOL_SEQUENCE) { // Format tool sequence directly result.addAll(formatToolSeq(group.messages)); + } else if (group.type == GroupType.BYPASS) { + result.add(messageConverter.convertToMessage(group.messages.get(0), false)); } } @@ -171,14 +167,7 @@ public List formatMultiModal(List msgs) { // Process system message first (if any) if (!msgs.isEmpty() && msgs.get(0).getRole() == MsgRole.SYSTEM) { - result.add( - DashScopeMessage.builder() - .role("system") - .content( - List.of( - DashScopeContentPart.text( - extractTextContent(msgs.get(0))))) - .build()); + result.add(messageConverter.convertToMessage(msgs.get(0), true)); startIndex = 1; } @@ -200,6 +189,8 @@ public List formatMultiModal(List msgs) { } else if (group.type == GroupType.TOOL_SEQUENCE) { // Format tool sequence directly result.addAll(formatMultiModalToolSeq(group.messages)); + } else if (group.type == GroupType.BYPASS) { + result.add(messageConverter.convertToMessage(group.messages.get(0), true)); } } @@ -244,6 +235,16 @@ private List groupMessagesSequentially(List msgs) { List currentGroup = new ArrayList<>(); for (Msg msg : msgs) { + if (shouldBypassHistory(msg)) { + if (!currentGroup.isEmpty()) { + result.add(new MessageGroup(currentType, new ArrayList<>(currentGroup))); + currentGroup.clear(); + } + result.add(new MessageGroup(GroupType.BYPASS, List.of(msg))); + currentType = null; + continue; + } + boolean isToolRelated = msg.getRole() == MsgRole.TOOL || msg.hasContentBlocks(ToolUseBlock.class) @@ -347,7 +348,8 @@ private List formatMultiModalToolSeq(List msgs) { */ private enum GroupType { AGENT_MESSAGE, - TOOL_SEQUENCE + TOOL_SEQUENCE, + BYPASS } /** @@ -364,27 +366,40 @@ private static class MessageGroup { } /** - * Apply cache control to DashScope messages. - * - *

Adds cache_control: {"type": "ephemeral"} to all system messages and the last - * message in the list. Messages that are explicitly excluded from caching or that already carry - * a cache_control value are left untouched. + * Apply automatic prompt cache control to DashScope messages. * * @param messages the list of formatted DashScope messages */ public void applyCacheControl(List messages) { + applyCacheControl(messages, true); + } + + /** Normalize explicit cache markers and optionally apply automatic cache breakpoints. */ + public void applyCacheControl(List messages, boolean automatic) { if (messages == null || messages.isEmpty()) { return; } - Map ephemeral = DashScopeChatFormatter.getEphemeralCacheControl(); - for (DashScopeMessage msg : messages) { - if ("system".equals(msg.getRole()) && DashScopeChatFormatter.shouldAutoCache(msg)) { - msg.setCacheControl(ephemeral); - } + List selected = + selectPromptCacheBreakpoints( + messages, + automatic, + DashScopeChatFormatter::hasExplicitCacheControl, + message -> "system".equals(message.getRole()), + DashScopeChatFormatter::isCacheable); + for (DashScopeMessage message : selected) { + DashScopeChatFormatter.applyCacheControlToContentBlock(message); } - DashScopeMessage lastMsg = messages.get(messages.size() - 1); - if (DashScopeChatFormatter.shouldAutoCache(lastMsg)) { - lastMsg.setCacheControl(ephemeral); + for (DashScopeMessage message : messages) { + message.setCacheControl(null); + } + + int markerCount = DashScopeChatFormatter.countCacheControlMarkers(messages); + if (markerCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "DashScope supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " cache_control markers, but got " + + markerCount); } } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParser.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParser.java index b882d34ea7..f4f8cd7bcb 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParser.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/main/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParser.java @@ -101,6 +101,11 @@ public ChatResponse parseResponse(DashScopeResponse response, Instant startTime) ChatUsage.builder() .inputTokens(u.getInputTokens() != null ? u.getInputTokens() : 0) .outputTokens(u.getOutputTokens() != null ? u.getOutputTokens() : 0) + .cachedTokens(u.getCachedTokens() != null ? u.getCachedTokens() : 0) + .cacheCreationInputTokens( + u.getCacheCreationInputTokens() != null + ? u.getCacheCreationInputTokens() + : 0) .time( Duration.between(startTime, Instant.now()).toMillis() / 1000.0) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/DashScopeChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/DashScopeChatModelTest.java index 1dbac5c29b..5a85403ab0 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/DashScopeChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/DashScopeChatModelTest.java @@ -24,6 +24,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.fasterxml.jackson.databind.JsonNode; +import io.agentscope.core.message.MessageMetadataKeys; import io.agentscope.core.message.Msg; import io.agentscope.core.message.MsgRole; import io.agentscope.core.message.TextBlock; @@ -34,6 +36,7 @@ import io.agentscope.core.model.test.ModelTestUtils; import io.agentscope.core.model.transport.OkHttpTransport; import io.agentscope.core.model.transport.ProxyConfig; +import io.agentscope.core.util.JsonUtils; import io.agentscope.extensions.model.dashscope.dto.DashScopeParameters; import io.agentscope.extensions.model.dashscope.dto.DashScopeRequest; import io.agentscope.extensions.model.dashscope.dto.DashScopeSearchOptions; @@ -975,12 +978,53 @@ void testCacheControlApplied() throws Exception { RecordedRequest recorded = mockServer.takeRequest(); String body = recorded.getBody().readUtf8(); - assertTrue( - body.contains("\"cache_control\""), - "Request body should contain cache_control: " + body); - assertTrue( - body.contains("\"ephemeral\""), - "Request body should contain ephemeral cache type: " + body); + JsonNode tree = JsonUtils.getJsonCodec().fromJson(body, JsonNode.class); + JsonNode system = tree.at("/input/messages/0"); + JsonNode user = tree.at("/input/messages/1"); + assertFalse(system.has("cache_control"), body); + assertFalse(user.has("cache_control"), body); + assertTrue(system.get("content").isArray(), body); + assertTrue(user.get("content").isArray(), body); + assertEquals("text", system.at("/content/0/type").asText()); + assertEquals("ephemeral", system.at("/content/0/cache_control/type").asText()); + assertEquals("ephemeral", user.at("/content/0/cache_control/type").asText()); + + mockServer.shutdown(); + } + + @Test + @DisplayName("Should preserve explicit cache metadata when automatic cache control is disabled") + void testExplicitCacheControlAppliedWithoutAutomaticOption() throws Exception { + MockWebServer mockServer = new MockWebServer(); + mockServer.start(); + mockServer.enqueue( + new MockResponse() + .setResponseCode(200) + .setBody("{\"request_id\":\"test\",\"output\":{\"choices\":[]}}") + .setHeader("Content-Type", "application/json")); + + DashScopeChatModel chatModel = + DashScopeChatModel.builder().apiKey(mockApiKey).modelName("qwen-plus").stream(false) + .baseUrl(mockServer.url("/").toString().replaceAll("/$", "")) + .httpTransport(OkHttpTransport.builder().build()) + .build(); + Msg explicit = + Msg.builder() + .role(MsgRole.USER) + .textContent("stable context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + + chatModel + .doStream(List.of(explicit), List.of(), GenerateOptions.builder().build()) + .blockLast(); + + JsonNode tree = + JsonUtils.getJsonCodec() + .fromJson(mockServer.takeRequest().getBody().readUtf8(), JsonNode.class); + JsonNode message = tree.at("/input/messages/0"); + assertFalse(message.has("cache_control")); + assertEquals("ephemeral", message.at("/content/0/cache_control/type").asText()); mockServer.shutdown(); } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java index 50360e7183..f10ba785aa 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/dto/DashScopeDtoSerializationTest.java @@ -238,6 +238,58 @@ void testDashScopeResponseDeserialization() throws Exception { assertEquals(8, usage.getOutputTokens()); } + @Test + void testDashScopeCacheUsageDeserialization() { + String json = + """ + { + "request_id": "req-cache", + "usage": { + "input_tokens": 2048, + "output_tokens": 128, + "prompt_tokens_details": { + "cached_tokens": 1536, + "cache_creation_input_tokens": 256 + } + } + } + """; + + DashScopeResponse response = jsonCodec.fromJson(json, DashScopeResponse.class); + + assertNotNull(response.getUsage()); + assertEquals(1536, response.getUsage().getCachedTokens()); + assertEquals(256, response.getUsage().getCacheCreationInputTokens()); + } + + @Test + void testDashScopeNestedCacheCreationUsageDeserialization() { + String json = + """ + { + "request_id": "req-cache-nested", + "usage": { + "input_tokens": 2048, + "output_tokens": 128, + "prompt_tokens_details": { + "cached_tokens": 1536, + "cache_creation": { + "cache_creation_input_tokens": 384, + "ephemeral_5m_input_tokens": 999, + "cache_type": "ephemeral" + } + } + } + } + """; + + DashScopeResponse response = jsonCodec.fromJson(json, DashScopeResponse.class); + + assertNotNull(response.getUsage()); + assertEquals(1536, response.getUsage().getCachedTokens()); + assertEquals(384, response.getUsage().getCacheCreationInputTokens()); + } + @Test void testDashScopeResponseWithToolCalls() throws Exception { String json = diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeCacheControlTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeCacheControlTest.java index 006cf4f5d9..5a32d34245 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeCacheControlTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeCacheControlTest.java @@ -17,15 +17,19 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import com.fasterxml.jackson.databind.JsonNode; import io.agentscope.core.message.MessageMetadataKeys; import io.agentscope.core.message.Msg; import io.agentscope.core.message.MsgRole; import io.agentscope.core.util.JsonUtils; +import io.agentscope.extensions.model.dashscope.dto.DashScopeContentPart; import io.agentscope.extensions.model.dashscope.dto.DashScopeMessage; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.jupiter.api.BeforeEach; @@ -33,9 +37,7 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -/** - * Tests for cache_control support in DashScope formatter. - */ +/** Tests DashScope content-block prompt cache markers. */ class DashScopeCacheControlTest { private static final Map EPHEMERAL = Map.of("type", "ephemeral"); @@ -49,257 +51,260 @@ void setUp() { } @Nested - @DisplayName("applyCacheControl - automatic strategy") - class ApplyCacheControlTest { + @DisplayName("automatic cache strategy") + class AutomaticCacheStrategyTest { @Test - @DisplayName("should add cache_control to system and last message") - void systemAndLastMessage() { - List messages = new ArrayList<>(); - messages.add( - DashScopeMessage.builder().role("system").content("You are helpful.").build()); - messages.add(DashScopeMessage.builder().role("user").content("Hello").build()); - messages.add(DashScopeMessage.builder().role("assistant").content("Hi").build()); - messages.add(DashScopeMessage.builder().role("user").content("Question").build()); - - formatter.applyCacheControl(messages); - - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - assertNull(messages.get(1).getCacheControl()); - assertNull(messages.get(2).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(3).getCacheControl()); + void marksFirstSystemAndLastCacheableNonSystemMessage() { + List messages = + new ArrayList<>( + List.of( + message("system", "stable system"), + message("system", "dynamic system"), + message("user", "question"), + message("assistant", "answer"))); + + formatter.applyCacheControl(messages, true); + + assertEquals(EPHEMERAL, lastPart(messages.get(0)).getCacheControl()); + assertNoCacheControl(messages.get(1)); + assertNoCacheControl(messages.get(2)); + assertEquals(EPHEMERAL, lastPart(messages.get(3)).getCacheControl()); + messages.forEach(message -> assertNull(message.getCacheControl())); } @Test - @DisplayName("should handle no system message - only last message") - void noSystemMessage() { - List messages = new ArrayList<>(); - messages.add(DashScopeMessage.builder().role("user").content("Hello").build()); - messages.add(DashScopeMessage.builder().role("assistant").content("Hi").build()); + void automaticFalseKeepsOnlyExplicitMarkers() { + DashScopeMessage system = message("system", "system"); + DashScopeMessage explicit = + DashScopeMessage.builder() + .role("user") + .content("explicit") + .cacheControl(Map.of("type", "custom")) + .build(); + DashScopeMessage last = message("assistant", "last"); - formatter.applyCacheControl(messages); + formatter.applyCacheControl(new ArrayList<>(List.of(system, explicit, last)), false); - assertNull(messages.get(0).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(1).getCacheControl()); + assertNoCacheControl(system); + assertEquals(Map.of("type", "custom"), lastPart(explicit).getCacheControl()); + assertNull(explicit.getCacheControl()); + assertNoCacheControl(last); } @Test - @DisplayName("should handle empty list without error") - void emptyList() { + void rejectsMoreThanFourExplicitMarkers() { List messages = new ArrayList<>(); - formatter.applyCacheControl(messages); - // No exception thrown + for (int i = 0; i < 5; i++) { + DashScopeContentPart part = + DashScopeContentPart.builder() + .type("text") + .text("part-" + i) + .cacheControl(EPHEMERAL) + .build(); + messages.add( + DashScopeMessage.builder() + .role("user") + .content(new ArrayList<>(List.of(part))) + .build()); + } + + IllegalArgumentException error = + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyCacheControl(messages, false)); + + assertTrue(error.getMessage().contains("at most 4")); } @Test - @DisplayName("should handle null list without error") - void nullList() { - formatter.applyCacheControl(null); - // No exception thrown + void handlesNullAndEmptyLists() { + formatter.applyCacheControl(null, true); + formatter.applyCacheControl(List.of(), true); } + } - @Test - @DisplayName("should handle single system message (both system and last)") - void singleSystemMessage() { - List messages = new ArrayList<>(); - messages.add( - DashScopeMessage.builder().role("system").content("You are helpful.").build()); - - formatter.applyCacheControl(messages); - - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - } + @Nested + @DisplayName("wire format") + class WireFormatTest { @Test - @DisplayName("should not overwrite manually marked cache_control") - void manuallyMarkedNotOverridden() { - Map customCacheControl = Map.of("type", "custom"); - - List messages = new ArrayList<>(); - messages.add( - DashScopeMessage.builder() - .role("system") - .content("System") - .cacheControl(customCacheControl) - .build()); - messages.add(DashScopeMessage.builder().role("user").content("User").build()); - - formatter.applyCacheControl(messages); - - // System message keeps its custom cache_control - assertEquals(customCacheControl, messages.get(0).getCacheControl()); - // Last message gets ephemeral - assertEquals(EPHEMERAL, messages.get(1).getCacheControl()); + void serializesMarkerInsideTextContentBlockOnly() { + List messages = + new ArrayList<>(List.of(message("system", "You are helpful"))); + + formatter.applyCacheControl(messages, true); + + JsonNode tree = + JsonUtils.getJsonCodec() + .fromJson( + JsonUtils.getJsonCodec().toJson(messages.get(0)), + JsonNode.class); + assertFalse(tree.has("cache_control")); + assertTrue(tree.get("content").isArray()); + assertEquals("text", tree.at("/content/0/type").asText()); + assertEquals("You are helpful", tree.at("/content/0/text").asText()); + assertEquals("ephemeral", tree.at("/content/0/cache_control/type").asText()); } @Test - @DisplayName("should not overwrite last message with existing cache_control") - void lastMessageManuallyMarkedNotOverridden() { - Map customCacheControl = Map.of("type", "custom"); - - List messages = new ArrayList<>(); - messages.add(DashScopeMessage.builder().role("system").content("System").build()); - messages.add( + void marksLastExistingMultimodalContentPart() { + DashScopeContentPart text = DashScopeContentPart.text("describe"); + DashScopeContentPart image = DashScopeContentPart.image("https://example.com/a.png"); + DashScopeMessage message = DashScopeMessage.builder() .role("user") - .content("User") - .cacheControl(customCacheControl) - .build()); - - formatter.applyCacheControl(messages); - - // System message gets ephemeral - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - // Last message keeps its custom cache_control - assertEquals(customCacheControl, messages.get(1).getCacheControl()); - } - - @Test - @DisplayName("should handle multiple system messages") - void multipleSystemMessages() { - List messages = new ArrayList<>(); - messages.add(DashScopeMessage.builder().role("system").content("System 1").build()); - messages.add(DashScopeMessage.builder().role("system").content("System 2").build()); - messages.add(DashScopeMessage.builder().role("user").content("User").build()); + .content(new ArrayList<>(List.of(text, image))) + .build(); - formatter.applyCacheControl(messages); + formatter.applyCacheControl(new ArrayList<>(List.of(message)), true); - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(1).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(2).getCacheControl()); + assertNull(text.getCacheControl()); + assertEquals(EPHEMERAL, image.getCacheControl()); + assertNull(message.getCacheControl()); } } @Nested - @DisplayName("metadata-based cache_control marking") - class MetadataMarkingTest { + @DisplayName("metadata and multi-agent formatting") + class MetadataAndMultiAgentTest { @Test - @DisplayName("should set cache_control from Msg metadata") - void metadataMarking() { - Map metadata = new HashMap<>(); - metadata.put(MessageMetadataKeys.CACHE_CONTROL, true); + void metadataMarkerIsPlacedOnContentBlockWithoutAutomaticOption() { Msg msg = Msg.builder() .role(MsgRole.USER) - .textContent("Important context") - .metadata(metadata) + .textContent("cache this") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) .build(); - List result = formatter.format(List.of(msg)); + DashScopeMessage formatted = formatter.format(List.of(msg)).get(0); - assertEquals(1, result.size()); - assertEquals(EPHEMERAL, result.get(0).getCacheControl()); + assertNull(formatted.getCacheControl()); + assertEquals(EPHEMERAL, lastPart(formatted).getCacheControl()); } @Test - @DisplayName("should not set cache_control when metadata flag is absent") - void noMetadata() { - Msg msg = Msg.builder().role(MsgRole.USER).textContent("Hello").build(); - - List result = formatter.format(List.of(msg)); - - assertEquals(1, result.size()); - assertNull(result.get(0).getCacheControl()); - } - - @Test - @DisplayName("should mark explicit no-cache when metadata flag is false") - void metadataFalse() { - Map metadata = new HashMap<>(); - metadata.put(MessageMetadataKeys.CACHE_CONTROL, false); + void explicitNoCacheIsNeitherSerializedNorAutomaticallyReenabled() { Msg msg = Msg.builder() .role(MsgRole.USER) - .textContent("Hello") - .metadata(metadata) + .textContent("do not cache") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, false)) .build(); - List result = formatter.format(List.of(msg)); + List formatted = formatter.format(List.of(msg)); + assertEquals(NO_CACHE, formatted.get(0).getCacheControl()); + + formatter.applyCacheControl(formatted, true); - assertEquals(1, result.size()); - assertEquals(NO_CACHE, result.get(0).getCacheControl()); + assertNoCacheControl(formatted.get(0)); + JsonNode tree = + JsonUtils.getJsonCodec() + .fromJson( + JsonUtils.getJsonCodec().toJson(formatted.get(0)), + JsonNode.class); + assertFalse(tree.has("cache_control")); + assertFalse(tree.toString().contains("cache_control")); } @Test - @DisplayName("should not auto-cache a system message explicitly marked false") - void systemMessageExplicitNoCache() { - Map metadata = new HashMap<>(); - metadata.put(MessageMetadataKeys.CACHE_CONTROL, false); - Msg systemMsg = + void automaticStrategySkipsExplicitNoCacheSystemMessage() { + Msg system = Msg.builder() .role(MsgRole.SYSTEM) - .textContent("System prompt") - .metadata(metadata) + .textContent("dynamic system") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, false)) .build(); - Msg userMsg = Msg.builder().role(MsgRole.USER).textContent("User msg").build(); + Msg user = Msg.builder().role(MsgRole.USER).textContent("cacheable user").build(); + List formatted = formatter.format(List.of(system, user)); - List result = formatter.format(List.of(systemMsg, userMsg)); - formatter.applyCacheControl(result); + formatter.applyCacheControl(formatted, true); - assertEquals(NO_CACHE, result.get(0).getCacheControl()); - assertEquals(EPHEMERAL, result.get(1).getCacheControl()); + assertNoCacheControl(formatted.get(0)); + assertEquals(EPHEMERAL, lastPart(formatted.get(1)).getCacheControl()); } @Test - @DisplayName("should not serialize the no-cache marker into the API payload") - void noCacheNotSerialized() throws Exception { - Map metadata = new HashMap<>(); - metadata.put(MessageMetadataKeys.CACHE_CONTROL, false); - Msg msg = + void explicitMetadataPreventsMultiAgentHistoryMerge() { + DashScopeMultiAgentFormatter multiFormatter = new DashScopeMultiAgentFormatter(); + Msg cached = Msg.builder() + .name("agent-a") .role(MsgRole.USER) - .textContent("Hello") - .metadata(metadata) + .textContent("stable context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + Msg following = + Msg.builder() + .name("agent-b") + .role(MsgRole.USER) + .textContent("dynamic question") .build(); - List result = formatter.format(List.of(msg)); - String json = JsonUtils.getJsonCodec().toJson(result.get(0)); + List formatted = multiFormatter.format(List.of(cached, following)); - assertEquals(NO_CACHE, result.get(0).getCacheControl()); - assertFalse(json.contains("no_cache")); - assertFalse(json.contains("cache_control")); + assertEquals(2, formatted.size()); + assertEquals(EPHEMERAL, lastPart(formatted.get(0)).getCacheControl()); + assertNoCacheControl(formatted.get(1)); } @Test - @DisplayName("should set cache_control on system message via metadata") - void systemMessageMetadata() { - Map metadata = new HashMap<>(); - metadata.put(MessageMetadataKeys.CACHE_CONTROL, true); - Msg systemMsg = + void explicitNoCachePreventsMultiAgentHistoryMerge() { + DashScopeMultiAgentFormatter multiFormatter = new DashScopeMultiAgentFormatter(); + Msg notCached = Msg.builder() - .role(MsgRole.SYSTEM) - .textContent("System prompt") - .metadata(metadata) + .name("agent-a") + .role(MsgRole.USER) + .textContent("dynamic context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, false)) + .build(); + Msg following = + Msg.builder() + .name("agent-b") + .role(MsgRole.USER) + .textContent("cacheable question") .build(); - Msg userMsg = Msg.builder().role(MsgRole.USER).textContent("User msg").build(); - List result = formatter.format(List.of(systemMsg, userMsg)); + List formatted = multiFormatter.format(List.of(notCached, following)); + assertEquals(2, formatted.size()); - assertEquals(2, result.size()); - assertEquals(EPHEMERAL, result.get(0).getCacheControl()); - assertNull(result.get(1).getCacheControl()); - } - } + multiFormatter.applyCacheControl(formatted, true); - @Nested - @DisplayName("DashScopeMultiAgentFormatter cache_control") - class MultiAgentFormatterTest { + assertNoCacheControl(formatted.get(0)); + assertEquals(EPHEMERAL, lastPart(formatted.get(1)).getCacheControl()); + } @Test - @DisplayName("should add cache_control to system and last message") - void applyCacheControl() { + void multiAgentAutomaticStrategyUsesContentBlocks() { DashScopeMultiAgentFormatter multiFormatter = new DashScopeMultiAgentFormatter(); + List messages = + new ArrayList<>(List.of(message("system", "system"), message("user", "user"))); - List messages = new ArrayList<>(); - messages.add( - DashScopeMessage.builder().role("system").content("You are helpful.").build()); - messages.add(DashScopeMessage.builder().role("user").content("Hello").build()); + multiFormatter.applyCacheControl(messages, true); - multiFormatter.applyCacheControl(messages); + assertEquals(EPHEMERAL, lastPart(messages.get(0)).getCacheControl()); + assertEquals(EPHEMERAL, lastPart(messages.get(1)).getCacheControl()); + messages.forEach(message -> assertNull(message.getCacheControl())); + } + } + + private static DashScopeMessage message(String role, String content) { + return DashScopeMessage.builder().role(role).content(content).build(); + } + + private static DashScopeContentPart lastPart(DashScopeMessage message) { + List parts = message.getContentAsList(); + assertNotNull(parts); + assertFalse(parts.isEmpty()); + return parts.get(parts.size() - 1); + } - assertEquals(EPHEMERAL, messages.get(0).getCacheControl()); - assertEquals(EPHEMERAL, messages.get(1).getCacheControl()); + private static void assertNoCacheControl(DashScopeMessage message) { + assertNull(message.getCacheControl()); + List parts = message.getContentAsList(); + if (parts != null) { + assertTrue(parts.stream().allMatch(part -> part.getCacheControl() == null)); } } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java index 645d17abe0..f731e3b0dc 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-dashscope/src/test/java/io/agentscope/extensions/model/dashscope/formatter/DashScopeResponseParserTest.java @@ -30,6 +30,7 @@ import io.agentscope.extensions.model.dashscope.dto.DashScopeMessage; import io.agentscope.extensions.model.dashscope.dto.DashScopeOutput; import io.agentscope.extensions.model.dashscope.dto.DashScopeResponse; +import io.agentscope.extensions.model.dashscope.dto.DashScopeTokenDetails; import io.agentscope.extensions.model.dashscope.dto.DashScopeToolCall; import io.agentscope.extensions.model.dashscope.dto.DashScopeUsage; import java.time.Instant; @@ -191,6 +192,73 @@ void testParseResponseWithUsage() { assertEquals(5, chatResponse.getUsage().getOutputTokens()); } + @Test + void testParsePromptCacheUsageDetails() { + DashScopeTokenDetails details = new DashScopeTokenDetails(); + details.setCachedTokens(60); + details.setCacheCreationInputTokens(30); + + DashScopeUsage usage = new DashScopeUsage(); + usage.setInputTokens(100); + usage.setOutputTokens(20); + usage.setPromptTokensDetails(details); + + DashScopeResponse response = new DashScopeResponse(); + response.setRequestId("req-cache-usage"); + response.setUsage(usage); + + ChatResponse chatResponse = parser.parseResponse(response, startTime); + + assertNotNull(chatResponse.getUsage()); + assertEquals(100, chatResponse.getUsage().getInputTokens()); + assertEquals(20, chatResponse.getUsage().getOutputTokens()); + assertEquals(60, chatResponse.getUsage().getCachedTokens()); + assertEquals(30, chatResponse.getUsage().getCacheCreationInputTokens()); + } + + @Test + void testParseNestedPromptCacheCreationUsageDetails() { + DashScopeTokenDetails.CacheCreationDetails cacheCreation = + new DashScopeTokenDetails.CacheCreationDetails(); + cacheCreation.setEphemeral5mInputTokens(35); + DashScopeTokenDetails details = new DashScopeTokenDetails(); + details.setCachedTokens(55); + details.setCacheCreation(cacheCreation); + + DashScopeUsage usage = new DashScopeUsage(); + usage.setInputTokens(100); + usage.setOutputTokens(20); + usage.setPromptTokensDetails(details); + + DashScopeResponse response = new DashScopeResponse(); + response.setRequestId("req-nested-cache-usage"); + response.setUsage(usage); + + ChatResponse chatResponse = parser.parseResponse(response, startTime); + + assertNotNull(chatResponse.getUsage()); + assertEquals(55, chatResponse.getUsage().getCachedTokens()); + assertEquals(35, chatResponse.getUsage().getCacheCreationInputTokens()); + } + + @Test + void testParseLegacyTopLevelCacheUsage() { + DashScopeUsage usage = new DashScopeUsage(); + usage.setInputTokens(100); + usage.setOutputTokens(20); + usage.setCachedTokens(40); + usage.setCacheCreationInputTokens(25); + + DashScopeResponse response = new DashScopeResponse(); + response.setRequestId("req-legacy-cache-usage"); + response.setUsage(usage); + + ChatResponse chatResponse = parser.parseResponse(response, startTime); + + assertEquals(40, chatResponse.getUsage().getCachedTokens()); + assertEquals(25, chatResponse.getUsage().getCacheCreationInputTokens()); + } + @Test void testParseEmptyResponse() { DashScopeResponse response = new DashScopeResponse(); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/GeminiChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/GeminiChatModel.java index 9f1f08119a..23ef40e2f3 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/GeminiChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/GeminiChatModel.java @@ -59,6 +59,7 @@ *

  • Multi-agent conversation with history merging
    • *
  • Vision capabilities (images, audio, video)
    • *
  • Thinking mode (extended reasoning)
    • + *
  • Implicit caching and explicit cached content resource references
    • * */ public class GeminiChatModel extends ChatModelBase { @@ -250,29 +251,11 @@ protected Flux doStream0( return Flux.defer( () -> { try { - // Build generate content config - GenerateContentConfig.Builder configBuilder = - GenerateContentConfig.builder(); - // Use formatter to convert Msg to Gemini // Content List formattedMessages = formatter.format(messages); - - // Add tools if provided - if (tools != null && !tools.isEmpty()) { - formatter.applyTools(configBuilder, tools); - - // Apply tool choice if present - if (options != null && options.getToolChoice() != null) { - formatter.applyToolChoice( - configBuilder, options.getToolChoice()); - } - } - - // Apply generation options via formatter - formatter.applyOptions(configBuilder, options, defaultOptions); - - GenerateContentConfig config = configBuilder.build(); + GenerateContentConfig config = + buildGenerateContentConfig(tools, options); // Choose API based on streaming flag if (streamEnabled) { @@ -325,6 +308,27 @@ protected Flux doStream0( .subscribeOn(Schedulers.boundedElastic()); } + GenerateContentConfig buildGenerateContentConfig( + List tools, GenerateOptions options) { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + + // Apply options first because an explicit cached content resource also fixes tools and + // toolConfig. Gemini rejects those fields when cachedContent is set. + formatter.applyOptions(configBuilder, options, defaultOptions); + boolean usesCachedContent = configBuilder.build().cachedContent().isPresent(); + + if (!usesCachedContent && tools != null && !tools.isEmpty()) { + formatter.applyTools(configBuilder, tools); + GenerateOptions effectiveOptions = + GenerateOptions.mergeOptions(options, defaultOptions); + if (effectiveOptions != null && effectiveOptions.getToolChoice() != null) { + formatter.applyToolChoice(configBuilder, effectiveOptions.getToolChoice()); + } + } + + return configBuilder.build(); + } + @Override public String getModelName() { return modelName; diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java index ea37916c01..f4b0bb634f 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatter.java @@ -29,6 +29,8 @@ import io.agentscope.core.model.ToolSchema; import java.time.Instant; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; import java.util.function.Function; @@ -53,6 +55,12 @@ public class GeminiChatFormatter extends AbstractBaseFormatter< Content, GenerateContentResponse, GenerateContentConfig.Builder> { + /** Canonical additional body parameter for referencing a Gemini cached content resource. */ + public static final String CACHED_CONTENT_PARAM = "cachedContent"; + + /** Snake-case alias accepted for consistency with other Google Gen AI client conventions. */ + public static final String CACHED_CONTENT_PARAM_ALIAS = "cached_content"; + private final GeminiMessageConverter messageConverter; private final GeminiResponseParser responseParser; private final GeminiToolsHelper toolsHelper; @@ -127,6 +135,58 @@ public void applyOptions( thinkingConfigBuilder.thinkingBudget(thinkingBudget); configBuilder.thinkingConfig(thinkingConfigBuilder.build()); } + + String cachedContent = resolveCachedContent(options, defaultOptions); + if (cachedContent != null) { + configBuilder.cachedContent(cachedContent); + } + } + + private String resolveCachedContent(GenerateOptions options, GenerateOptions defaultOptions) { + CacheReference requestReference = cacheReference(options); + Object value = + requestReference.present + ? requestReference.value + : cacheReference(defaultOptions).value; + + if (value == null) { + return null; + } + if (!(value instanceof String cacheName) || cacheName.isBlank()) { + throw new IllegalArgumentException( + "Gemini cachedContent must be a non-blank String resource name"); + } + return cacheName; + } + + private CacheReference cacheReference(GenerateOptions options) { + if (options == null) { + return CacheReference.absent(); + } + + Map params = options.getAdditionalBodyParams(); + boolean hasCanonical = params.containsKey(CACHED_CONTENT_PARAM); + boolean hasAlias = params.containsKey(CACHED_CONTENT_PARAM_ALIAS); + Object canonical = params.get(CACHED_CONTENT_PARAM); + Object alias = params.get(CACHED_CONTENT_PARAM_ALIAS); + + if (hasCanonical && hasAlias && !Objects.equals(canonical, alias)) { + throw new IllegalArgumentException( + "Gemini cachedContent and cached_content must not specify different values"); + } + if (hasCanonical) { + return new CacheReference(true, canonical); + } + if (hasAlias) { + return new CacheReference(true, alias); + } + return CacheReference.absent(); + } + + private record CacheReference(boolean present, Object value) { + private static CacheReference absent() { + return new CacheReference(false, null); + } } /** diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParser.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParser.java index 605844f7c2..df8d7447b8 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParser.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/main/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParser.java @@ -95,13 +95,8 @@ public ChatResponse parseResponse(GenerateContentResponse response, Instant star GenerateContentResponseUsageMetadata metadata = response.usageMetadata().get(); int inputTokens = metadata.promptTokenCount().orElse(0); + int outputTokens = metadata.candidatesTokenCount().orElse(0); int cachedTokens = metadata.cachedContentTokenCount().orElse(0); - int totalOutputTokens = metadata.candidatesTokenCount().orElse(0); - int thinkingTokens = metadata.thoughtsTokenCount().orElse(0); - - // Output tokens exclude thinking tokens (following DashScope behavior) - // In Gemini, candidatesTokenCount includes thinking, so we subtract it - int outputTokens = totalOutputTokens - thinkingTokens; usage = ChatUsage.builder() diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/GeminiChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/GeminiChatModelTest.java index f78b03232e..e74735e9d7 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/GeminiChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/GeminiChatModelTest.java @@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.genai.types.ClientOptions; +import com.google.genai.types.GenerateContentConfig; import com.google.genai.types.HttpOptions; import com.google.genai.types.ProxyOptions; import io.agentscope.core.message.Msg; @@ -283,6 +284,43 @@ void testGenerateOptionsConfiguration() { assertNotNull(modelWithOptions); } + @Test + @DisplayName("Should reference cached content without resending tools or tool config") + void testCachedContentRequestSuppressesCachedToolConfiguration() { + GeminiChatModel model = + GeminiChatModel.builder().apiKey(mockApiKey).modelName("gemini-2.5-flash").build(); + ToolSchema tool = + ToolSchema.builder() + .name("search") + .description("Search") + .parameters(java.util.Map.of("type", "object")) + .build(); + GenerateOptions options = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/cache-123") + .toolChoice(new ToolChoice.Required()) + .build(); + + GenerateContentConfig config = model.buildGenerateContentConfig(List.of(tool), options); + + assertEquals("cachedContents/cache-123", config.cachedContent().orElseThrow()); + assertTrue(config.tools().isEmpty()); + assertTrue(config.toolConfig().isEmpty()); + } + + @Test + @DisplayName("Should leave Gemini implicit caching to the provider") + void testCacheControlDoesNotCreateExplicitCachedContentReference() { + GeminiChatModel model = + GeminiChatModel.builder().apiKey(mockApiKey).modelName("gemini-2.5-flash").build(); + + GenerateContentConfig config = + model.buildGenerateContentConfig( + List.of(), GenerateOptions.builder().cacheControl(true).build()); + + assertTrue(config.cachedContent().isEmpty()); + } + @Test @DisplayName("Should build with minimal parameters") void testMinimalBuilder() { diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java index b783a02c7b..f00a0f8ebd 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiChatFormatterTest.java @@ -17,6 +17,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.genai.types.Content; @@ -94,6 +95,97 @@ void testApplyOptions() { assertEquals(0.3f, config.presencePenalty().get(), 0.001f); } + @Test + void testApplyCachedContentFromCanonicalParameter() { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + GenerateOptions options = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/cache-123") + .build(); + + formatter.applyOptions(configBuilder, options, null); + + assertEquals( + "cachedContents/cache-123", configBuilder.build().cachedContent().orElseThrow()); + } + + @Test + void testApplyCachedContentFromSnakeCaseAlias() { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + GenerateOptions options = + GenerateOptions.builder() + .additionalBodyParam("cached_content", "cachedContents/cache-123") + .build(); + + formatter.applyOptions(configBuilder, options, null); + + assertEquals( + "cachedContents/cache-123", configBuilder.build().cachedContent().orElseThrow()); + } + + @Test + void testRequestCachedContentOverridesDefaultAlias() { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + GenerateOptions defaults = + GenerateOptions.builder() + .additionalBodyParam("cached_content", "cachedContents/default") + .build(); + GenerateOptions request = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/request") + .build(); + + formatter.applyOptions(configBuilder, request, defaults); + + assertEquals("cachedContents/request", configBuilder.build().cachedContent().orElseThrow()); + } + + @Test + void testRequestCachedContentShortCircuitsInvalidDefaults() { + GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); + GenerateOptions defaults = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/default-one") + .additionalBodyParam("cached_content", "cachedContents/default-two") + .build(); + GenerateOptions request = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/request") + .build(); + + formatter.applyOptions(configBuilder, request, defaults); + + assertEquals("cachedContents/request", configBuilder.build().cachedContent().orElseThrow()); + } + + @Test + void testRejectConflictingCachedContentAliases() { + GenerateOptions options = + GenerateOptions.builder() + .additionalBodyParam("cachedContent", "cachedContents/one") + .additionalBodyParam("cached_content", "cachedContents/two") + .build(); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyOptions(GenerateContentConfig.builder(), options, null)); + } + + @Test + void testRejectInvalidCachedContentReference() { + GenerateOptions blank = + GenerateOptions.builder().additionalBodyParam("cachedContent", " ").build(); + GenerateOptions nonString = + GenerateOptions.builder().additionalBodyParam("cachedContent", 123).build(); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyOptions(GenerateContentConfig.builder(), blank, null)); + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyOptions(GenerateContentConfig.builder(), nonString, null)); + } + @Test void testApplyTools() { GenerateContentConfig.Builder configBuilder = GenerateContentConfig.builder(); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParserTest.java index 0da20af442..08fb6c60c4 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-gemini/src/test/java/io/agentscope/extensions/model/gemini/formatter/GeminiResponseParserTest.java @@ -216,8 +216,9 @@ void testParseUsageMetadata() { GenerateContentResponseUsageMetadata usageMetadata = GenerateContentResponseUsageMetadata.builder() .promptTokenCount(100) - .candidatesTokenCount(60) // Includes thinking - .thoughtsTokenCount(10) // Thinking tokens + .cachedContentTokenCount(60) + .candidatesTokenCount(50) + .thoughtsTokenCount(10) .totalTokenCount(160) .build(); @@ -238,9 +239,13 @@ void testParseUsageMetadata() { // Input tokens = promptTokenCount assertEquals(100, usage.getInputTokens()); - // Output tokens = candidatesTokenCount - thoughtsTokenCount + // Output tokens = candidatesTokenCount; thoughts are reported separately by Gemini assertEquals(50, usage.getOutputTokens()); + // Cached tokens are a subset of promptTokenCount + assertEquals(60, usage.getCachedTokens()); + assertEquals(0, usage.getCacheCreationInputTokens()); + // Time should be > 0 assertTrue(usage.getTime() >= 0); } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java index 0fb533a544..ab7e5e55ac 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/OpenAIChatModel.java @@ -35,8 +35,10 @@ import io.agentscope.extensions.model.openai.dto.OpenAIStreamOptions; import io.agentscope.extensions.model.openai.formatter.OpenAIBaseFormatter; import io.agentscope.extensions.model.openai.formatter.OpenAIChatFormatter; +import java.net.URI; import java.time.Instant; import java.util.List; +import java.util.Locale; import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -155,10 +157,20 @@ protected Flux doStream0( formatter.applyToolChoice(request, effectiveOptions.getToolChoice()); } - // Apply cache control if enabled (adds cache_control to system msgs + last msg) - if (Boolean.TRUE.equals(effectiveOptions.getCacheControl()) - && formatter instanceof OpenAIBaseFormatter openAIFormatter) { - openAIFormatter.applyCacheControl(request.getMessages()); + // DashScope-compatible endpoints require content-block markers. Normalize explicit + // metadata even when automatic cache control is disabled. + if (formatter instanceof OpenAIBaseFormatter openAIFormatter) { + boolean automaticCacheControl = Boolean.TRUE.equals(effectiveOptions.getCacheControl()); + if (isDashScopeCompatibleBaseUrl(baseUrl)) { + openAIFormatter.applyDashScopeCacheControl( + request.getMessages(), automaticCacheControl); + } else if (isOfficialOpenAIBaseUrl(baseUrl)) { + openAIFormatter.applyOpenAIPromptCache(request); + } else { + // The OpenAI-compatible surface does not imply a provider's cache protocol. + // Never leak AgentScope's internal legacy marker to an unknown endpoint. + openAIFormatter.clearLegacyCacheControl(request.getMessages()); + } } // Make the API call @@ -198,6 +210,43 @@ protected Flux doStream0( } } + static boolean isDashScopeCompatibleBaseUrl(String baseUrl) { + if (baseUrl == null || baseUrl.isBlank()) { + return false; + } + try { + String host = URI.create(baseUrl).getHost(); + if (host == null) { + return false; + } + String normalizedHost = host.toLowerCase(Locale.ROOT); + return normalizedHost.equals("dashscope.aliyuncs.com") + || (normalizedHost.startsWith("dashscope-") + && normalizedHost.endsWith(".aliyuncs.com")) + || normalizedHost.endsWith(".dashscope.aliyuncs.com") + || normalizedHost.endsWith(".maas.aliyuncs.com"); + } catch (IllegalArgumentException e) { + return false; + } + } + + static boolean isOfficialOpenAIBaseUrl(String baseUrl) { + if (baseUrl == null || baseUrl.isBlank()) { + return true; + } + try { + String host = URI.create(baseUrl).getHost(); + if (host == null) { + return false; + } + String normalizedHost = host.toLowerCase(Locale.ROOT); + return normalizedHost.equals("api.openai.com") + || normalizedHost.endsWith(".api.openai.com"); + } catch (IllegalArgumentException e) { + return false; + } + } + /** * Gets the model name for logging and identification. * diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java index 6a8612e33a..0f9491d830 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIContentPart.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; /** * OpenAI content part DTO for multimodal messages. @@ -65,6 +66,14 @@ public class OpenAIContentPart { @JsonProperty("video_url") private OpenAIVideoUrl videoUrl; + /** Prompt cache control marker used by compatible providers such as DashScope. */ + @JsonProperty("cache_control") + private Map cacheControl; + + /** Explicit prompt cache breakpoint used by the official OpenAI API. */ + @JsonProperty("prompt_cache_breakpoint") + private Map promptCacheBreakpoint; + public OpenAIContentPart() {} public String getType() { @@ -107,6 +116,22 @@ public void setVideoUrl(OpenAIVideoUrl videoUrl) { this.videoUrl = videoUrl; } + public Map getCacheControl() { + return cacheControl; + } + + public void setCacheControl(Map cacheControl) { + this.cacheControl = cacheControl; + } + + public Map getPromptCacheBreakpoint() { + return promptCacheBreakpoint; + } + + public void setPromptCacheBreakpoint(Map promptCacheBreakpoint) { + this.promptCacheBreakpoint = promptCacheBreakpoint; + } + /** * Create a text content part. * @@ -206,6 +231,16 @@ public Builder videoUrl(OpenAIVideoUrl videoUrl) { return this; } + public Builder cacheControl(Map cacheControl) { + part.setCacheControl(cacheControl); + return this; + } + + public Builder promptCacheBreakpoint(Map promptCacheBreakpoint) { + part.setPromptCacheBreakpoint(promptCacheBreakpoint); + return this; + } + public OpenAIContentPart build() { OpenAIContentPart result = part; part = new OpenAIContentPart(); diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java index 6a711355fb..2d82d1bbb1 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/dto/OpenAIUsage.java @@ -108,6 +108,15 @@ public static class PromptTokensDetails { @JsonProperty("audio_tokens") private Integer audioTokens; + @JsonProperty("cache_write_tokens") + private Integer cacheWriteTokens; + + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + @JsonProperty("cache_creation") + private CacheCreationDetails cacheCreation; + public Integer getCachedTokens() { return cachedTokens; } @@ -123,6 +132,74 @@ public Integer getAudioTokens() { public void setAudioTokens(Integer audioTokens) { this.audioTokens = audioTokens; } + + public Integer getCacheWriteTokens() { + return cacheWriteTokens; + } + + public void setCacheWriteTokens(Integer cacheWriteTokens) { + this.cacheWriteTokens = cacheWriteTokens; + } + + public Integer getCacheCreationInputTokens() { + return cacheCreationInputTokens; + } + + /** Returns prompt-cache creation tokens across OpenAI and compatible response shapes. */ + public Integer resolveCacheCreationInputTokens() { + if (cacheWriteTokens != null) { + return cacheWriteTokens; + } + if (cacheCreationInputTokens != null) { + return cacheCreationInputTokens; + } + return cacheCreation != null ? cacheCreation.getInputTokens() : null; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } + + public CacheCreationDetails getCacheCreation() { + return cacheCreation; + } + + public void setCacheCreation(CacheCreationDetails cacheCreation) { + this.cacheCreation = cacheCreation; + } + + /** Nested explicit-cache creation details returned by compatible providers. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class CacheCreationDetails { + + @JsonProperty("cache_creation_input_tokens") + private Integer cacheCreationInputTokens; + + @JsonProperty("ephemeral_5m_input_tokens") + private Integer ephemeral5mInputTokens; + + public Integer getCacheCreationInputTokens() { + return cacheCreationInputTokens; + } + + public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { + this.cacheCreationInputTokens = cacheCreationInputTokens; + } + + public Integer getEphemeral5mInputTokens() { + return ephemeral5mInputTokens; + } + + public void setEphemeral5mInputTokens(Integer ephemeral5mInputTokens) { + this.ephemeral5mInputTokens = ephemeral5mInputTokens; + } + + Integer getInputTokens() { + return cacheCreationInputTokens != null + ? cacheCreationInputTokens + : ephemeral5mInputTokens; + } + } } /** diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java index a1aef6620d..0beb8bbe7c 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIBaseFormatter.java @@ -20,10 +20,12 @@ import io.agentscope.core.model.GenerateOptions; import io.agentscope.core.model.ToolChoice; import io.agentscope.core.model.ToolSchema; +import io.agentscope.extensions.model.openai.dto.OpenAIContentPart; import io.agentscope.extensions.model.openai.dto.OpenAIMessage; import io.agentscope.extensions.model.openai.dto.OpenAIRequest; import io.agentscope.extensions.model.openai.dto.OpenAIResponse; import java.time.Instant; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -44,6 +46,8 @@ public abstract class OpenAIBaseFormatter extends AbstractBaseFormatter { private static final Map EPHEMERAL_CACHE_CONTROL = Map.of("type", "ephemeral"); + private static final Map EXPLICIT_PROMPT_CACHE_BREAKPOINT = + Map.of("mode", "explicit"); /** * Sentinel for an explicit "no cache" intent. An empty map is serialized away (via @@ -178,14 +182,18 @@ public OpenAIRequest buildRequest( } /** - * Apply cache control to OpenAI messages. + * Apply legacy message-level cache control. * - *

    Adds cache_control: {"type": "ephemeral"} to all system messages and the last - * message in the list. Messages that already carry a cache_control value (including the empty - * "no cache" sentinel) are left untouched. + *

    This method emits the provider-specific {@code cache_control} field used by older + * OpenAI-compatible integrations. The official OpenAI request path does not call it: OpenAI + * automatic caching is provider-managed, while explicit AgentScope markers are normalized to + * {@code prompt_cache_breakpoint} by {@link #applyOpenAIPromptCache(OpenAIRequest)}. * * @param messages the list of formatted OpenAI messages + * @deprecated use the model request path so the selected endpoint can apply its native cache + * protocol */ + @Deprecated(since = "2.0.3", forRemoval = false) public void applyCacheControl(List messages) { if (messages == null || messages.isEmpty()) { return; @@ -201,6 +209,239 @@ public void applyCacheControl(List messages) { } } + /** + * Apply DashScope-compatible content-block cache control. + * + *

    Legacy message-level markers are migrated to the last content block. Automatic + * breakpoints use the shared first-system and last-conversation strategy. + * + * @param messages formatted OpenAI-compatible messages + * @param automatic whether to add automatic breakpoints + */ + public void applyDashScopeCacheControl(List messages, boolean automatic) { + if (messages == null || messages.isEmpty()) { + return; + } + + List selected = + selectPromptCacheBreakpoints( + messages, + automatic, + OpenAIBaseFormatter::hasExplicitCacheControl, + message -> "system".equals(message.getRole()), + OpenAIBaseFormatter::isCacheable); + for (OpenAIMessage message : selected) { + applyDashScopeCacheControlToContentBlock(message); + } + for (OpenAIMessage message : messages) { + message.setCacheControl(null); + } + + int markerCount = countContentBlockCacheMarkers(messages); + if (markerCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "DashScope supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " cache_control markers, but got " + + markerCount); + } + } + + /** + * Normalize manually marked messages to the official OpenAI explicit prompt-cache protocol. + * + *

    OpenAI enables automatic prompt caching by default, so this method never invents + * breakpoints for {@link GenerateOptions#getCacheControl() cacheControl=true}. It only + * migrates explicit AgentScope metadata (temporarily represented by legacy + * {@code cache_control}) to a content-part {@code prompt_cache_breakpoint}, and enables + * explicit mode at request level. + * + * @param request request to normalize + */ + public void applyOpenAIPromptCache(OpenAIRequest request) { + if (request == null || request.getMessages() == null) { + return; + } + + for (OpenAIMessage message : request.getMessages()) { + boolean legacyExplicit = hasSerializableCacheControl(message.getCacheControl()); + List parts = message.getContentAsList(); + if (parts != null) { + for (OpenAIContentPart part : parts) { + if (hasSerializableCacheControl(part.getCacheControl())) { + legacyExplicit = true; + } + if (part.getCacheControl() != null) { + part.setCacheControl(null); + } + } + } + + if (legacyExplicit && !hasPromptCacheBreakpoint(message)) { + parts = ensureContentArray(message); + if (parts.isEmpty()) { + throw new IllegalArgumentException( + "Cannot apply prompt_cache_breakpoint to a message without cacheable" + + " content"); + } + parts.get(parts.size() - 1) + .setPromptCacheBreakpoint(EXPLICIT_PROMPT_CACHE_BREAKPOINT); + } + message.setCacheControl(null); + } + + int markerCount = countPromptCacheBreakpoints(request.getMessages()); + if (markerCount > MAX_PROMPT_CACHE_BREAKPOINTS) { + throw new IllegalArgumentException( + "OpenAI supports at most " + + MAX_PROMPT_CACHE_BREAKPOINTS + + " prompt_cache_breakpoint markers, but got " + + markerCount); + } + if (markerCount > 0) { + Map promptCacheOptions = new java.util.LinkedHashMap<>(); + Object configured = + request.getExtraParams() != null + ? request.getExtraParams().get("prompt_cache_options") + : null; + if (configured instanceof Map configuredMap) { + configuredMap.forEach( + (key, value) -> { + if (key instanceof String stringKey) { + promptCacheOptions.put(stringKey, value); + } + }); + } + promptCacheOptions.put("mode", "explicit"); + request.addExtraParam("prompt_cache_options", promptCacheOptions); + } + } + + /** Remove AgentScope's internal legacy markers before calling an unknown compatible API. */ + public void clearLegacyCacheControl(List messages) { + if (messages == null) { + return; + } + for (OpenAIMessage message : messages) { + message.setCacheControl(null); + List parts = message.getContentAsList(); + if (parts != null) { + parts.forEach(part -> part.setCacheControl(null)); + } + } + } + + static void applyDashScopeCacheControlToContentBlock(OpenAIMessage message) { + if (isExplicitNoCache(message)) { + message.setCacheControl(null); + return; + } + if (hasContentBlockCacheControl(message)) { + message.setCacheControl(null); + return; + } + + List parts = ensureContentArray(message); + if (parts.isEmpty()) { + throw new IllegalArgumentException( + "Cannot apply cache_control to a message without cacheable content"); + } + Map cacheControl = + message.getCacheControl() != null + ? message.getCacheControl() + : EPHEMERAL_CACHE_CONTROL; + parts.get(parts.size() - 1).setCacheControl(cacheControl); + message.setCacheControl(null); + } + + @SuppressWarnings("unchecked") + static List ensureContentArray(OpenAIMessage message) { + Object content = message.getContent(); + if (content instanceof List) { + return (List) content; + } + if (content instanceof String text) { + List parts = new ArrayList<>(List.of(OpenAIContentPart.text(text))); + message.setContent(parts); + return parts; + } + return List.of(); + } + + private static boolean isCacheable(OpenAIMessage message) { + if (isExplicitNoCache(message)) { + return false; + } + if (!("system".equals(message.getRole()) + || "user".equals(message.getRole()) + || "assistant".equals(message.getRole()) + || "tool".equals(message.getRole()))) { + return false; + } + Object content = message.getContent(); + return (content instanceof String text && !text.isEmpty()) + || (content instanceof List parts && !parts.isEmpty()); + } + + private static boolean hasExplicitCacheControl(OpenAIMessage message) { + return hasSerializableCacheControl(message.getCacheControl()) + || hasContentBlockCacheControl(message); + } + + private static boolean isExplicitNoCache(OpenAIMessage message) { + return message.getCacheControl() != null && message.getCacheControl().isEmpty(); + } + + private static boolean hasSerializableCacheControl(Map cacheControl) { + return cacheControl != null && !cacheControl.isEmpty(); + } + + private static boolean hasContentBlockCacheControl(OpenAIMessage message) { + List parts = message.getContentAsList(); + return parts != null + && parts.stream() + .anyMatch(part -> hasSerializableCacheControl(part.getCacheControl())); + } + + private static boolean hasPromptCacheBreakpoint(OpenAIMessage message) { + List parts = message.getContentAsList(); + return parts != null + && parts.stream().anyMatch(part -> part.getPromptCacheBreakpoint() != null); + } + + private static int countPromptCacheBreakpoints(List messages) { + int count = 0; + for (OpenAIMessage message : messages) { + List parts = message.getContentAsList(); + if (parts != null) { + count += + (int) + parts.stream() + .filter(part -> part.getPromptCacheBreakpoint() != null) + .count(); + } + } + return count; + } + + private static int countContentBlockCacheMarkers(List messages) { + int count = 0; + for (OpenAIMessage message : messages) { + List parts = message.getContentAsList(); + if (parts != null) { + count += + (int) + parts.stream() + .filter( + part -> + hasSerializableCacheControl( + part.getCacheControl())) + .count(); + } + } + return count; + } + /** * Get the ephemeral cache control constant. * diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java index 1e4e940238..94836c30ad 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/main/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParser.java @@ -90,6 +90,15 @@ private long getSafeCachedTokens(OpenAIUsage usage) { return cachedTokens != null ? cachedTokens : 0; } + private long getSafeCacheCreationInputTokens(OpenAIUsage usage) { + if (usage == null || usage.getPromptTokensDetails() == null) { + return 0; + } + Integer cacheCreationInputTokens = + usage.getPromptTokensDetails().resolveCacheCreationInputTokens(); + return cacheCreationInputTokens != null ? cacheCreationInputTokens : 0; + } + public OpenAIResponseParser() {} /** @@ -128,6 +137,8 @@ protected ChatResponse parseCompletionResponse(OpenAIResponse response, Instant .inputTokens((int) getSafePromptTokens(openAIUsage)) .outputTokens((int) getSafeCompletionTokens(openAIUsage)) .cachedTokens((int) getSafeCachedTokens(openAIUsage)) + .cacheCreationInputTokens( + (int) getSafeCacheCreationInputTokens(openAIUsage)) .time( Duration.between(startTime, Instant.now()).toMillis() / 1000.0) @@ -372,6 +383,8 @@ protected ChatResponse parseChunkResponse(OpenAIResponse response, Instant start ? openAIUsage.getCompletionTokens() : 0) .cachedTokens((int) getSafeCachedTokens(openAIUsage)) + .cacheCreationInputTokens( + (int) getSafeCacheCreationInputTokens(openAIUsage)) .time( Duration.between(startTime, Instant.now()).toMillis() / 1000.0) diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java index 9b3cbf5b44..3b50962b11 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/OpenAIChatModelTest.java @@ -113,6 +113,40 @@ void testNativeStructuredOutputDisabled() { assertFalse(m.supportsNativeStructuredOutput()); } + @Test + @DisplayName("Should recognize Alibaba OpenAI-compatible endpoints") + void testDashScopeCompatibleBaseUrlDetection() { + assertTrue( + OpenAIChatModel.isDashScopeCompatibleBaseUrl( + "https://dashscope.aliyuncs.com/compatible-mode/v1")); + assertTrue( + OpenAIChatModel.isDashScopeCompatibleBaseUrl( + "https://dashscope-intl.aliyuncs.com/compatible-mode/v1")); + assertTrue( + OpenAIChatModel.isDashScopeCompatibleBaseUrl( + "https://dashscope-us.aliyuncs.com/compatible-mode/v1")); + assertTrue( + OpenAIChatModel.isDashScopeCompatibleBaseUrl( + "https://workspace.cn-beijing.maas.aliyuncs.com/compatible-mode/v1")); + assertFalse(OpenAIChatModel.isDashScopeCompatibleBaseUrl("https://api.openai.com/v1")); + assertFalse(OpenAIChatModel.isDashScopeCompatibleBaseUrl(null)); + assertFalse(OpenAIChatModel.isDashScopeCompatibleBaseUrl("not a valid URL")); + + assertTrue(OpenAIChatModel.isOfficialOpenAIBaseUrl(null)); + assertTrue(OpenAIChatModel.isOfficialOpenAIBaseUrl("")); + assertTrue(OpenAIChatModel.isOfficialOpenAIBaseUrl("https://api.openai.com/v1")); + for (String region : List.of("us", "eu", "au", "ca", "jp", "in", "sg", "kr", "gb", "ae")) { + assertTrue( + OpenAIChatModel.isOfficialOpenAIBaseUrl( + "https://" + region + ".api.openai.com/v1"), + region); + } + assertFalse( + OpenAIChatModel.isOfficialOpenAIBaseUrl("https://api.openai.com.evil.example/v1")); + assertFalse( + OpenAIChatModel.isOfficialOpenAIBaseUrl("https://gateway.example.com/openai/v1")); + } + @Test @DisplayName("Should make non-streaming call successfully") void testNonStreamingCall() throws Exception { @@ -336,8 +370,8 @@ void testBuildModelWithEndpointPath() throws Exception { } @Test - @DisplayName("Should apply cache_control to request when cacheControl option is enabled") - void testCacheControlApplied() throws Exception { + @DisplayName("Should not guess a cache protocol for an unknown compatible endpoint") + void testUnknownCompatibleCacheControlIsNotApplied() throws Exception { String responseJson = """ { @@ -385,12 +419,12 @@ void testCacheControlApplied() throws Exception { RecordedRequest request = mockServer.takeRequest(1, TimeUnit.SECONDS); assertNotNull(request); String body = request.getBody().readUtf8(); - assertTrue( + assertFalse( body.contains("\"cache_control\""), - "Request body should contain cache_control: " + body); - assertTrue( - body.contains("\"ephemeral\""), - "Request body should contain ephemeral cache type: " + body); + "Unknown compatible endpoint must not receive cache_control: " + body); + assertFalse( + body.contains("\"prompt_cache_breakpoint\""), + "Unknown compatible endpoint must not receive OpenAI cache fields: " + body); } @Test diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java index b896715270..3cff53dc87 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAICacheControlTest.java @@ -18,12 +18,16 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import com.fasterxml.jackson.databind.JsonNode; import io.agentscope.core.message.MessageMetadataKeys; import io.agentscope.core.message.Msg; import io.agentscope.core.message.MsgRole; import io.agentscope.core.util.JsonUtils; +import io.agentscope.extensions.model.openai.dto.OpenAIContentPart; import io.agentscope.extensions.model.openai.dto.OpenAIMessage; +import io.agentscope.extensions.model.openai.dto.OpenAIRequest; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -281,4 +285,349 @@ void systemMessageMetadata() { assertNull(result.get(1).getCacheControl()); } } + + @Nested + @DisplayName("Official OpenAI prompt caching") + class OfficialOpenAITest { + + @Test + @DisplayName("should serialize explicit metadata using the official protocol") + void exactExplicitProtocolJson() { + Msg msg = + Msg.builder() + .role(MsgRole.USER) + .textContent("Pinned context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + OpenAIRequest request = + OpenAIRequest.builder() + .model("gpt-5.6") + .messages(formatter.format(List.of(msg))) + .stream(false) + .build(); + + formatter.applyOpenAIPromptCache(request); + + JsonNode json = toJsonTree(request); + JsonNode message = json.path("messages").get(0); + assertFalse(message.has("cache_control")); + assertEquals( + "explicit", + message.path("content") + .get(0) + .path("prompt_cache_breakpoint") + .path("mode") + .asText()); + assertFalse(message.path("content").get(0).has("cache_control")); + assertEquals("explicit", json.path("prompt_cache_options").path("mode").asText()); + } + + @Test + @DisplayName("should rely on server automatic caching when there is no explicit marker") + void noInventedAutomaticMarkers() { + OpenAIRequest request = + OpenAIRequest.builder() + .model("gpt-5.6") + .messages( + new ArrayList<>( + List.of( + OpenAIMessage.builder() + .role("system") + .content("System") + .build(), + OpenAIMessage.builder() + .role("user") + .content("Question") + .build()))) + .build(); + + formatter.applyOpenAIPromptCache(request); + + JsonNode json = toJsonTree(request); + assertFalse(json.has("prompt_cache_options")); + assertFalse(json.toString().contains("prompt_cache_breakpoint")); + assertFalse(json.toString().contains("cache_control")); + } + + @Test + @DisplayName("should not turn explicit no-cache into an official breakpoint") + void explicitNoCacheDoesNotEnableOfficialBreakpoints() { + Msg msg = + Msg.builder() + .role(MsgRole.USER) + .textContent("Dynamic context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, false)) + .build(); + OpenAIRequest request = + OpenAIRequest.builder() + .model("gpt-5.6") + .messages(formatter.format(List.of(msg))) + .build(); + + formatter.applyOpenAIPromptCache(request); + + JsonNode json = toJsonTree(request); + assertFalse(json.has("prompt_cache_options")); + assertFalse(json.toString().contains("prompt_cache_breakpoint")); + assertFalse(json.toString().contains("cache_control")); + } + + @Test + @DisplayName("should preserve other prompt cache options and force explicit mode") + void mergesConfiguredPromptCacheOptions() { + OpenAIMessage message = + OpenAIMessage.builder() + .role("user") + .content("Context") + .cacheControl(EPHEMERAL) + .build(); + OpenAIRequest request = + OpenAIRequest.builder() + .model("gpt-5.6") + .messages(List.of(message)) + .extraParam( + "prompt_cache_options", + Map.of("mode", "implicit", "ttl", "30m")) + .build(); + + formatter.applyOpenAIPromptCache(request); + + JsonNode json = toJsonTree(request); + assertEquals("explicit", json.path("prompt_cache_options").path("mode").asText()); + assertEquals("30m", json.path("prompt_cache_options").path("ttl").asText()); + } + + @Test + @DisplayName("should place a multimodal breakpoint on the final content part") + void multimodalFinalPart() { + OpenAIMessage message = + OpenAIMessage.builder() + .role("user") + .content( + new ArrayList<>( + List.of( + OpenAIContentPart.text("Look"), + OpenAIContentPart.imageUrl( + "https://example.com/image.png")))) + .cacheControl(EPHEMERAL) + .build(); + OpenAIRequest request = OpenAIRequest.builder().messages(List.of(message)).build(); + + formatter.applyOpenAIPromptCache(request); + + assertNull(message.getContentAsList().get(0).getPromptCacheBreakpoint()); + assertEquals( + Map.of("mode", "explicit"), + message.getContentAsList().get(1).getPromptCacheBreakpoint()); + } + + @Test + @DisplayName("should reject more than four explicit breakpoints") + void tooManyBreakpoints() { + List messages = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + messages.add( + OpenAIMessage.builder() + .role("user") + .content("Context " + i) + .cacheControl(EPHEMERAL) + .build()); + } + OpenAIRequest request = OpenAIRequest.builder().messages(messages).build(); + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyOpenAIPromptCache(request)); + } + + @Test + @DisplayName("should clear legacy fields for unknown compatible providers") + void clearUnknownCompatibleMarkers() { + OpenAIContentPart part = + OpenAIContentPart.builder() + .type("text") + .text("Context") + .cacheControl(EPHEMERAL) + .build(); + OpenAIMessage message = + OpenAIMessage.builder() + .role("user") + .content(List.of(part)) + .cacheControl(EPHEMERAL) + .build(); + + formatter.clearLegacyCacheControl(List.of(message)); + + JsonNode json = toJsonTree(message); + assertFalse(json.toString().contains("cache_control")); + assertFalse(json.toString().contains("prompt_cache_breakpoint")); + } + } + + @Nested + @DisplayName("DashScope-compatible cache_control") + class DashScopeCompatibleTest { + + @Test + @DisplayName("should serialize markers on content blocks using shared strategy") + void exactContentBlockJson() { + List messages = + new ArrayList<>( + List.of( + OpenAIMessage.builder() + .role("system") + .content("System 1") + .build(), + OpenAIMessage.builder() + .role("system") + .content("System 2") + .build(), + OpenAIMessage.builder() + .role("user") + .content("Question") + .build())); + + formatter.applyDashScopeCacheControl(messages, true); + + JsonNode first = toJsonTree(messages.get(0)); + JsonNode second = toJsonTree(messages.get(1)); + JsonNode last = toJsonTree(messages.get(2)); + assertFalse(first.has("cache_control")); + assertEquals("text", first.path("content").get(0).path("type").asText()); + assertEquals( + "ephemeral", + first.path("content").get(0).path("cache_control").path("type").asText()); + assertFalse(second.has("cache_control")); + assertEquals("System 2", second.path("content").asText()); + assertEquals( + "ephemeral", + last.path("content").get(0).path("cache_control").path("type").asText()); + } + + @Test + @DisplayName("should retain explicit metadata marker when automatic strategy is disabled") + void explicitMetadataWithoutAutomaticStrategy() { + Msg msg = + Msg.builder() + .role(MsgRole.USER) + .textContent("Pinned context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); + List messages = formatter.format(List.of(msg)); + + formatter.applyDashScopeCacheControl(messages, false); + + assertNull(messages.get(0).getCacheControl()); + assertEquals(EPHEMERAL, messages.get(0).getContentAsList().get(0).getCacheControl()); + } + + @Test + @DisplayName("should skip explicit no-cache during automatic selection") + void explicitNoCacheIsNotReenabled() { + Msg system = + Msg.builder() + .role(MsgRole.SYSTEM) + .textContent("Dynamic system") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, false)) + .build(); + Msg user = Msg.builder().role(MsgRole.USER).textContent("Cacheable user").build(); + List messages = formatter.format(List.of(system, user)); + + formatter.applyDashScopeCacheControl(messages, true); + + JsonNode first = toJsonTree(messages.get(0)); + JsonNode last = toJsonTree(messages.get(1)); + assertFalse(first.toString().contains("cache_control")); + assertEquals( + "ephemeral", + last.path("content").get(0).path("cache_control").path("type").asText()); + } + + @Test + @DisplayName("should preserve explicit no-cache as a multi-agent boundary") + void explicitNoCachePreventsMultiAgentMerge() { + OpenAIMultiAgentFormatter multiFormatter = new OpenAIMultiAgentFormatter(); + Msg notCached = + Msg.builder() + .name("agent-a") + .role(MsgRole.USER) + .textContent("Dynamic context") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, false)) + .build(); + Msg following = + Msg.builder() + .name("agent-b") + .role(MsgRole.USER) + .textContent("Cacheable question") + .build(); + List messages = multiFormatter.format(List.of(notCached, following)); + assertEquals(2, messages.size()); + + multiFormatter.applyDashScopeCacheControl(messages, true); + + assertFalse(toJsonTree(messages.get(0)).toString().contains("cache_control")); + assertEquals( + "ephemeral", + toJsonTree(messages.get(1)) + .path("content") + .get(0) + .path("cache_control") + .path("type") + .asText()); + } + + @Test + @DisplayName("should place multimodal marker on the final content block") + void multimodalFinalBlock() { + OpenAIMessage message = + OpenAIMessage.builder() + .role("user") + .content( + new ArrayList<>( + List.of( + OpenAIContentPart.text("Look"), + OpenAIContentPart.imageUrl( + "https://example.com/image.png")))) + .build(); + + formatter.applyDashScopeCacheControl(List.of(message), true); + + assertNull(message.getContentAsList().get(0).getCacheControl()); + assertEquals(EPHEMERAL, message.getContentAsList().get(1).getCacheControl()); + } + + @Test + @DisplayName("should reject more than four explicit content-block markers") + void tooManyExplicitMarkers() { + List messages = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + messages.add( + OpenAIMessage.builder() + .role("user") + .content( + new ArrayList<>( + List.of( + OpenAIContentPart.builder() + .type("text") + .text("Context " + i) + .cacheControl(EPHEMERAL) + .build()))) + .build()); + } + + assertThrows( + IllegalArgumentException.class, + () -> formatter.applyDashScopeCacheControl(messages, false)); + } + + private JsonNode toJsonTree(OpenAIMessage message) { + return JsonUtils.getJsonCodec() + .fromJson(JsonUtils.getJsonCodec().toJson(message), JsonNode.class); + } + } + + private JsonNode toJsonTree(Object value) { + return JsonUtils.getJsonCodec() + .fromJson(JsonUtils.getJsonCodec().toJson(value), JsonNode.class); + } } diff --git a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java index 9b909b0773..3855f0e15a 100644 --- a/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java +++ b/agentscope-extensions/agentscope-extensions-model/agentscope-extensions-model-openai/src/test/java/io/agentscope/extensions/model/openai/formatter/OpenAIResponseParserTest.java @@ -26,6 +26,8 @@ import io.agentscope.core.message.ThinkingBlock; import io.agentscope.core.message.ToolUseBlock; import io.agentscope.core.model.ChatResponse; +import io.agentscope.core.util.JsonCodec; +import io.agentscope.core.util.JsonUtils; import io.agentscope.extensions.model.openai.dto.OpenAIChoice; import io.agentscope.extensions.model.openai.dto.OpenAIError; import io.agentscope.extensions.model.openai.dto.OpenAIFunction; @@ -55,11 +57,13 @@ class OpenAIResponseParserTest { private OpenAIResponseParser parser; private Instant startTime; + private JsonCodec jsonCodec; @BeforeEach void setUp() { parser = new OpenAIResponseParser(); startTime = Instant.now(); + jsonCodec = JsonUtils.getJsonCodec(); } @Test @@ -937,6 +941,7 @@ void testParseResponseWithCachedTokens() { usage.setCompletionTokens(20); OpenAIUsage.PromptTokensDetails details = new OpenAIUsage.PromptTokensDetails(); details.setCachedTokens(80); + details.setCacheWriteTokens(12); usage.setPromptTokensDetails(details); response.setUsage(usage); @@ -958,6 +963,7 @@ void testParseResponseWithCachedTokens() { assertEquals(100, result.getUsage().getInputTokens()); assertEquals(20, result.getUsage().getOutputTokens()); assertEquals(80, result.getUsage().getCachedTokens()); + assertEquals(12, result.getUsage().getCacheCreationInputTokens()); } @Test @@ -1032,6 +1038,7 @@ void testParseChunkResponseWithCachedTokens() { usage.setCompletionTokens(50); OpenAIUsage.PromptTokensDetails details = new OpenAIUsage.PromptTokensDetails(); details.setCachedTokens(64); + details.setCacheWriteTokens(7); usage.setPromptTokensDetails(details); response.setUsage(usage); @@ -1053,6 +1060,74 @@ void testParseChunkResponseWithCachedTokens() { assertEquals(100, result.getUsage().getInputTokens()); assertEquals(50, result.getUsage().getOutputTokens()); assertEquals(64, result.getUsage().getCachedTokens()); + assertEquals(7, result.getUsage().getCacheCreationInputTokens()); + } + + @Test + @DisplayName("Should parse nested compatible-provider cache creation tokens") + void testParseResponseWithNestedCacheCreationTokens() { + String json = + """ + { + "id": "chatcmpl-cache-nested", + "object": "chat.completion", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "Answer"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 20, + "prompt_tokens_details": { + "cached_tokens": 60, + "cache_creation": { + "ephemeral_5m_input_tokens": 30, + "cache_type": "ephemeral" + } + } + } + } + """; + OpenAIResponse response = jsonCodec.fromJson(json, OpenAIResponse.class); + + ChatResponse result = parser.parseResponse(response, startTime); + + assertNotNull(result.getUsage()); + assertEquals(60, result.getUsage().getCachedTokens()); + assertEquals(30, result.getUsage().getCacheCreationInputTokens()); + } + + @Test + @DisplayName("Should parse direct compatible-provider cache creation tokens from a chunk") + void testParseChunkResponseWithDirectCacheCreationTokens() { + String json = + """ + { + "id": "chatcmpl-cache-direct", + "object": "chat.completion.chunk", + "choices": [{ + "index": 0, + "delta": {"role": "assistant", "content": ""}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 20, + "prompt_tokens_details": { + "cached_tokens": 50, + "cache_creation_input_tokens": 25 + } + } + } + """; + OpenAIResponse response = jsonCodec.fromJson(json, OpenAIResponse.class); + + ChatResponse result = parser.parseResponse(response, startTime); + + assertNotNull(result.getUsage()); + assertEquals(50, result.getUsage().getCachedTokens()); + assertEquals(25, result.getUsage().getCacheCreationInputTokens()); } @Test diff --git a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agent-protocol/src/test/java/io/agentscope/extensions/agentprotocol/AgentProtocolTaskStoreHitlTest.java b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agent-protocol/src/test/java/io/agentscope/extensions/agentprotocol/AgentProtocolTaskStoreHitlTest.java index 84cc34baeb..cbc239c378 100644 --- a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agent-protocol/src/test/java/io/agentscope/extensions/agentprotocol/AgentProtocolTaskStoreHitlTest.java +++ b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agent-protocol/src/test/java/io/agentscope/extensions/agentprotocol/AgentProtocolTaskStoreHitlTest.java @@ -111,7 +111,11 @@ void submit_pausesOnPermissionAsking_andResumeDenyCompletes() throws Exception { store.resume("hitl-1", List.of(new RemoteConfirmDecision("tc-ask", false))); - awaitCondition(() -> "success".equals(store.snapshot("hitl-1").get("status")), 5_000); + awaitCondition( + () -> + "success".equals(store.snapshot("hitl-1").get("status")) + && !store.hasSubmitContext("hitl-1"), + 5_000); Map done = store.snapshot("hitl-1"); assertEquals("success", done.get("status")); @@ -130,7 +134,11 @@ void submitContextClearedOnDirectSuccess() throws Exception { new AgentEndEvent(null))); store.submit("ok-1", "worker", "hello", Map.of("user_id", "u1", "detail", "status")); - awaitCondition(() -> "success".equals(store.snapshot("ok-1").get("status")), 5_000); + awaitCondition( + () -> + "success".equals(store.snapshot("ok-1").get("status")) + && !store.hasSubmitContext("ok-1"), + 5_000); assertFalse(store.hasSubmitContext("ok-1")); } diff --git a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/AguiStreamContext.java b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/AguiStreamContext.java index d1eb74e315..3f77759b01 100644 --- a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/AguiStreamContext.java +++ b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/AguiStreamContext.java @@ -341,30 +341,39 @@ static final class TokenUsageAccumulator { private long cumulativeInputTokens; private long cumulativeOutputTokens; private long cumulativeCachedTokens; + private long cumulativeCacheCreationInputTokens; private double cumulativeTime; TokenUsageSnapshot add(ChatUsage usage) { cumulativeInputTokens += usage.getInputTokens(); cumulativeOutputTokens += usage.getOutputTokens(); cumulativeCachedTokens += usage.getCachedTokens(); + cumulativeCacheCreationInputTokens += usage.getCacheCreationInputTokens(); cumulativeTime += usage.getTime(); return new TokenUsageSnapshot( new TokenUsage( usage.getInputTokens(), usage.getOutputTokens(), usage.getCachedTokens(), + usage.getCacheCreationInputTokens(), usage.getTime()), new TokenUsage( cumulativeInputTokens, cumulativeOutputTokens, cumulativeCachedTokens, + cumulativeCacheCreationInputTokens, cumulativeTime)); } } record TokenUsageSnapshot(TokenUsage delta, TokenUsage cumulative) {} - record TokenUsage(long inputTokens, long outputTokens, long cachedTokens, double time) { + record TokenUsage( + long inputTokens, + long outputTokens, + long cachedTokens, + long cacheCreationInputTokens, + double time) { long totalTokens() { return inputTokens + outputTokens; diff --git a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/ModelCallUsageEventConverter.java b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/ModelCallUsageEventConverter.java index 6ac475e1d8..7348bc4f91 100644 --- a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/ModelCallUsageEventConverter.java +++ b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/main/java/io/agentscope/core/agui/adapter/strategy/ModelCallUsageEventConverter.java @@ -67,6 +67,7 @@ private static Map tokenUsageMap(AguiStreamContext.TokenUsage us value.put("inputTokens", usage.inputTokens()); value.put("outputTokens", usage.outputTokens()); value.put("cachedTokens", usage.cachedTokens()); + value.put("cacheCreationInputTokens", usage.cacheCreationInputTokens()); value.put("totalTokens", usage.totalTokens()); value.put("time", usage.time()); return value; diff --git a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/test/java/io/agentscope/core/agui/adapter/AguiAgentAdapterV2Test.java b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/test/java/io/agentscope/core/agui/adapter/AguiAgentAdapterV2Test.java index 9435ec1edc..c90ff89955 100644 --- a/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/test/java/io/agentscope/core/agui/adapter/AguiAgentAdapterV2Test.java +++ b/agentscope-extensions/agentscope-extensions-protocol/agentscope-extensions-agui/src/test/java/io/agentscope/core/agui/adapter/AguiAgentAdapterV2Test.java @@ -1132,14 +1132,15 @@ void testEnabledTokenUsageEmitsCustomEventWithDeltaAndCumulativeUsage() { List events = runReActEvents( config, - new ModelCallEndEvent("reply-usage", new ChatUsage(100, 20, 40, 0.8))); + new ModelCallEndEvent( + "reply-usage", new ChatUsage(100, 20, 40, 25, 0.8))); assertEquals(List.of(AguiEventType.CUSTOM), types(events)); AguiEvent.Custom usageEvent = assertCustomEvent(events.get(0), "token_usage"); Map value = customValue(usageEvent); - assertUsage(value.get("delta"), 100L, 20L, 40L, 120L, 0.8); - assertUsage(value.get("cumulative"), 100L, 20L, 40L, 120L, 0.8); + assertUsage(value.get("delta"), 100L, 20L, 40L, 25L, 120L, 0.8); + assertUsage(value.get("cumulative"), 100L, 20L, 40L, 25L, 120L, 0.8); assertEquals(Map.of("replyId", "reply-usage"), value.get("modelCall")); } @@ -1149,8 +1150,8 @@ void testEnabledTokenUsageAccumulatesAcrossModelCalls() { List events = runReActEvents( config, - new ModelCallEndEvent("reply-1", new ChatUsage(100, 20, 40, 0.8)), - new ModelCallEndEvent("reply-2", new ChatUsage(50, 30, 10, 1.2))); + new ModelCallEndEvent("reply-1", new ChatUsage(100, 20, 40, 25, 0.8)), + new ModelCallEndEvent("reply-2", new ChatUsage(50, 30, 10, 15, 1.2))); assertEquals(List.of(AguiEventType.CUSTOM, AguiEventType.CUSTOM), types(events)); @@ -1158,9 +1159,9 @@ void testEnabledTokenUsageAccumulatesAcrossModelCalls() { customValue(assertCustomEvent(events.get(0), "token_usage")); Map secondValue = customValue(assertCustomEvent(events.get(1), "token_usage")); - assertUsage(firstValue.get("cumulative"), 100L, 20L, 40L, 120L, 0.8); - assertUsage(secondValue.get("delta"), 50L, 30L, 10L, 80L, 1.2); - assertUsage(secondValue.get("cumulative"), 150L, 50L, 50L, 200L, 2.0); + assertUsage(firstValue.get("cumulative"), 100L, 20L, 40L, 25L, 120L, 0.8); + assertUsage(secondValue.get("delta"), 50L, 30L, 10L, 15L, 80L, 1.2); + assertUsage(secondValue.get("cumulative"), 150L, 50L, 50L, 40L, 200L, 2.0); assertEquals(Map.of("replyId", "reply-2"), secondValue.get("modelCall")); } @@ -1927,6 +1928,7 @@ private static void assertUsage( long inputTokens, long outputTokens, long cachedTokens, + long cacheCreationInputTokens, long totalTokens, double time) { assertInstanceOf(Map.class, value); @@ -1934,6 +1936,7 @@ private static void assertUsage( assertEquals(inputTokens, usage.get("inputTokens")); assertEquals(outputTokens, usage.get("outputTokens")); assertEquals(cachedTokens, usage.get("cachedTokens")); + assertEquals(cacheCreationInputTokens, usage.get("cacheCreationInputTokens")); assertEquals(totalTokens, usage.get("totalTokens")); assertEquals(time, (Double) usage.get("time"), 0.000001); } diff --git a/agentscope-extensions/agentscope-extensions-studio/src/main/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregator.java b/agentscope-extensions/agentscope-extensions-studio/src/main/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregator.java index a6b7441384..4688064053 100644 --- a/agentscope-extensions/agentscope-extensions-studio/src/main/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregator.java +++ b/agentscope-extensions/agentscope-extensions-studio/src/main/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregator.java @@ -43,6 +43,8 @@ final class StreamChatResponseAggregator { // Usage: take the max value from all chunks, since providers report cumulative totals private int inputTokens; private int outputTokens; + private int cachedTokens; + private int cacheCreationInputTokens; private double time; private String finishReason; @@ -74,6 +76,9 @@ public void append(ChatResponse chunk) { if (usage != null) { inputTokens = Math.max(inputTokens, usage.getInputTokens()); outputTokens = Math.max(outputTokens, usage.getOutputTokens()); + cachedTokens = Math.max(cachedTokens, usage.getCachedTokens()); + cacheCreationInputTokens = + Math.max(cacheCreationInputTokens, usage.getCacheCreationInputTokens()); time = usage.getTime(); } @@ -96,6 +101,8 @@ public ChatResponse getResponse() { ChatUsage.builder() .inputTokens(inputTokens) .outputTokens(outputTokens) + .cachedTokens(cachedTokens) + .cacheCreationInputTokens(cacheCreationInputTokens) .time(time) .build()) .finishReason(finishReason) diff --git a/agentscope-extensions/agentscope-extensions-studio/src/test/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregatorTest.java b/agentscope-extensions/agentscope-extensions-studio/src/test/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregatorTest.java index ad42246bd4..96ffe34be9 100644 --- a/agentscope-extensions/agentscope-extensions-studio/src/test/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregatorTest.java +++ b/agentscope-extensions/agentscope-extensions-studio/src/test/java/io/agentscope/core/tracing/telemetry/StreamChatResponseAggregatorTest.java @@ -41,6 +41,8 @@ void testCumulativeUsageTakesMax() { ChatUsage.builder() .inputTokens(100) .outputTokens(i * 20) + .cachedTokens(i * 10) + .cacheCreationInputTokens(i * 5) .time(i * 0.5) .build()) .finishReason(i == 5 ? "stop" : null) @@ -51,6 +53,8 @@ void testCumulativeUsageTakesMax() { assertEquals("test-id", response.getId()); assertEquals(100, response.getUsage().getInputTokens()); assertEquals(100, response.getUsage().getOutputTokens()); + assertEquals(50, response.getUsage().getCachedTokens()); + assertEquals(25, response.getUsage().getCacheCreationInputTokens()); assertEquals("stop", response.getFinishReason()); } @@ -70,12 +74,20 @@ void testOnlyLastChunkHasUsage() { agg.append( ChatResponse.builder() .id("openai-id") - .usage(ChatUsage.builder().inputTokens(200).outputTokens(150).build()) + .usage( + ChatUsage.builder() + .inputTokens(200) + .outputTokens(150) + .cachedTokens(120) + .cacheCreationInputTokens(30) + .build()) .finishReason("stop") .build()); ChatResponse response = agg.getResponse(); assertEquals(200, response.getUsage().getInputTokens()); assertEquals(150, response.getUsage().getOutputTokens()); + assertEquals(120, response.getUsage().getCachedTokens()); + assertEquals(30, response.getUsage().getCacheCreationInputTokens()); } } diff --git a/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentDynamicHookBuilderTest.java b/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentDynamicHookBuilderTest.java index 12f6183a60..6254219827 100644 --- a/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentDynamicHookBuilderTest.java +++ b/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentDynamicHookBuilderTest.java @@ -44,11 +44,13 @@ import io.agentscope.harness.agent.middleware.SubagentsMiddleware; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; @@ -75,17 +77,24 @@ class HarnessAgentDynamicHookBuilderTest { @TempDir Path workspace; + private final List agents = new ArrayList<>(); + + @AfterEach + void closeAgents() { + agents.forEach(HarnessAgent::close); + } @Test void defaultBuild_registersDynamicSkillAndSubagentMiddlewares() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .build()); List mws = agent.getDelegate().getMiddlewares(); assertTrue( @@ -108,13 +117,14 @@ void customSkillRepository_composesWithDynamicMiddleware() throws Exception { AgentSkillRepository emptyRepo = new EmptySkillRepository(); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(emptyRepo) - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(emptyRepo) + .build()); List mws = agent.getDelegate().getMiddlewares(); assertTrue( @@ -126,13 +136,14 @@ void customSkillRepository_composesWithDynamicMiddleware() throws Exception { void disableDynamicSkills_skipsDynamicSkillMiddleware() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .disableDynamicSkills() + .build()); List mws = agent.getDelegate().getMiddlewares(); assertFalse( @@ -154,14 +165,15 @@ void disableDynamicSkills_freezesRepositoriesIntoStaticMiddleware() throws Excep null))); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(model) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(repository) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(model) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(repository) + .disableDynamicSkills() + .build()); assertEquals( 1, @@ -223,21 +235,27 @@ void disableDynamicSkills_appliesBuilderAndVisibilityFiltersToPromptAndLoader() skill("gamma", "gamma description"))); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(model) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(repository) - .enableSkills("alpha", "beta") - .enableSkillPromotionGate( - null, - (skills, ctx) -> - skills.stream() - .filter(skill -> !"alpha".equals(skill.getName())) - .toList()) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(model) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(repository) + .enableSkills("alpha", "beta") + .enableSkillPromotionGate( + null, + (skills, ctx) -> + skills.stream() + .filter( + skill -> + !"alpha" + .equals( + skill + .getName())) + .toList()) + .disableDynamicSkills() + .build()); RuntimeContext ctx = RuntimeContext.builder().sessionId("filtered-static").build(); agent.call("hello", ctx).block(); @@ -266,15 +284,16 @@ void disableDynamicSkills_skillsEnabledFalseLeavesCatalogEmpty() throws Exceptio new CountingSkillRepository(List.of(skill("disabled", "must stay hidden"))); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(model) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(repository) - .skillsEnabled(false) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(model) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(repository) + .skillsEnabled(false) + .disableDynamicSkills() + .build()); RuntimeContext ctx = RuntimeContext.builder().sessionId("no-static-skills").build(); agent.call("hello", ctx).block(); @@ -294,13 +313,14 @@ void disableDynamicSkills_keepsWorkspaceLazyResourcesLoadable() throws Exception Files.writeString(skillDir.resolve("references/guide.md"), "lazy reference body"); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .disableDynamicSkills() + .build()); RuntimeContext ctx = RuntimeContext.builder().sessionId("lazy-static").build(); agent.call("hello", ctx).block(); @@ -332,13 +352,14 @@ void getSkillRepositories_exposesComposedListInOrder() throws Exception { AgentSkillRepository custom = new EmptySkillRepository(); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .skillRepository(custom) - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .skillRepository(custom) + .build()); List repos = agent.getSkillRepositories(); assertNotNull(repos, "getSkillRepositories() must never return null"); @@ -357,13 +378,14 @@ void getSkillRepositories_exposesComposedListInOrder() throws Exception { void getSkillRepositories_isEmptyWhenNothingComposed() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .disableDynamicSkills() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .disableDynamicSkills() + .build()); assertNotNull(agent.getSkillRepositories()); } @@ -372,12 +394,13 @@ void getSkillRepositories_isEmptyWhenNothingComposed() throws Exception { void getSkillRepositories_returnsImmutableList() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .build()); List first = agent.getSkillRepositories(); List second = agent.getSkillRepositories(); @@ -395,13 +418,14 @@ void getSkillRepositories_returnsImmutableList() throws Exception { void disableDynamicSubagents_fallsBackToStaticSubagentsMiddleware() throws Exception { Files.createDirectories(workspace); HarnessAgent agent = - HarnessAgent.builder() - .name("t") - .model(stubModel("ok")) - .workspace(workspace) - .abstractFilesystem(new LocalFilesystem(workspace)) - .disableDynamicSubagents() - .build(); + track( + HarnessAgent.builder() + .name("t") + .model(stubModel("ok")) + .workspace(workspace) + .abstractFilesystem(new LocalFilesystem(workspace)) + .disableDynamicSubagents() + .build()); List mws = agent.getDelegate().getMiddlewares(); assertFalse( @@ -424,6 +448,11 @@ void disableDynamicSubagents_fallsBackToStaticSubagentsMiddleware() throws Excep .orElseThrow(); } + private HarnessAgent track(HarnessAgent agent) { + agents.add(agent); + return agent; + } + private static AgentSkill skill(String name, String description) { return new AgentSkill(name, description, "# " + name, null); } diff --git a/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentIntegrationExampleTest.java b/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentIntegrationExampleTest.java index 3731ec3984..e77d8f5662 100644 --- a/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentIntegrationExampleTest.java +++ b/agentscope-harness/src/test/java/io/agentscope/harness/agent/HarnessAgentIntegrationExampleTest.java @@ -117,56 +117,62 @@ void example_fullWorkspace_singleTurn_seesSessionSubagentsAndWorkspaceContext() """); Model model = stubModel("integration-main-reply"); - HarnessAgent agent = + try (HarnessAgent agent = HarnessAgent.builder() .name("integration-main") .description("integration example main agent") .sysPrompt("You are the main agent in an integration test.") .model(model) .workspace(workspace) - .build(); + .build()) { - Msg reply = - agent.call( - userText("Run the integration scenario."), - RuntimeContext.builder().sessionId("integration-session-1").build()) - .block(); + Msg reply = + agent.call( + userText("Run the integration scenario."), + RuntimeContext.builder() + .sessionId("integration-session-1") + .build()) + .block(); - assertTrue(reply.getTextContent().contains("integration-main-reply")); + assertTrue(reply.getTextContent().contains("integration-main-reply")); - @SuppressWarnings("unchecked") - ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); - verify(model, atLeast(1)).stream(captor.capture(), any(), any()); - String combined = - captor.getAllValues().stream() - .map(HarnessAgentIntegrationExampleTest::joinAllText) - .filter(s -> s.contains("## AgentStateStore Context")) - .findFirst() - .orElse(""); + @SuppressWarnings("unchecked") + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(model, atLeast(1)).stream(captor.capture(), any(), any()); + String combined = + captor.getAllValues().stream() + .map(HarnessAgentIntegrationExampleTest::joinAllText) + .filter(s -> s.contains("## AgentStateStore Context")) + .findFirst() + .orElse(""); - assertTrue( - combined.contains("## AgentStateStore Context"), - "AgentStateStore context should be injected; model saw: " - + captor.getAllValues().stream() - .map(HarnessAgentIntegrationExampleTest::joinAllText) - .toList()); - // Current WorkspaceContextHook uses markdown (##) guidance + XML blocks - assertTrue( - combined.contains("## Domain Knowledge") || combined.contains("## Workspace"), - "expected workspace guidance sections"); - assertTrue(combined.contains("`AGENTS.md`") || combined.contains("agents_context")); - assertTrue( - combined.contains(agentsPersona), "AGENTS.md should appear under workspace hook"); - assertTrue(combined.contains("memory_context") || combined.contains("MEMORY.md")); - assertTrue(combined.contains(memoryNote)); - assertTrue( - combined.contains("domain_knowledge_context") || combined.contains("KNOWLEDGE.md")); - assertTrue(combined.contains(knowledgeLine)); - assertTrue( - combined.contains("## Subagents") || combined.contains("Subagents:"), - "subagent list should be injected into the system prompt"); - assertTrue(combined.contains("`" + helperSubId + "`")); - assertTrue(combined.contains("`" + reviewerSubId + "`")); + assertTrue( + combined.contains("## AgentStateStore Context"), + "AgentStateStore context should be injected; model saw: " + + captor.getAllValues().stream() + .map(HarnessAgentIntegrationExampleTest::joinAllText) + .toList()); + // Current WorkspaceContextHook uses markdown (##) guidance + XML + // blocks + assertTrue( + combined.contains("## Domain Knowledge") || combined.contains("## Workspace"), + "expected workspace guidance sections"); + assertTrue(combined.contains("`AGENTS.md`") || combined.contains("agents_context")); + assertTrue( + combined.contains(agentsPersona), + "AGENTS.md should appear under workspace hook"); + assertTrue(combined.contains("memory_context") || combined.contains("MEMORY.md")); + assertTrue(combined.contains(memoryNote)); + assertTrue( + combined.contains("domain_knowledge_context") + || combined.contains("KNOWLEDGE.md")); + assertTrue(combined.contains(knowledgeLine)); + assertTrue( + combined.contains("## Subagents") || combined.contains("Subagents:"), + "subagent list should be injected into the system prompt"); + assertTrue(combined.contains("`" + helperSubId + "`")); + assertTrue(combined.contains("`" + reviewerSubId + "`")); + } } /** @@ -217,23 +223,25 @@ void example_subagentFactory_markdownSpec_runsChildHarnessAgent() throws Excepti .orElseThrow( () -> new AssertionError("missing subagent entry: " + childId)); - Agent sub = child.factory().create(RuntimeContext.empty()); - assertInstanceOf(HarnessAgent.class, sub); - assertEquals(childId, sub.getName()); + try (HarnessAgent sub = + assertInstanceOf( + HarnessAgent.class, child.factory().create(RuntimeContext.empty()))) { + assertEquals(childId, sub.getName()); - Msg subReply = sub.call(List.of(userText("task for child"))).block(); - assertTrue(subReply.getTextContent().contains("integration-child-reply")); + Msg subReply = sub.call(List.of(userText("task for child"))).block(); + assertTrue(subReply.getTextContent().contains("integration-child-reply")); - @SuppressWarnings("unchecked") - ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); - verify(model, atLeast(1)).stream(captor.capture(), any(), any()); - boolean childSysSeen = - captor.getAllValues().stream() - .map(HarnessAgentIntegrationExampleTest::joinAllText) - .anyMatch(s -> s.contains("INTEGRATION_CHILD_SYS")); - assertTrue( - childSysSeen, - "child HarnessAgent should use spec sysPrompt in its system prompt bundle"); + @SuppressWarnings("unchecked") + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(model, atLeast(1)).stream(captor.capture(), any(), any()); + boolean childSysSeen = + captor.getAllValues().stream() + .map(HarnessAgentIntegrationExampleTest::joinAllText) + .anyMatch(s -> s.contains("INTEGRATION_CHILD_SYS")); + assertTrue( + childSysSeen, + "child HarnessAgent should use spec sysPrompt in its system prompt bundle"); + } } private static Msg userText(String text) { diff --git a/docs/v2/en/blogs/usecases/aidc-logistics.md b/docs/v2/en/blogs/usecases/aidc-logistics.md index a1ffb14f75..ec31137b70 100644 --- a/docs/v2/en/blogs/usecases/aidc-logistics.md +++ b/docs/v2/en/blogs/usecases/aidc-logistics.md @@ -291,7 +291,7 @@ As the bottom-most layer of the architecture, its core is interacting with the L **5. Zero-intrusion integration of observability and advanced reasoning capabilities** - Rule: Trace instrumentation, Prompt caching, and tool invocation enhancements are automatically completed at the model layer; business code needs no manual handling. -- Source analysis: `ChatModelBase.stream()` automatically wraps calls via `TracerRegistry.get().callModel()`; when cacheControl=true, `OpenAIBaseFormatter.applyCacheControl()` automatically adds cache markers; toolChoice and parallelToolCalls parameters directly control tool behavior. +- Source analysis: `ChatModelBase.stream()` automatically wraps calls via `TracerRegistry.get().callModel()`; official OpenAI endpoints use server-managed automatic prompt caching, while manually marked messages are translated to provider-native explicit breakpoints when supported; toolChoice and parallelToolCalls parameters directly control tool behavior. ![Model layer architecture I](https://mmbiz.qpic.cn/sz_mmbiz_png/bvDbzNRia8j2vyuibOsbQibMibMjVQOymQcVxoTOX2VY8z2jHJ6XdAN5A5FCfD8zWgxt5Abdt2sGI95MLD7eJFMF6pKYduAc8jvaMYS0VfMWw8c/640?wx_fmt=png&from=appmsg) diff --git a/docs/v2/en/blogs/usecases/logistics.md b/docs/v2/en/blogs/usecases/logistics.md index a1ffb14f75..ec31137b70 100644 --- a/docs/v2/en/blogs/usecases/logistics.md +++ b/docs/v2/en/blogs/usecases/logistics.md @@ -291,7 +291,7 @@ As the bottom-most layer of the architecture, its core is interacting with the L **5. Zero-intrusion integration of observability and advanced reasoning capabilities** - Rule: Trace instrumentation, Prompt caching, and tool invocation enhancements are automatically completed at the model layer; business code needs no manual handling. -- Source analysis: `ChatModelBase.stream()` automatically wraps calls via `TracerRegistry.get().callModel()`; when cacheControl=true, `OpenAIBaseFormatter.applyCacheControl()` automatically adds cache markers; toolChoice and parallelToolCalls parameters directly control tool behavior. +- Source analysis: `ChatModelBase.stream()` automatically wraps calls via `TracerRegistry.get().callModel()`; official OpenAI endpoints use server-managed automatic prompt caching, while manually marked messages are translated to provider-native explicit breakpoints when supported; toolChoice and parallelToolCalls parameters directly control tool behavior. ![Model layer architecture I](https://mmbiz.qpic.cn/sz_mmbiz_png/bvDbzNRia8j2vyuibOsbQibMibMjVQOymQcVxoTOX2VY8z2jHJ6XdAN5A5FCfD8zWgxt5Abdt2sGI95MLD7eJFMF6pKYduAc8jvaMYS0VfMWw8c/640?wx_fmt=png&from=appmsg) diff --git a/docs/v2/en/docs/building-blocks/model.md b/docs/v2/en/docs/building-blocks/model.md index c34fb54882..c98f718156 100644 --- a/docs/v2/en/docs/building-blocks/model.md +++ b/docs/v2/en/docs/building-blocks/model.md @@ -59,6 +59,34 @@ Other provider artifacts follow the same pattern: `agentscope-extensions-model-o ``` +## Prompt caching + +Prompt caching is provider-specific. `GenerateOptions.cacheControl(true)` asks a supporting adapter to use its automatic strategy, while `MessageMetadataKeys.CACHE_CONTROL` marks a manual cache boundary. A manual boundary remains effective when the automatic strategy is disabled or unset. + +```java +GenerateOptions automaticCache = + GenerateOptions.builder().cacheControl(true).build(); + +Msg stableContext = + Msg.builder() + .role(MsgRole.USER) + .textContent("Stable context to cache") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); +``` + +The same option does not have identical wire semantics for every provider: + +| Provider | Automatic/provider-managed behavior | Explicit behavior | +|----------|-------------------------------------|-------------------| +| OpenAI | Supported models cache eligible prefixes automatically. `cacheControl(true)` does not add an artificial breakpoint. | A marked `Msg` is converted to OpenAI's native explicit breakpoint protocol on official OpenAI endpoints. Internal markers are removed for unknown OpenAI-compatible endpoints. | +| DashScope | `cacheControl(true)` marks the first system message and the last cacheable conversation message. | A marked `Msg` becomes a native content-block cache marker on DashScope native and recognized DashScope-compatible endpoints. | +| Anthropic | `cacheControl(true)` enables Anthropic automatic prompt caching. | A marked `Msg` becomes an Anthropic content-block breakpoint. Automatic caching consumes one of the provider's four available breakpoints. | +| Gemini | Gemini 2.5+ manages implicit caching. The boolean option does not create a cache resource. | Reference an existing explicit resource with `additionalBodyParam("cachedContent", resourceName)`; resource CRUD and TTL remain in the Google SDK. Message markers do not create resources. | +| Ollama | No prompt-cache request or usage protocol is exposed by this adapter. | Not supported. | + +Setting `cacheControl(false)` only disables an AgentScope automatic strategy; it cannot disable provider-managed implicit caching. `ChatUsage.inputTokens` is the total input count. `cachedTokens` and `cacheCreationInputTokens` are subsets of that input count and must not be added again when computing total tokens. + ## Choose a creation path ### String model id diff --git a/docs/v2/en/integration/model/gemini.md b/docs/v2/en/integration/model/gemini.md index 68015f3cd3..1a5c9d9373 100644 --- a/docs/v2/en/integration/model/gemini.md +++ b/docs/v2/en/integration/model/gemini.md @@ -37,6 +37,40 @@ GeminiChatModel model = GeminiChatModel.builder() .build(); ``` +## Prompt caching + +### Implicit caching + +Gemini 2.5 and newer models enable implicit caching on the provider side. No request field is required. Setting `GenerateOptions.cacheControl(true)` does not create a Gemini cache resource, and setting it to `false` does not disable provider-managed implicit caching. + +Likewise, adding `MessageMetadataKeys.CACHE_CONTROL` to a `Msg` does not create or manage a Gemini `cachedContents` resource. Gemini explicit caching has a separate, provider-managed resource lifecycle. + +### Reference explicit cached content + +Create and manage the cache resource with the Google Gen AI SDK, then pass its resource name through `GenerateOptions`: + +```java +GenerateOptions options = GenerateOptions.builder() + .additionalBodyParam("cachedContent", cache.name().orElseThrow()) + .build(); + +model.stream(dynamicMessages, tools, options); +``` + +The resource name is passed to `GenerateContentConfig.cachedContent(...)` unchanged. Use a cache created for the same model and, for Vertex AI, the same project and location. The resource must also be unexpired. + +Cached content is a prefix to the request. Pass only the dynamic suffix in `dynamicMessages`; do not resend messages already stored in the cache. If the request uses a system instruction, tool declarations, or tool configuration, include them when creating the cache resource. Gemini does not allow these fields to be sent again in a `generateContent` request that references cached content. Tool schemas may still be supplied to AgentScope for local tool execution, but they must match the declarations stored in the cache. + +The Google SDK owns cache creation, TTL updates, lookup, and deletion. AgentScope only references the supplied resource name during generation. + +### Cache usage + +Gemini cache usage is normalized into `ChatUsage` as follows: + +- `inputTokens` is the total effective prompt size and already includes cached tokens. +- `cachedTokens` is the cached subset reported by Gemini's `cachedContentTokenCount`. +- `cacheCreationInputTokens` is `0` for `generateContent`; cache creation is a separate resource operation. + ## Spring Boot Spring Boot applications can use the Gemini starter: diff --git a/docs/v2/zh/blogs/usecases/aidc-logistics.md b/docs/v2/zh/blogs/usecases/aidc-logistics.md index dccb1f79e3..5b1c3e397c 100644 --- a/docs/v2/zh/blogs/usecases/aidc-logistics.md +++ b/docs/v2/zh/blogs/usecases/aidc-logistics.md @@ -291,7 +291,7 @@ return "NEED_CONFIRM: 金额过大，请确认"; **5. 可观测性与高级推理能力零侵入集成** - 规则：Trace 埋点、Prompt 缓存、工具调用增强等能力在模型层自动完成，业务代码无需手动处理。 -- 源码分析：`ChatModelBase.stream()` 通过 `TracerRegistry.get().callModel()` 自动包裹调用；cacheControl=true 时 `OpenAIBaseFormatter.applyCacheControl()` 自动添加缓存标记；toolChoice 与 parallelToolCalls 参数直接控制工具行为。 +- 源码分析：`ChatModelBase.stream()` 通过 `TracerRegistry.get().callModel()` 自动包裹调用；OpenAI 官方端点使用服务端管理的自动 Prompt 缓存，手动标记的消息则会在支持的端点上转换为提供商原生的显式断点；toolChoice 与 parallelToolCalls 参数直接控制工具行为。 ![模型层架构一](https://mmbiz.qpic.cn/sz_mmbiz_png/bvDbzNRia8j2vyuibOsbQibMibMjVQOymQcVxoTOX2VY8z2jHJ6XdAN5A5FCfD8zWgxt5Abdt2sGI95MLD7eJFMF6pKYduAc8jvaMYS0VfMWw8c/640?wx_fmt=png&from=appmsg) diff --git a/docs/v2/zh/blogs/usecases/logistics.md b/docs/v2/zh/blogs/usecases/logistics.md index 4bb3ef373a..9617bd8758 100644 --- a/docs/v2/zh/blogs/usecases/logistics.md +++ b/docs/v2/zh/blogs/usecases/logistics.md @@ -291,7 +291,7 @@ Thread.sleep(1000); // 占用宝贵线程资源 **5. 可观测性与高级推理能力零侵入集成** - 规则：Trace 埋点、Prompt 缓存、工具调用增强等能力在模型层自动完成，业务代码无需手动处理。 -- 源码分析：`ChatModelBase.stream()` 通过 `TracerRegistry.get().callModel()` 自动包裹调用；cacheControl=true 时 `OpenAIBaseFormatter.applyCacheControl()` 自动添加缓存标记；toolChoice 与 parallelToolCalls 参数直接控制工具行为。 +- 源码分析：`ChatModelBase.stream()` 通过 `TracerRegistry.get().callModel()` 自动包裹调用；OpenAI 官方端点使用服务端管理的自动 Prompt 缓存，手动标记的消息则会在支持的端点上转换为提供商原生的显式断点；toolChoice 与 parallelToolCalls 参数直接控制工具行为。 ![模型层架构一](https://mmbiz.qpic.cn/sz_mmbiz_png/bvDbzNRia8j2vyuibOsbQibMibMjVQOymQcVxoTOX2VY8z2jHJ6XdAN5A5FCfD8zWgxt5Abdt2sGI95MLD7eJFMF6pKYduAc8jvaMYS0VfMWw8c/640?wx_fmt=png&from=appmsg) diff --git a/docs/v2/zh/docs/building-blocks/model.md b/docs/v2/zh/docs/building-blocks/model.md index af207671d8..499ff9f6b7 100644 --- a/docs/v2/zh/docs/building-blocks/model.md +++ b/docs/v2/zh/docs/building-blocks/model.md @@ -59,6 +59,34 @@ CredentialBase/ ``` +## Prompt 缓存 + +Prompt 缓存是模型提供商专属能力。`GenerateOptions.cacheControl(true)` 会请求支持该能力的适配器使用自动策略；`MessageMetadataKeys.CACHE_CONTROL` 用于手动标记缓存边界。即使自动策略未设置或已关闭，手动边界仍然生效。 + +```java +GenerateOptions automaticCache = + GenerateOptions.builder().cacheControl(true).build(); + +Msg stableContext = + Msg.builder() + .role(MsgRole.USER) + .textContent("需要缓存的稳定上下文") + .metadata(Map.of(MessageMetadataKeys.CACHE_CONTROL, true)) + .build(); +``` + +同一选项在不同提供商的线上协议含义不完全相同： + +| 提供商 | 自动/服务端管理行为 | 显式行为 | +|----------|----------------------|----------| +| OpenAI | 受支持的模型会自动缓存符合条件的前缀；`cacheControl(true)` 不会人为新增断点。 | 官方 OpenAI 端点会把已标记的 `Msg` 转成 OpenAI 原生显式断点协议；未知 OpenAI-compatible 端点会移除内部标记。 | +| DashScope | `cacheControl(true)` 会标记第一条 system 消息和最后一条可缓存对话消息。 | 在 DashScope 原生端点和已识别的 DashScope-compatible 端点上，已标记 `Msg` 会转成内容块缓存标记。 | +| Anthropic | `cacheControl(true)` 启用 Anthropic 自动 prompt 缓存。 | 已标记 `Msg` 会转成 Anthropic 内容块断点；自动缓存会占用提供商最多四个断点中的一个。 | +| Gemini | Gemini 2.5+ 由服务端管理隐式缓存，布尔选项不会创建缓存资源。 | 通过 `additionalBodyParam("cachedContent", resourceName)` 引用已存在的显式资源；资源 CRUD 和 TTL 由 Google SDK 管理，消息标记不会创建资源。 | +| Ollama | 当前适配器未暴露 prompt 缓存请求或 usage 协议。 | 不支持。 | + +设置 `cacheControl(false)` 只会关闭 AgentScope 的自动策略，无法关闭提供商管理的隐式缓存。`ChatUsage.inputTokens` 是输入 token 总数，`cachedTokens` 和 `cacheCreationInputTokens` 是其子集，计算总 token 时不能重复相加。 + ## 选择模型创建方式 ### 字符串 model id diff --git a/docs/v2/zh/integration/model/gemini.md b/docs/v2/zh/integration/model/gemini.md index 20e2da57c1..1fbd676d7c 100644 --- a/docs/v2/zh/integration/model/gemini.md +++ b/docs/v2/zh/integration/model/gemini.md @@ -37,6 +37,40 @@ GeminiChatModel model = GeminiChatModel.builder() .build(); ``` +## Prompt 缓存 + +### 隐式缓存 + +Gemini 2.5 及更新模型由服务端默认启用隐式缓存，请求无需增加额外字段。设置 `GenerateOptions.cacheControl(true)` 不会创建 Gemini 缓存资源，设为 `false` 也不会关闭服务端管理的隐式缓存。 + +同样，为 `Msg` 添加 `MessageMetadataKeys.CACHE_CONTROL` 不会创建或管理 Gemini `cachedContents` 资源。Gemini 显式缓存拥有独立的服务端资源生命周期。 + +### 引用显式缓存资源 + +先使用 Google Gen AI SDK 创建和管理缓存资源，再通过 `GenerateOptions` 传入它的资源名称： + +```java +GenerateOptions options = GenerateOptions.builder() + .additionalBodyParam("cachedContent", cache.name().orElseThrow()) + .build(); + +model.stream(dynamicMessages, tools, options); +``` + +资源名称会原样传给 `GenerateContentConfig.cachedContent(...)`。必须使用由相同模型创建且未过期的缓存；对于 Vertex AI，project 和 location 也必须一致。 + +缓存内容是当前请求的前缀。`dynamicMessages` 中只传递动态后缀，不要重复传入已写入缓存的消息。如果请求需要 system instruction、工具声明或 tool config，应在创建缓存资源时写入。Gemini 不允许在引用缓存的 `generateContent` 请求中重复发送这些字段。AgentScope 仍可以接收工具 schema 用于本地工具执行，但它必须与缓存资源中的声明一致。 + +缓存的创建、TTL 更新、查询和删除由 Google SDK 管理。AgentScope 在生成请求中只引用用户传入的资源名称。 + +### 缓存用量 + +Gemini 的缓存用量会按以下规则归一化到 `ChatUsage`： + +- `inputTokens` 是完整的有效输入 token 数，已包含缓存 token。 +- `cachedTokens` 是 Gemini `cachedContentTokenCount` 报告的缓存子集。 +- `cacheCreationInputTokens` 在 `generateContent` 中为 `0`；缓存创建是独立的资源操作。 + ## Spring Boot Spring Boot 应用可以使用 Gemini starter：