From 8d9b709a570eaa9b7f898a769e855c520a4ffa8b Mon Sep 17 00:00:00 2001
From: Yufeng He <40085740+he-yufeng@users.noreply.github.com>
Date: Wed, 15 Jul 2026 11:24:15 +0800
Subject: [PATCH] Python: fix compaction token count inflating non-ASCII text

TokenBudgetComposedStrategy estimates tokens by feeding a JSON-serialized
message to the tokenizer, but _serialize_message() used ensure_ascii=True.
That escapes non-ASCII text into \uXXXX sequences, so CJK and other
non-Latin content is token-counted as the escape sequences rather than the
characters the model actually sees, inflating the estimate (~1.6x for mixed
Japanese, more for pure CJK) and skewing compaction/token-budget decisions.

Serialize with ensure_ascii=False, matching the ensure_ascii=False already
used elsewhere in this module. Only affects token estimation; the serialized
string is never stored or transmitted.
---
 .../core/agent_framework/_compaction.py         |  4 +++-
 .../packages/core/tests/core/test_compaction.py | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/python/packages/core/agent_framework/_compaction.py b/python/packages/core/agent_framework/_compaction.py
index 2cbd413c9a1..e6457663df9 100644
--- a/python/packages/core/agent_framework/_compaction.py
+++ b/python/packages/core/agent_framework/_compaction.py
@@ -515,7 +515,9 @@ def _serialize_message(message: Message) -> str:
         "message_id": message.message_id,
         "contents": serialized_contents,
     }
-    return json.dumps(payload, ensure_ascii=True, sort_keys=True, default=str)
+    # ensure_ascii=False so non-ASCII text (e.g. CJK) is token-counted as the
+    # actual characters the model sees, not as inflated ``\uXXXX`` escapes.
+    return json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str)
 
 
 def annotate_token_counts(
diff --git a/python/packages/core/tests/core/test_compaction.py b/python/packages/core/tests/core/test_compaction.py
index d070655dfd3..25ab21a9de5 100644
--- a/python/packages/core/tests/core/test_compaction.py
+++ b/python/packages/core/tests/core/test_compaction.py
@@ -35,6 +35,7 @@
     included_token_count,
 )
 from agent_framework._compaction import (
+    _serialize_message,
     append_compaction_message,
     extend_compaction_messages,
 )
@@ -1293,3 +1294,19 @@ def test_context_window_strategy_validates_thresholds() -> None:
             tool_eviction_threshold=0.8,
             truncation_threshold=0.5,
         )
+
+
+def test_serialize_message_preserves_non_ascii_for_token_count() -> None:
+    """Non-ASCII text is token-counted as the characters the model sees, not as
+    inflated ``\\uXXXX`` escapes, so the token estimate isn't skewed (#7022)."""
+    text = "こんにちは、元気ですか"
+    message = Message(role="user", contents=[text])
+    tokenizer = CharacterEstimatorTokenizer()
+
+    serialized = _serialize_message(message)
+    # the same payload as it would serialize with ensure_ascii=True
+    escaped = serialized.encode("ascii", "backslashreplace").decode("ascii")
+
+    assert text in serialized
+    assert "\\u3053" not in serialized
+    assert tokenizer.count_tokens(serialized) < tokenizer.count_tokens(escaped)