diff --git a/python/packages/core/agent_framework/_compaction.py b/python/packages/core/agent_framework/_compaction.py index 2cbd413c9a..e6457663df 100644 --- a/python/packages/core/agent_framework/_compaction.py +++ b/python/packages/core/agent_framework/_compaction.py @@ -515,7 +515,9 @@ def _serialize_message(message: Message) -> str: "message_id": message.message_id, "contents": serialized_contents, } - return json.dumps(payload, ensure_ascii=True, sort_keys=True, default=str) + # ensure_ascii=False so non-ASCII text (e.g. CJK) is token-counted as the + # actual characters the model sees, not as inflated ``\uXXXX`` escapes. + return json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str) def annotate_token_counts( diff --git a/python/packages/core/tests/core/test_compaction.py b/python/packages/core/tests/core/test_compaction.py index d070655dfd..25ab21a9de 100644 --- a/python/packages/core/tests/core/test_compaction.py +++ b/python/packages/core/tests/core/test_compaction.py @@ -35,6 +35,7 @@ included_token_count, ) from agent_framework._compaction import ( + _serialize_message, append_compaction_message, extend_compaction_messages, ) @@ -1293,3 +1294,19 @@ def test_context_window_strategy_validates_thresholds() -> None: tool_eviction_threshold=0.8, truncation_threshold=0.5, ) + + +def test_serialize_message_preserves_non_ascii_for_token_count() -> None: + """Non-ASCII text is token-counted as the characters the model sees, not as + inflated ``\\uXXXX`` escapes, so the token estimate isn't skewed (#7022).""" + text = "こんにちは、元気ですか" + message = Message(role="user", contents=[text]) + tokenizer = CharacterEstimatorTokenizer() + + serialized = _serialize_message(message) + # the same payload as it would serialize with ensure_ascii=True + escaped = serialized.encode("ascii", "backslashreplace").decode("ascii") + + assert text in serialized + assert "\\u3053" not in serialized + assert tokenizer.count_tokens(serialized) < tokenizer.count_tokens(escaped)