From 7338bbbbc781353d46e18f0c7e9057c6e22dce11 Mon Sep 17 00:00:00 2001
From: Ishaan <ishaangupta0408@gmail.com>
Date: Wed, 15 Jul 2026 21:28:51 +0000
Subject: [PATCH] fix: use Unicode-aware tokenizer in ROUGE-1 scorer to support
 non-English eval

The `rouge_score` library's default tokenizer strips all non-ASCII characters using the regex `[^a-z0-9]+`, which causes any non-English text (Thai, Chinese, Arabic, etc.) to be reduced to empty token lists. As a result, the ROUGE-1 f-measure always returned 0.0 for non-English text, even when the candidate and reference strings were identical.

Signed-off-by: Ishaan <ishaangupta0408@gmail.com>
---
 src/google/adk/dependencies/rouge_scorer.py    |  1 +
 .../adk/evaluation/final_response_match_v1.py  | 18 +++++++++++++++++-
 .../evaluation/test_final_response_match_v1.py | 18 ++++++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/src/google/adk/dependencies/rouge_scorer.py b/src/google/adk/dependencies/rouge_scorer.py
index 622a190ab73..e7e6cfcd8c0 100644
--- a/src/google/adk/dependencies/rouge_scorer.py
+++ b/src/google/adk/dependencies/rouge_scorer.py
@@ -15,3 +15,4 @@
 from __future__ import annotations
 
 from rouge_score import rouge_scorer as rouge_scorer
+from rouge_score import tokenizers as tokenizers
diff --git a/src/google/adk/evaluation/final_response_match_v1.py b/src/google/adk/evaluation/final_response_match_v1.py
index 794d3a6d641..ad23e6953e6 100644
--- a/src/google/adk/evaluation/final_response_match_v1.py
+++ b/src/google/adk/evaluation/final_response_match_v1.py
@@ -20,6 +20,7 @@
 from typing_extensions import override
 
 from ..dependencies.rouge_scorer import rouge_scorer
+from ..dependencies.rouge_scorer import tokenizers
 from .eval_case import ConversationScenario
 from .eval_case import Invocation
 from .eval_metrics import EvalMetric
@@ -29,6 +30,19 @@
 from .evaluator import PerInvocationResult
 
 
+class _UnicodeFriendlyTokenizer(tokenizers.Tokenizer):
+  """A tokenizer that splits on whitespace and preserves Unicode characters.
+
+  The default rouge_score tokenizer strips all non-ASCII characters, causing
+  non-English text (e.g., Thai, Chinese, Arabic) to tokenize to empty strings
+  and receive a score of 0 regardless of content.  This tokenizer uses simple
+  whitespace splitting and lowercasing, which correctly handles any language.
+  """
+
+  def tokenize(self, text: str) -> list[str]:
+    return text.lower().split()
+
+
 class RougeEvaluator(Evaluator):
   """Evaluates if agent's final response matches a golden/expected final response using Rouge_1 metric.
 
@@ -110,7 +124,9 @@ def _calculate_rouge_1_scores(candidate: str, reference: str):
   Returns:
       A dictionary containing the ROUGE-1 precision, recall, and f-measure.
   """
-  scorer = rouge_scorer.RougeScorer(["rouge1"], use_stemmer=True)
+  scorer = rouge_scorer.RougeScorer(
+      ["rouge1"], tokenizer=_UnicodeFriendlyTokenizer()
+  )
 
   # The score method returns a dictionary where keys are the ROUGE types
   # and values are Score objects (tuples) with precision, recall, and fmeasure.
diff --git a/tests/unittests/evaluation/test_final_response_match_v1.py b/tests/unittests/evaluation/test_final_response_match_v1.py
index b60c4b46be9..b1f5d7ccf4d 100644
--- a/tests/unittests/evaluation/test_final_response_match_v1.py
+++ b/tests/unittests/evaluation/test_final_response_match_v1.py
@@ -87,6 +87,24 @@ def test_calculate_rouge_1_scores():
   assert rouge_1_score.fmeasure == pytest.approx(8 / 11)
 
 
+def test_calculate_rouge_1_scores_non_english_identical():
+  """Non-English text that is identical should score 1.0, not 0."""
+  candidate = "สวัสดี"
+  reference = "สวัสดี"
+  rouge_1_score = _calculate_rouge_1_scores(candidate, reference)
+  assert rouge_1_score.precision == pytest.approx(1.0)
+  assert rouge_1_score.recall == pytest.approx(1.0)
+  assert rouge_1_score.fmeasure == pytest.approx(1.0)
+
+
+def test_calculate_rouge_1_scores_non_english_partial_overlap():
+  """Non-English text with partial overlap should return a partial score."""
+  candidate = "สวัสดี โลก"
+  reference = "สวัสดี ทุกคน"
+  rouge_1_score = _calculate_rouge_1_scores(candidate, reference)
+  assert rouge_1_score.fmeasure == pytest.approx(0.5)
+
+
 @pytest.mark.parametrize(
     "candidates, references, expected_score, expected_status",
     [