From 73e24ac6769fc55ab53a28273812d7657cbd2133 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 20 Oct 2026 13:37:19 -0400 Subject: [PATCH 1/7] docs(financial_services): describe the discoverable-tool gate by what it does The tools.yaml header that gen-assets writes, the probe's docstring and README, and the spec comments state the rule: a discoverable tool is callable only after a document describing it has been retrieved. The 14 shipped tools.yaml headers carry the new header. Signed-off-by: Yev Meyer --- src/usersim/asset_gen/financial_services/pipeline.py | 7 +++---- .../financial_services/region_spec/en_US.yaml | 5 ++--- src/usersim/asset_gen/financial_services/spec.py | 10 +++++----- .../en_IN/bank/chandrika_bank/tools.yaml | 7 +++---- .../en_IN/brokerage/tarang_invest/tools.yaml | 7 +++---- .../en_IN/insurer/abhaya_insurance/tools.yaml | 7 +++---- .../en_IN/nbfc/kalash_finserv/tools.yaml | 7 +++---- .../en_IN/neobank/paysetu/tools.yaml | 7 +++---- .../en_US/bank/northwind_bank/tools.yaml | 7 +++---- .../en_US/brokerage/solstice_invest/tools.yaml | 7 +++---- .../evergreen_retirement/tools.yaml | 7 +++---- .../en_US/wealth_manager/meridian_wealth/tools.yaml | 7 +++---- .../hi_Deva_IN/bank/chandrika_bank/tools.yaml | 7 +++---- .../hi_Deva_IN/brokerage/tarang_invest/tools.yaml | 7 +++---- .../hi_Deva_IN/insurer/abhaya_insurance/tools.yaml | 7 +++---- .../hi_Deva_IN/nbfc/kalash_finserv/tools.yaml | 7 +++---- .../hi_Deva_IN/neobank/paysetu/tools.yaml | 7 +++---- src/usersim/engine/probes/financial_services/README.md | 6 +++--- src/usersim/engine/probes/financial_services/tools.py | 4 ++-- tests/test_asset_gen_region_spec.py | 6 ------ 20 files changed, 57 insertions(+), 79 deletions(-) diff --git a/src/usersim/asset_gen/financial_services/pipeline.py b/src/usersim/asset_gen/financial_services/pipeline.py index 70fb53f..aa776fe 100644 --- a/src/usersim/asset_gen/financial_services/pipeline.py +++ b/src/usersim/asset_gen/financial_services/pipeline.py @@ -933,10 +933,9 @@ def institution_id(self) -> str: "# parameters here. Their real schema (signature + args) lives in each tool's\n" "# discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its\n" "# arguments by RETRIEVING that doc via kb_search, then calls it through the\n" - "# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas\n" - "# here would defeat that discovery challenge. Only the always-offered\n" - "# primitives (kb_search / verify_identity) carry a schema; side_effect_class\n" - "# is always set (the scorer uses it).\n" + "# call_tool gate. Putting schemas here would defeat that discovery challenge.\n" + "# Only the always-offered primitives (kb_search / verify_identity) carry a\n" + "# schema; side_effect_class is always set (the scorer uses it).\n" ) diff --git a/src/usersim/asset_gen/financial_services/region_spec/en_US.yaml b/src/usersim/asset_gen/financial_services/region_spec/en_US.yaml index b046e34..8d5c753 100644 --- a/src/usersim/asset_gen/financial_services/region_spec/en_US.yaml +++ b/src/usersim/asset_gen/financial_services/region_spec/en_US.yaml @@ -1,9 +1,8 @@ # Structured region_spec for en_US — the generation INPUT (data, not code). # Validated against asset_gen/financial_services/spec.py. The DD pipeline expands # this into the committed per-institution assets/financial_services/en_US/ bank. -# A locale is a SET of institutions; each spans one or more domains. Fictional -# brands only; do not reuse tau-Banking's fictional bank or product names -# (contamination-avoidance policy). +# A locale is a SET of institutions; each spans one or more domains. Brands are +# fictional. schema_version: v0.1 region: US locale: en_US diff --git a/src/usersim/asset_gen/financial_services/spec.py b/src/usersim/asset_gen/financial_services/spec.py index 31846f4..d9572cf 100644 --- a/src/usersim/asset_gen/financial_services/spec.py +++ b/src/usersim/asset_gen/financial_services/spec.py @@ -198,9 +198,9 @@ class Tool(BaseModel): # A region spec that explicitly declares a tool of the same name wins (no dup). COMMON_UNIVERSAL_OPERATIONS: tuple[tuple[str, SideEffectClass], ...] = ( # Read primitive that lists a VERIFIED customer's accounts (id, type, last4, - # balance). Analogous to tau-Banking's get_user_details: it lets the agent - # resolve a last-4 the customer gives into the account_id that every - # state-changing tool requires, instead of fabricating an id or deflecting. + # balance). It lets the agent resolve a last-4 the customer gives into the + # account_id that every state-changing tool requires, instead of fabricating + # an id or deflecting. ("get_accounts", "read_only"), ("open_account", "state_changing"), ("close_account", "irreversible"), @@ -213,8 +213,8 @@ class Tool(BaseModel): # Deposit-taking institutions (bank / neobank / credit_union) share a servicing # set; brokerages / robo-advisors share a trading + cash set. Most of these are # never gold for a task — they exist as realistic, gold-ADJACENT tool docs so -# retrieval must discriminate the right procedure among many similar ones (the -# tau-Knowledge difficulty), and so the discoverable-tool gate is non-trivial. +# retrieval must discriminate the right procedure among many similar ones, and +# so the discoverable-tool gate is non-trivial. _DEPOSIT_OPS: tuple[tuple[str, SideEffectClass], ...] = ( ("transfer_funds", "state_changing"), ("order_replacement_card", "state_changing"), diff --git a/src/usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml b/src/usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml index ea9d5ca..c4429f1 100644 --- a/src/usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml +++ b/src/usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml b/src/usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml index 05849c8..b36ff53 100644 --- a/src/usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml +++ b/src/usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml b/src/usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml index 1053555..00ab0a8 100644 --- a/src/usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml +++ b/src/usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml b/src/usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml index 8ab659a..11d4636 100644 --- a/src/usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml +++ b/src/usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml b/src/usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml index 306f6e2..8a1f750 100644 --- a/src/usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml +++ b/src/usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml b/src/usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml index f88da1b..1dc0c52 100644 --- a/src/usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml +++ b/src/usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml b/src/usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml index 0577e95..4999bca 100644 --- a/src/usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml +++ b/src/usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml b/src/usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml index 33fae28..7a04c2c 100644 --- a/src/usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml +++ b/src/usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml b/src/usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml index ed06fa9..9fac49b 100644 --- a/src/usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml +++ b/src/usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml b/src/usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml index ea9d5ca..c4429f1 100644 --- a/src/usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml +++ b/src/usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml b/src/usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml index 05849c8..b36ff53 100644 --- a/src/usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml +++ b/src/usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml b/src/usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml index 1053555..00ab0a8 100644 --- a/src/usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml +++ b/src/usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml b/src/usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml index 8ab659a..11d4636 100644 --- a/src/usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml +++ b/src/usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml b/src/usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml index 306f6e2..8a1f750 100644 --- a/src/usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml +++ b/src/usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml @@ -2,10 +2,9 @@ # parameters here. Their real schema (signature + args) lives in each tool's # discoverable_tool_doc in corpus.yaml -- the assistant learns a tool + its # arguments by RETRIEVING that doc via kb_search, then calls it through the -# call_tool gate (the tau-Banking discoverable-tool mechanic). Putting schemas -# here would defeat that discovery challenge. Only the always-offered -# primitives (kb_search / verify_identity) carry a schema; side_effect_class -# is always set (the scorer uses it). +# call_tool gate. Putting schemas here would defeat that discovery challenge. +# Only the always-offered primitives (kb_search / verify_identity) carry a +# schema; side_effect_class is always set (the scorer uses it). schema_version: v0.1 tools: - name: kb_search diff --git a/src/usersim/engine/probes/financial_services/README.md b/src/usersim/engine/probes/financial_services/README.md index 36f237e..88e0b7b 100644 --- a/src/usersim/engine/probes/financial_services/README.md +++ b/src/usersim/engine/probes/financial_services/README.md @@ -94,8 +94,8 @@ flowchart TD - **Knowledge-base search** (`kb_search`) returns whole documents scoped to the institution. -- **Discoverable tools** follow the tau-Banking mechanic: a tool is only callable - once a document describing it has been retrieved. +- **Discoverable tools**: a tool is only callable once a document describing it + has been retrieved. - **Account state** is a lightweight in-memory record mutated by tool calls, so within-conversation responses stay consistent. - A `verifiable` conversation is only considered complete once the assistant has @@ -457,7 +457,7 @@ in-memory account record rather than a full mutable database; conversations are multilingual across many institutions and domains rather than English-only against a single bank; the open-ended `dynamic` tier is generated from an authored taxonomy; and every institution is a distinct fictional brand authored -per locale (no reuse of tau-Banking's names or products). +per locale. ## Deferred diff --git a/src/usersim/engine/probes/financial_services/tools.py b/src/usersim/engine/probes/financial_services/tools.py index ca4e6f2..8855769 100644 --- a/src/usersim/engine/probes/financial_services/tools.py +++ b/src/usersim/engine/probes/financial_services/tools.py @@ -7,8 +7,8 @@ ``verify_identity``, ...) plus a single ``call_tool(tool_name, arguments)`` gate for DISCOVERABLE tools. A discoverable tool is only usable once its documentation has been retrieved (its name appears in a retrieved document's -``mentions_tools``) — the tau-Banking discoverable-tool mechanic. This module is -pure helpers; the probe's ``after_assistant_turn`` tool loop owns state. +``mentions_tools``). This module is pure helpers; the probe's +``after_assistant_turn`` tool loop owns state. """ from __future__ import annotations diff --git a/tests/test_asset_gen_region_spec.py b/tests/test_asset_gen_region_spec.py index 03d149f..d8d4e40 100644 --- a/tests/test_asset_gen_region_spec.py +++ b/tests/test_asset_gen_region_spec.py @@ -128,12 +128,6 @@ def test_common_operations_do_not_override_explicit_tools(): assert open_tools[0].description == "custom" # explicit declaration wins -def test_no_rho_bank_contamination(): - """Contamination policy: no reuse of tau-Banking's fictional institution.""" - raw = _EN_US.read_text().lower() - assert "rho-bank" not in raw and "rho bank" not in raw - - def test_validator_rejects_unknown_institution_type(): bad = _raw(_EN_US) bad["institutions"][0]["type"] = "not_a_type" From 0d7b2eedf85e38ccfaeac9dfc7d30b7e226ad07c Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 20 Oct 2026 13:37:23 -0400 Subject: [PATCH 2/7] fix(asset_gen): keep discoverable tool descriptions out of tools.yaml A description in the region spec now steers the generated documentation of a discoverable tool, while tools.yaml keeps it empty, so the assistant still learns the tool only by retrieving its document. Signed-off-by: Yev Meyer --- .../asset_gen/financial_services/pipeline.py | 4 ++- tests/test_asset_gen_pipeline.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/usersim/asset_gen/financial_services/pipeline.py b/src/usersim/asset_gen/financial_services/pipeline.py index aa776fe..2639fb7 100644 --- a/src/usersim/asset_gen/financial_services/pipeline.py +++ b/src/usersim/asset_gen/financial_services/pipeline.py @@ -1076,9 +1076,11 @@ def _institution_meta(inst: Institution) -> dict[str, Any]: def _tool_dict(tool: Tool) -> dict[str, Any]: + # A discoverable tool is learned from its retrieved doc, so tools.yaml keeps + # its description empty even when the spec supplies one to steer that doc. return { "name": tool.name, - "description": tool.description, + "description": "" if tool.discoverable else tool.description, "discoverable": tool.discoverable, "side_effect_class": tool.side_effect_class, "parameters": tool.parameters, diff --git a/tests/test_asset_gen_pipeline.py b/tests/test_asset_gen_pipeline.py index bc798b4..1ba4dbd 100644 --- a/tests/test_asset_gen_pipeline.py +++ b/tests/test_asset_gen_pipeline.py @@ -78,6 +78,38 @@ def test_backfill_primitive_tools_fills_empty_primitive_schemas(): assert out["file_transaction_dispute"]["description"] == "" +def test_a_discoverable_tool_description_steers_its_doc_but_stays_out_of_tools_yaml(): + """A spec description reaches the shared clusters that write a discoverable + tool's doc, while tools.yaml ships it empty: the assistant learns the tool + only by retrieving that doc.""" + import json + + import pandas as pd + import yaml + + from usersim.asset_gen.financial_services.pipeline import _tool_dict + from usersim.asset_gen.financial_services.spec import validate_region_spec + + raw = yaml.safe_load(_REGION_SPEC.read_text(encoding="utf-8")) + bank = raw["institutions"][0] + described = "Block the card until the customer asks to unblock it." + bank["tool_taxonomy"] = [ + {**t, "description": described} if t["name"] == "freeze_card" else t for t in bank["tool_taxonomy"] + ] + spec = validate_region_spec(raw) + tool = next(t for t in spec.institutions[0].tool_taxonomy if t.name == "freeze_card") + assert tool.discoverable and tool.description == described + + brief_df = pd.DataFrame( + [{"institution_id": i.id, "institution_brief": {"positioning": "p"}} for i in spec.institutions] + ) + rows = explode_to_cluster_rows(brief_df, spec, docs_per_product=10, max_docs_per_cluster=10) + shared = rows[(rows.institution_id == spec.institutions[0].id) & (rows.cluster_kind == "shared")] + focus_tools = {t["name"]: t for t in json.loads(shared.iloc[0]["focus_json"])["tools"]} + assert focus_tools["freeze_card"]["description"] == described + assert _tool_dict(tool)["description"] == "" + + def test_build_institution_seed_one_row_per_institution(): spec = load_region_spec(_REGION_SPEC) seed = build_institution_seed(spec) From e5ccca16cf448fe49c07885d059d4dd9f7562239 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 20 Oct 2026 13:37:25 -0400 Subject: [PATCH 3/7] feat(asset_gen): let an institution omit common operations it does not offer omit_common_operations removes injected operations an institution does not offer, such as card tools at a building-savings bank. Naming an operation its type is not given, or both declaring and omitting a tool, is rejected. Signed-off-by: Yev Meyer --- .../asset_gen/financial_services/spec.py | 18 ++++++++++++++-- .../assets/financial_services/SCHEMA.md | 6 +++++- tests/test_asset_gen_region_spec.py | 21 +++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/usersim/asset_gen/financial_services/spec.py b/src/usersim/asset_gen/financial_services/spec.py index d9572cf..b50c99e 100644 --- a/src/usersim/asset_gen/financial_services/spec.py +++ b/src/usersim/asset_gen/financial_services/spec.py @@ -309,6 +309,10 @@ class Institution(BaseModel): domains: list[Domain] = Field(min_length=1) products: list[Product] = Field(default_factory=list) tool_taxonomy: list[Tool] = Field(min_length=1) + #: Common operations of this institution's type that it does not offer, by + #: name: a building-savings bank issues no cards, so it omits the card + #: operations every ``bank`` would otherwise be given. + omit_common_operations: list[str] = Field(default_factory=list) task_templates: list[TaskTemplate] = Field(min_length=1) @model_validator(mode="after") @@ -316,10 +320,20 @@ def _inject_common_operations(self) -> "Institution": """Merge the common-operations layer (account lifecycle + servicing + per-type ops) into the tool taxonomy. Runs FIRST so injected tools count as in-scope for the gold-tool check below; an explicitly-declared tool of - the same name is left untouched (no duplicate).""" + the same name is left untouched (no duplicate), and a name listed in + ``omit_common_operations`` is not injected.""" + common = {name for name, _ in common_operations_for_type(self.type)} + omitted = set(self.omit_common_operations) + if unknown := sorted(omitted - common): + raise ValueError( + f"{self.id}: omit_common_operations lists {unknown}, which a {self.type!r} is not " + f"given; its common operations are {sorted(common)}" + ) existing = {t.name for t in self.tool_taxonomy} + if both := sorted(omitted & existing): + raise ValueError(f"{self.id}: {both} both declared in tool_taxonomy and omitted") for name, side_effect in common_operations_for_type(self.type): - if name not in existing: + if name not in existing and name not in omitted: self.tool_taxonomy.append(Tool(name=name, discoverable=True, side_effect_class=side_effect)) existing.add(name) return self diff --git a/src/usersim/engine/assets/financial_services/SCHEMA.md b/src/usersim/engine/assets/financial_services/SCHEMA.md index ec1cca2..eebaad7 100644 --- a/src/usersim/engine/assets/financial_services/SCHEMA.md +++ b/src/usersim/engine/assets/financial_services/SCHEMA.md @@ -242,7 +242,11 @@ institution offers (`open_account`, `close_account`, `update_contact_info`, `update_policy`), injected by `asset_gen/financial_services/spec.py`. Most are never gold: they are realistic, gold-ADJACENT tool docs that make retrieval (and the discoverable-tool gate) non-trivial, and they stay consistent across -locales. +locales. An institution that does not offer one of them lists it in its region +spec's `omit_common_operations` (a building-savings bank issues no cards), and +one whose local procedure differs declares the tool with a `description` that +steers its generated documentation; `tools.yaml` still ships that description +empty. ## tasks.yaml diff --git a/tests/test_asset_gen_region_spec.py b/tests/test_asset_gen_region_spec.py index d8d4e40..0061291 100644 --- a/tests/test_asset_gen_region_spec.py +++ b/tests/test_asset_gen_region_spec.py @@ -128,6 +128,27 @@ def test_common_operations_do_not_override_explicit_tools(): assert open_tools[0].description == "custom" # explicit declaration wins +def test_an_institution_can_omit_common_operations_it_does_not_offer(): + """A building-savings bank issues no cards, so it must be able to drop the + card operations its type injects; a name its type is not given, or a tool + both declared and omitted, is an authoring error.""" + raw = _raw(_EN_US) + bank = raw["institutions"][0] + bank["omit_common_operations"] = ["order_replacement_card", "activate_card"] + names = {t.name for t in validate_region_spec(raw).institutions[0].tool_taxonomy} + assert not names & {"order_replacement_card", "activate_card"} + assert {"transfer_funds", "open_account"} <= names + + bank["omit_common_operations"] = ["file_claim"] + with pytest.raises(RegionSpecError, match="file_claim"): + validate_region_spec(raw) + + bank["tool_taxonomy"].append({"name": "stop_payment", "discoverable": True, "side_effect_class": "state_changing"}) + bank["omit_common_operations"] = ["stop_payment"] + with pytest.raises(RegionSpecError, match="stop_payment"): + validate_region_spec(raw) + + def test_validator_rejects_unknown_institution_type(): bad = _raw(_EN_US) bad["institutions"][0]["type"] = "not_a_type" From 1069153a766b49e176de665ea52b9d9917ef7c1a Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 20 Oct 2026 13:37:28 -0400 Subject: [PATCH 4/7] fix(asset_gen): write amounts and rates in the locale's notation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each typed value reaches the generation prompts with a display form in the locale's notation (7,90 € in France, £7.90 in the UK), and the prompts and the cluster judge ask for that form. format_amount and fot_percent in finance_bank.py build that form, and the probe's money formatting shares them. validate-assets gains a number_notation check that fails a decimal-comma corpus on any English decimal before a unit. Signed-off-by: Yev Meyer --- .../asset_gen/financial_services/pipeline.py | 32 +++++++- .../asset_gen/financial_services/prompts.py | 18 +++-- .../asset_gen/financial_services/validate.py | 28 +++++++ src/usersim/engine/core/finance_bank.py | 75 +++++++++++++------ .../engine/probes/test_financial_services.py | 10 +++ tests/test_asset_gen_pipeline.py | 32 ++++++++ tests/test_asset_gen_validate.py | 27 +++++++ 7 files changed, 190 insertions(+), 32 deletions(-) diff --git a/src/usersim/asset_gen/financial_services/pipeline.py b/src/usersim/asset_gen/financial_services/pipeline.py index 2639fb7..f2c1388 100644 --- a/src/usersim/asset_gen/financial_services/pipeline.py +++ b/src/usersim/asset_gen/financial_services/pipeline.py @@ -550,6 +550,32 @@ def _batches(items: list[Any], cap: int) -> Iterator[list[Any]]: # ── Phase 1: institution seed + brief config ──────────────────────────── +def products_for_prompt(products: list[Product], spec: RegionSpec) -> list[dict[str, Any]]: + """Each product with a ``display`` rendering of every typed value in the + locale's own notation (``7,90 €``, ``15,9 %``), which the prompts ask the + model to write instead of the raw ``value``.""" + from usersim.engine.core.finance_bank import format_amount, format_percent + + number_format = spec.number_format or spec.locale + out: list[dict[str, Any]] = [] + for product in products: + dumped = product.model_dump() + for var in dumped["variables"].values(): + if var["type"] == "money": + var["display"] = format_amount( + var["value"], + currency_symbol=spec.currency_symbol, + number_format=number_format, + drop_zero_cents=True, + ) + elif var["type"] == "percent": + var["display"] = format_percent(var["value"], number_format=number_format) + else: + var["display"] = str(var["value"]) + out.append(dumped) + return out + + def build_institution_seed(spec: RegionSpec): """One seed row per institution for the Phase-1 brief (pure).""" import pandas as pd @@ -568,7 +594,7 @@ def build_institution_seed(spec: RegionSpec): "brand_voice": inst.brand_voice, "language": spec.language, "currency_symbol": spec.currency_symbol, - "products_json": json.dumps([p.model_dump() for p in inst.products], ensure_ascii=False), + "products_json": json.dumps(products_for_prompt(inst.products, spec), ensure_ascii=False), "regulators_json": json.dumps(regulators, ensure_ascii=False), # Market-specific emphasis: region override else region-neutral # default, resolved here so the prompt stays free of any one @@ -667,8 +693,8 @@ def explode_to_cluster_rows( "sibling_doc_index": json.dumps(sibling_index, ensure_ascii=False), } - for product in inst.products: - focus = json.dumps(product.model_dump(), ensure_ascii=False) + for product, prompt_product in zip(inst.products, products_for_prompt(inst.products, spec)): + focus = json.dumps(prompt_product, ensure_ascii=False) for batch in _batches(product_plans[product.id], max_docs_per_cluster): rows.append( { diff --git a/src/usersim/asset_gen/financial_services/prompts.py b/src/usersim/asset_gen/financial_services/prompts.py index 96729db..611fe54 100644 --- a/src/usersim/asset_gen/financial_services/prompts.py +++ b/src/usersim/asset_gen/financial_services/prompts.py @@ -32,7 +32,9 @@ form in {{ language }} prose, and keep it identical everywhere){% endif %} (id: {{ institution_id }}). Author a concise internal BRIEF that every downstream document will follow, so the institution's voice and policies stay consistent. Write ALL prose in {{ language }}; keep product ids and tool/function -names verbatim. Use the {{ currency_symbol }} symbol for money. +names verbatim. Use the {{ currency_symbol }} symbol for money, and write every number +the way {{ language }} writes it: each typed value carries a `display` form, so quote +that form rather than the raw `value`. Products offered (authoritative typed values -- reference exactly, never invent): {{ products_json }} @@ -115,7 +117,9 @@ def concept_guidance_for(institution_type: str, overrides: dict[str, str] | None {{ institution_brief }} Write ALL body prose in {{ language }}. Keep tool/function identifiers, product ids, and -doc_keys VERBATIM. Use the {{ currency_symbol }} symbol for money. +doc_keys VERBATIM. Use the {{ currency_symbol }} symbol for money, and write every number +the way {{ language }} writes it (its decimal mark and digit grouping), never another +language's notation. TITLES ARE GIVEN, NOT AUTHORED: copy each entry's `topic` into its `title` EXACTLY as provided -- already in {{ language }} -- and do not translate, shorten, re-word, or re-punctuate it. The same string is what every sibling document is told to cite when it @@ -124,8 +128,9 @@ def concept_guidance_for(institution_type: str, overrides: dict[str, str] | None {{ language }} prose -- never re-translate it and never vary it between documents, because readers (and retrieval) rely on one product having one name across the knowledge base. NUMERIC FIDELITY IS THE #1 REQUIREMENT: reproduce every fee, rate, yield, balance, limit, -threshold, duration, date, and percentage from the focus values EXACTLY as given (same digits, -symbol, %, and units) -- never round, approximate, paraphrase, bucket, omit, or invent a number. +threshold, duration, date, and percentage from the focus values EXACTLY, written as its +`display` form (the same digits, with {{ language }}'s decimal mark, digit grouping, symbol, +%, and units) -- never round, approximate, paraphrase, bucket, omit, or invent a number. The VALUE is verbatim; its NAME is not. The focus keys are machine field names, so write what each one means in ordinary prose and NEVER print the key itself: say "an annual interest rate of 0%", not "an interest_rate_annual of 0%"; "a one-time processing fee of {{ currency_symbol }}499", @@ -153,7 +158,7 @@ def concept_guidance_for(institution_type: str, overrides: dict[str, str] | None Requirements: - Faithfulness (HIGHEST PRIORITY -- overrides brevity): every numeric value present in the - focus values MUST appear VERBATIM in the document(s) where it belongs. If stating all exact + focus values MUST appear, in its `display` form, in the document(s) where it belongs. If stating all exact values conflicts with the length target, keep the exact values and cut prose instead -- a document that rounds, buckets, or omits a value is a failure even if it reads well. - Length: aim for each body's per-entry `length` target -- a SOFT target that yields to @@ -265,6 +270,9 @@ def concept_guidance_for(institution_type: str, overrides: dict[str, str] | None other numbers in the documents (dates, illustrative examples, regulatory citations) are EXPECTED and must NOT lower the faithfulness score. If this context lists no product values (e.g. an institution-wide shared set), there is nothing to contradict -> score 4. +Each value carries a `display` form, its notation in the documents' language: a value +written that way is faithful, while the right digits in another language's notation (a +decimal point where the language uses a decimal comma) are a notation error. {{ focus_json }} Generated document set: diff --git a/src/usersim/asset_gen/financial_services/validate.py b/src/usersim/asset_gen/financial_services/validate.py index 77cef77..691b53e 100644 --- a/src/usersim/asset_gen/financial_services/validate.py +++ b/src/usersim/asset_gen/financial_services/validate.py @@ -277,6 +277,33 @@ def _check_numeric_faithfulness(spec: RegionSpec | None, insts: dict[str, _InstD rep.add("numeric_faithfulness", inst.id, PASS, f"{product.id}: all typed values present") +def _check_number_notation(spec: RegionSpec | None, insts: dict[str, _InstData], rep: ValidationReport) -> None: + """A decimal-comma locale must not quote amounts or rates in English notation. + + ``7.9 €`` or ``15.9 %`` in a French or German document is a visible error, and + the numeric check cannot see it: it only asks whether the locale's own form + appears somewhere in a product's documents, so a corpus can pass while most + of its decimals are written the English way. + """ + if spec is None: + return + if _decimal_comma_group_separators(spec.number_format or spec.locale) is None: + return + units = "|".join(re.escape(u) for u in ("%", spec.currency_symbol, spec.currency) if u) + english = re.compile(rf"(? None: """Flag customer-facing prose that prints a spec FIELD KEY. @@ -675,6 +702,7 @@ def validate_bank(bank_dir: str | Path, region_spec: RegionSpec | None = None) - _check_structural(bank_dir, insts, rep) _check_coverage(insts, rep) _check_numeric_faithfulness(region_spec, insts, rep) + _check_number_notation(region_spec, insts, rep) _check_variable_name_leak(region_spec, insts, rep) _check_tool_name_leak(insts, rep) _check_doc_key_leak(insts, rep) diff --git a/src/usersim/engine/core/finance_bank.py b/src/usersim/engine/core/finance_bank.py index 344cac6..6fa5bc8 100644 --- a/src/usersim/engine/core/finance_bank.py +++ b/src/usersim/engine/core/finance_bank.py @@ -149,6 +149,49 @@ def to_openai_tool(self) -> dict[str, Any]: _DEFAULT_DATE_FORMAT = "%d/%m/%Y" +def format_amount(amount: Any, *, currency_symbol: str, number_format: str, drop_zero_cents: bool = False) -> str: + """``amount`` in a region's currency and number format. + + ``$1,234.50`` in the US, ``₹1,23,456.00`` in India, ``1 234,50 €`` in France + and ``1.234,50 €`` in Germany. ``drop_zero_cents`` writes a whole amount + without decimals (``22 950 €``). A value that isn't a number comes back as + given, after the symbol. + """ + try: + value = Decimal(str(amount)) + except InvalidOperation: + return f"{currency_symbol}{amount}" + if not value.is_finite(): + return f"{currency_symbol}{amount}" + sign = "-" if value < 0 else "" + whole, cents = f"{abs(value):.2f}".split(".") + if number_format.endswith("_IN"): + grouped, decimal_mark, symbol_after = _indian_grouped(whole), ".", False + else: + group, decimal_mark, symbol_after = _MONEY_STYLES.get(number_format.split("_")[0].lower(), (",", ".", False)) + grouped = f"{int(whole):,}".replace(",", group) + number = grouped if drop_zero_cents and cents == "00" else f"{grouped}{decimal_mark}{cents}" + if symbol_after: + return f"{sign}{number} {currency_symbol}".rstrip() + return f"{sign}{currency_symbol}{number}" + + +def format_percent(value: Any, *, number_format: str) -> str: + """``value`` as a percentage in a region's number format, without trailing + zeros: ``15.9%`` in English, ``15,9 %`` in French and German.""" + try: + number = Decimal(str(value)) + except InvalidOperation: + return f"{value}%" + if not number.is_finite(): + return f"{value}%" + text = format(number.normalize(), "f") + style = _MONEY_STYLES.get(number_format.split("_")[0].lower()) + if style is None: + return f"{text}%" + return f"{text.replace('.', style[1])} %" + + def _indian_grouped(digits: str) -> str: """``"123456"`` -> ``"1,23,456"``: the last three digits, then pairs.""" head, tail = digits[:-3], digits[-3:] @@ -444,30 +487,14 @@ def domain_account_label(self, domain: str) -> str: return DEFAULT_DOMAIN_ACCOUNT_LABELS.get(domain, "account") def format_money(self, amount: Any) -> str: - """``amount`` in this region's currency and number format. - - ``$1,234.50`` in the US, ``₹1,23,456.00`` in India, ``1 234,50 €`` in - France and ``1.234,50 €`` in Germany, from ``region_meta``'s - ``currency_symbol`` and ``number_format`` (else the locale). A value that - isn't a number comes back as given, after the symbol. - """ - symbol = str(self.region_meta.get("currency_symbol") or "") - try: - value = Decimal(str(amount)) - except InvalidOperation: - return f"{symbol}{amount}" - if not value.is_finite(): - return f"{symbol}{amount}" - sign = "-" if value < 0 else "" - whole, cents = f"{abs(value):.2f}".split(".") - number_format = str(self.region_meta.get("number_format") or self.locale) - if number_format.endswith("_IN"): - return f"{sign}{symbol}{_indian_grouped(whole)}.{cents}" - group, decimal_mark, symbol_after = _MONEY_STYLES.get(number_format.split("_")[0].lower(), (",", ".", False)) - number = f"{int(whole):,}".replace(",", group) + decimal_mark + cents - if symbol_after: - return f"{sign}{number} {symbol}".rstrip() - return f"{sign}{symbol}{number}" + """``amount`` in this region's currency and number format, from + ``region_meta``'s ``currency_symbol`` and ``number_format`` (else the + locale); see :func:`format_amount`.""" + return format_amount( + amount, + currency_symbol=str(self.region_meta.get("currency_symbol") or ""), + number_format=str(self.region_meta.get("number_format") or self.locale), + ) def format_date(self, value: date) -> str: """``value`` as this region writes dates. diff --git a/tests/engine/probes/test_financial_services.py b/tests/engine/probes/test_financial_services.py index 3039aca..ad37dfc 100644 --- a/tests/engine/probes/test_financial_services.py +++ b/tests/engine/probes/test_financial_services.py @@ -2225,6 +2225,16 @@ def test_money_follows_the_region(self): assert us.format_money("-12") == "-$12.00" assert us.format_money("n/a") == "$n/a" + def test_percentages_and_whole_amounts_follow_the_region(self): + from usersim.engine.core.finance_bank import format_amount, format_percent + + assert format_percent(15.9, number_format="fr_FR") == "15,9 %" + assert format_percent(1.70, number_format="de_DE") == "1,7 %" + assert format_percent(50, number_format="en_GB") == "50%" + assert format_amount(22950, currency_symbol="€", number_format="fr_FR", drop_zero_cents=True) == "22 950 €" + assert format_amount(7.9, currency_symbol="€", number_format="de_DE", drop_zero_cents=True) == "7,90 €" + assert format_amount(7.9, currency_symbol="£", number_format="en_GB") == "£7.90" + def test_dates_follow_the_region(self): day = datetime.date(1980, 3, 27) us = load_finance_bank_for_locale("en_US") diff --git a/tests/test_asset_gen_pipeline.py b/tests/test_asset_gen_pipeline.py index 1ba4dbd..b320ebf 100644 --- a/tests/test_asset_gen_pipeline.py +++ b/tests/test_asset_gen_pipeline.py @@ -78,6 +78,38 @@ def test_backfill_primitive_tools_fills_empty_primitive_schemas(): assert out["file_transaction_dispute"]["description"] == "" +def test_prompt_values_carry_the_locale_notation(): + """The prompts hand the model each typed value with its rendering in the + locale's notation, so a decimal-comma corpus does not copy ``7.9`` from JSON.""" + import copy + + import yaml + + from usersim.asset_gen.financial_services.pipeline import products_for_prompt + from usersim.asset_gen.financial_services.spec import validate_region_spec + + raw = yaml.safe_load(_REGION_SPEC.read_text(encoding="utf-8")) + product = raw["institutions"][0]["products"][0] + product["variables"] = { + "monthly_fee": {"type": "money", "value": 7.9}, + "ceiling": {"type": "money", "value": 22950}, + "rate": {"type": "percent", "value": 15.9}, + "term_months": {"type": "int", "value": 48}, + } + us = validate_region_spec(copy.deepcopy(raw)) + raw.update(locale="fr_FR", region="FR", currency="EUR", currency_symbol="€", number_format="fr_FR") + france = validate_region_spec(raw) + + def displays(spec): + variables = products_for_prompt(spec.institutions[0].products, spec)[0]["variables"] + return {name: var["display"] for name, var in variables.items()} + + assert displays(france) == {"monthly_fee": "7,90 €", "ceiling": "22 950 €", "rate": "15,9 %", "term_months": "48"} + assert displays(us) == {"monthly_fee": "$7.90", "ceiling": "$22,950", "rate": "15.9%", "term_months": "48"} + seed = build_institution_seed(france) + assert '"display": "7,90 €"' in seed.iloc[0]["products_json"] + + def test_a_discoverable_tool_description_steers_its_doc_but_stays_out_of_tools_yaml(): """A spec description reaches the shared clusters that write a discoverable tool's doc, while tools.yaml ships it empty: the assistant learns the tool diff --git a/tests/test_asset_gen_validate.py b/tests/test_asset_gen_validate.py index f2c2d05..ddec5b0 100644 --- a/tests/test_asset_gen_validate.py +++ b/tests/test_asset_gen_validate.py @@ -274,6 +274,33 @@ def test_validate_bank_reads_values_in_the_spec_number_format(tmp_path: Path): assert _status(validate_bank(english, spec), "numeric_faithfulness", "testbank") == FAIL +def test_a_decimal_comma_corpus_fails_on_any_english_decimal(tmp_path: Path): + """The numeric check passes once the locale's form appears anywhere, so a + corpus with mostly English decimals would slip through without this gate.""" + import copy + + spec_dict = copy.deepcopy(_SPEC_DICT) + spec_dict.update(locale="fr_FR", region="FR", currency="EUR", currency_symbol="€", language="French") + spec_dict["number_format"] = "fr_FR" + spec_dict["institutions"][0]["products"][0]["variables"]["overdraft_fee"]["value"] = 7.9 + spec = RegionSpec.model_validate(spec_dict) + + mixed = _write_bank( + tmp_path / "mixed", + "fr_FR", + [_doc("D1", "Les frais s'élèvent à 7,90 € par mois."), _doc("D2", "Un taux de 15.9% s'applique.")], + ) + report = validate_bank(mixed, spec) + assert _status(report, "numeric_faithfulness", "testbank") == PASS + assert _status(report, "number_notation", "testbank") == FAIL + + french = _write_bank(tmp_path / "fr", "fr_FR", [_doc("D1", "Les frais s'élèvent à 7,90 € ; un taux de 15,9 %.")]) + assert _status(validate_bank(french, spec), "number_notation", "testbank") == PASS + + english = _write_bank(tmp_path / "en", "en_US", [_doc("D1", "The overdraft fee is $34 per item, or 15.9%.")]) + assert _status(validate_bank(english, _spec()), "number_notation", "testbank") is None + + def test_variable_name_leak_flags_field_keys_in_prose(tmp_path: Path): """A doc that prints a spec FIELD KEY reads like an unfilled template. From c962543b25cc7c71b89384b237f2c62a0c4ec5ea Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 20 Oct 2026 13:37:30 -0400 Subject: [PATCH 5/7] fix(asset_gen): stop flagging one-word field keys as leaked names A key such as excess is also the ordinary word, so the variable-name check matches only snake_case keys, as the tool-name check already does. Signed-off-by: Yev Meyer --- src/usersim/asset_gen/financial_services/validate.py | 4 +++- tests/test_asset_gen_validate.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/usersim/asset_gen/financial_services/validate.py b/src/usersim/asset_gen/financial_services/validate.py index 691b53e..46f5f25 100644 --- a/src/usersim/asset_gen/financial_services/validate.py +++ b/src/usersim/asset_gen/financial_services/validate.py @@ -320,7 +320,9 @@ def _check_variable_name_leak(spec: RegionSpec | None, insts: dict[str, _InstDat idata = insts.get(inst.id) if idata is None: continue - keys = {k for p in inst.products for k in p.variables} + # A one-word key such as ``excess`` is also an ordinary word, which an exact + # token match cannot tell apart from prose. + keys = {k for p in inst.products for k in p.variables if "_" in k} if not keys: continue pattern = re.compile(r"\b(" + "|".join(sorted((re.escape(k) for k in keys), key=len, reverse=True)) + r")\b") diff --git a/tests/test_asset_gen_validate.py b/tests/test_asset_gen_validate.py index ddec5b0..9d28d17 100644 --- a/tests/test_asset_gen_validate.py +++ b/tests/test_asset_gen_validate.py @@ -319,6 +319,17 @@ def test_variable_name_leak_flags_field_keys_in_prose(tmp_path: Path): assert _status(rep, "variable_name_leak", "testbank") == PASS +def test_a_one_word_field_key_is_not_mistaken_for_a_leak(tmp_path: Path): + """``excess`` is a field key and also the everyday insurance word for it.""" + import copy + + spec_dict = copy.deepcopy(_SPEC_DICT) + spec_dict["institutions"][0]["products"][0]["variables"]["excess"] = {"type": "money", "value": 99} + bank = _write_bank(tmp_path, "en_US", [_doc("D1", "Our overdraft fee is $34 per item; the excess is $99.")]) + rep = validate_bank(bank, RegionSpec.model_validate(spec_dict)) + assert _status(rep, "variable_name_leak", "testbank") == PASS + + # ── tool names and doc_keys must not appear in customer-facing prose ───────── From 55697d49c8a8cccc9485d095c9a962a1d56fce69 Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 20 Oct 2026 13:37:33 -0400 Subject: [PATCH 6/7] test(asset_gen): check the labels of every region spec that authors them The label-coverage test discovers each region spec with variable_labels instead of listing locales. Signed-off-by: Yev Meyer --- tests/test_asset_gen_pipeline.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_asset_gen_pipeline.py b/tests/test_asset_gen_pipeline.py index b320ebf..43d0b79 100644 --- a/tests/test_asset_gen_pipeline.py +++ b/tests/test_asset_gen_pipeline.py @@ -39,6 +39,11 @@ def _asset_gen_dir() -> Path: _REGION_SPEC = _asset_gen_dir() / "financial_services" / "region_spec" / "en_US.yaml" +#: Region specs that author their own title vocabulary. +_LABELLED_SPECS = sorted( + p.stem for p in _REGION_SPEC.parent.glob("*.yaml") if "variable_labels:" in p.read_text(encoding="utf-8") +) + def test_backfill_primitive_tools_fills_empty_primitive_schemas(): """Region specs list kb_search / verify_identity by name only; serialization @@ -639,11 +644,12 @@ def devanagari_fraction(text: str) -> float: latin = {t for t in titles if devanagari_fraction(t) < 0.25} assert all("टूल संदर्भ" in t or "का उपयोग कब करें" in t for t in latin) - def test_hindi_labels_cover_every_variable_and_domain(self): + @pytest.mark.parametrize("locale", _LABELLED_SPECS) + def test_localized_labels_cover_every_variable_and_domain(self, locale: str): """An unmapped name falls back to the machine field name, which lands a raw ``below_mab_charge`` / ``retail_banking`` in reader-facing prose. Adding a product to the region model must not silently reintroduce that.""" - spec = load_region_spec(self._SPEC_DIR / "hi_Deva_IN.yaml") + spec = load_region_spec(self._SPEC_DIR / f"{locale}.yaml") variables = {v for inst in spec.institutions for p in inst.products for v in p.variables} domains = {str(d) for inst in spec.institutions for d in inst.domains} assert variables - set(spec.doc_titles.variable_labels) == set() From 839ee7b317ae69baea1e71f4828d9cfc3aca5a1a Mon Sep 17 00:00:00 2001 From: Yev Meyer Date: Tue, 20 Oct 2026 13:37:36 -0400 Subject: [PATCH 7/7] build: keep evaluate-assets scorecards out of git and the wheel evaluate-assets writes _eval_scorecard.parquet, and archives earlier ones under _scorecards/, beside the bank. Like embeddings.parquet, they are now gitignored, excluded from package data, and rejected by make check-dist. Signed-off-by: Yev Meyer --- .gitignore | 3 +++ pyproject.toml | 9 +++++++-- scripts/check_dist.py | 5 ++++- tests/test_check_dist.py | 7 ++++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 5942f4e..a832802 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,9 @@ artifacts/ output/ # Dense-retrieval vectors `gen-assets` writes beside a bank stay local. src/usersim/engine/assets/financial_services/**/embeddings.parquet +# So do the judge scorecards `evaluate-assets` writes there. +src/usersim/engine/assets/financial_services/**/_eval_scorecard.parquet +src/usersim/engine/assets/financial_services/**/_scorecards/ # IDE .vscode/ diff --git a/pyproject.toml b/pyproject.toml index 1ca1cc5..8859df5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,8 +165,13 @@ include = ["usersim*"] [tool.setuptools.exclude-package-data] # Retrieval embeds each query with the model that produced the corpus vectors, # so shipped vectors would tie every install to one embedding model. The -# vectors `gen-assets` writes beside a bank stay local. -"usersim.engine" = ["assets/financial_services/**/embeddings.parquet"] +# vectors `gen-assets` writes beside a bank stay local, as do the judge +# scorecards `evaluate-assets` writes there. +"usersim.engine" = [ + "assets/financial_services/**/embeddings.parquet", + "assets/financial_services/**/_eval_scorecard.parquet", + "assets/financial_services/**/_scorecards/*", +] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/scripts/check_dist.py b/scripts/check_dist.py index 28c101e..582d006 100644 --- a/scripts/check_dist.py +++ b/scripts/check_dist.py @@ -11,7 +11,8 @@ from the sdist, so a file only the wheel has came from somewhere else, such as a stale ``build/`` directory; - the wheel ships the asset banks, and no ``embeddings.parquet`` (retrieval - vectors stay with the embedding model that produced them); + vectors stay with the embedding model that produced them) or + ``evaluate-assets`` scorecard; - the sdist ships no ``tests/``. """ @@ -61,6 +62,8 @@ def check_archives(sdist: Path, wheel: Path) -> list[str]: problems.append("the wheel ships no asset banks, so every probe would fail at asset resolution") if any(f.endswith("embeddings.parquet") for f in wheel_files): problems.append("the wheel ships embeddings.parquet") + if any(f.endswith("_eval_scorecard.parquet") or "/_scorecards/" in f for f in wheel_files): + problems.append("the wheel ships an evaluate-assets scorecard") tests = [f for f in sdist_files if f.startswith("tests/")] if tests: problems.append(f"the sdist ships {len(tests)} file(s) under tests/") diff --git a/tests/test_check_dist.py b/tests/test_check_dist.py index 07ecc80..167fac8 100644 --- a/tests/test_check_dist.py +++ b/tests/test_check_dist.py @@ -58,9 +58,14 @@ def test_a_matching_pair_passes(tmp_path: Path) -> None: [*_PACKAGE, "usersim/engine/assets/x/embeddings.parquet"], "embeddings.parquet", ), + ( + ["pyproject.toml", *(f"src/{f}" for f in [*_PACKAGE, "usersim/engine/assets/x/_eval_scorecard.parquet"])], + [*_PACKAGE, "usersim/engine/assets/x/_eval_scorecard.parquet"], + "scorecard", + ), (["pyproject.toml", "tests/test_x.py", *(f"src/{f}" for f in _PACKAGE)], None, "under tests/"), ], - ids=["file-not-from-the-sdist", "no-banks", "embeddings", "tests-in-sdist"], + ids=["file-not-from-the-sdist", "no-banks", "embeddings", "scorecard", "tests-in-sdist"], ) def test_each_problem_is_reported(tmp_path: Path, sdist, wheel, expected: str) -> None: problems = check_dist.check_archives(*_pair(tmp_path, sdist, wheel))