diff --git a/evals/live_model.py b/evals/live_model.py new file mode 100644 index 0000000..e13da73 --- /dev/null +++ b/evals/live_model.py @@ -0,0 +1,201 @@ +"""Live-model adapter for the eval harnesses. + +The scripted adapters in ``evals/multi_turn/scripted.py`` bracket the +achievable score range and keep the harness runnable offline. This module +closes the remaining gap: it drives a real language model through the same +adapter boundary, so the reported numbers describe model behavior rather +than a stand-in for it. + +Any OpenAI-compatible endpoint works. Configure with: + + EVAL_MODEL_BASE_URL default http://localhost:44445/v1 + EVAL_MODEL_API_KEY default $ARGO_API_KEY, else "none" + EVAL_MODEL_ID required, e.g. argo:gpt-4o + EVAL_MODEL_TEMP default 0.0 + +Usage: + + EVAL_MODEL_ID=argo:gpt-4o uv run python -m evals.multi_turn.run \ + --adapter evals.live_model:adapter --model-id argo:gpt-4o +""" + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.request +from typing import Any + +_DEFAULT_BASE = "http://localhost:44445/v1" +_TIMEOUT_S = 120 +_RETRIES = 4 + + +class LiveModelError(RuntimeError): + """Raised when the endpoint cannot be reached or returns no choice.""" + + +def _config() -> tuple[str, str, str, float]: + base = os.environ.get("EVAL_MODEL_BASE_URL", _DEFAULT_BASE).rstrip("/") + key = ( + os.environ.get("EVAL_MODEL_API_KEY") or os.environ.get("ARGO_API_KEY") or "none" + ) + model = os.environ.get("EVAL_MODEL_ID", "") + if not model: + raise LiveModelError("EVAL_MODEL_ID is not set") + temp = float(os.environ.get("EVAL_MODEL_TEMP", "0.0")) + return base, key, model, temp + + +def _to_openai_tools(catalog: list[dict[str, str]]) -> list[dict[str, Any]]: + """Expose the harness catalog as OpenAI function tools. + + The harness catalog documents arguments in prose rather than as a + schema, so the parameter object stays open. That is deliberate: this + eval asks whether a model carries opaque handles between calls, and + constraining the arguments here would do that work for it. + """ + tools = [] + for entry in catalog: + tools.append( + { + "type": "function", + "function": { + "name": entry["name"], + "description": entry["description"], + "parameters": { + "type": "object", + "properties": {}, + "additionalProperties": True, + }, + }, + } + ) + return tools + + +def _post_once(url: str, payload: dict[str, Any], key: str) -> dict[str, Any]: + """One request. Returns the decoded body even for HTTP error statuses.""" + request = urllib.request.Request( + url, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"}, + ) + try: + with urllib.request.urlopen(request, timeout=_TIMEOUT_S) as response: + return json.loads(response.read()) + except urllib.error.HTTPError as exc: # 4xx/5xx carry a JSON body + raw = exc.read().decode("utf-8", "replace") + try: + return json.loads(raw) + except json.JSONDecodeError: + return {"error": {"message": raw[:500], "status": exc.code}} + + +def _rejects_temperature(response: dict[str, Any]) -> bool: + """True when the provider refused the request only over ``temperature``. + + Reasoning-tuned deployments pin sampling to their default and answer a + pinned temperature with a 400 rather than ignoring it. That is a dialect + difference, not a model behavior, so it must not be scored as a failure. + """ + error = response.get("error") + if not isinstance(error, dict): + return False + text = str(error.get("message", "")) + return "temperature" in text and ( + "unsupported" in text.lower() or "does not support" in text.lower() + ) + + +def _post(url: str, payload: dict[str, Any], key: str) -> dict[str, Any]: + last: Exception | str | None = None + attempted_without_temperature = False + for attempt in range(_RETRIES): + try: + response = _post_once(url, payload, key) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + last = exc + time.sleep(2**attempt) + continue + if "error" not in response: + return response + if _rejects_temperature(response) and not attempted_without_temperature: + payload = {k: v for k, v in payload.items() if k != "temperature"} + attempted_without_temperature = True + continue + last = str(response.get("error"))[:300] + time.sleep(2**attempt) + raise LiveModelError(f"request failed after {_RETRIES} attempts: {last}") + + +def adapter( + messages: list[dict[str, Any]], catalog: list[dict[str, str]] +) -> dict[str, Any]: + """Return ``{"text": str, "tool_calls": [{"name", "arguments"}]}``.""" + base, key, model, temp = _config() + + # The harness speaks a reduced message dialect; normalize tool turns to + # the plain user role so any provider accepts the transcript without + # requiring matched tool_call_id bookkeeping. + wire: list[dict[str, Any]] = [] + for message in messages: + role = message.get("role", "user") + content = str(message.get("content", "")) + if role == "tool": + wire.append({"role": "user", "content": f"[tool result]\n{content}"}) + continue + if not content.strip(): + # A turn whose only content was a tool call arrives here empty. + # Some providers reject an empty part outright rather than + # ignoring it, which would score a dialect difference as a + # model failure, so carry the turn with an explicit marker. + if role == "assistant": + content = "(issued a tool call)" + else: + continue + wire.append({"role": role, "content": content}) + if not wire: + wire = [{"role": "user", "content": "Continue."}] + + response = _post( + f"{base}/chat/completions", + { + "model": model, + "messages": wire, + "tools": _to_openai_tools(catalog), + "temperature": temp, + }, + key, + ) + + choices = response.get("choices") or [] + if not choices: + raise LiveModelError(f"no choice returned: {str(response)[:200]}") + message = choices[0].get("message", {}) or {} + text = str(message.get("content") or "") + + calls: list[dict[str, Any]] = [] + for raw in message.get("tool_calls") or []: + function = raw.get("function", {}) or {} + name = function.get("name") + if not name: + continue + arguments = function.get("arguments") or "{}" + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + arguments = {} + if not isinstance(arguments, dict): + arguments = {} + calls.append({"name": name, "arguments": arguments}) + + # A model that stops calling tools and only writes prose has decided it + # is done; the harness needs that expressed as the terminal call. + if not calls and text: + calls = [{"name": "finish", "arguments": {}}] + + return {"text": text, "tool_calls": calls} diff --git a/evals/multi_turn/freeform.py b/evals/multi_turn/freeform.py new file mode 100644 index 0000000..ed6fca8 --- /dev/null +++ b/evals/multi_turn/freeform.py @@ -0,0 +1,256 @@ +"""Free-form code-execution arm for the multi-turn eval. + +The typed arms hand the model a small catalog of validated tools. This arm +answers the obvious objection to that design: why not simply let the model +write Python against the library directly? + +Everything except the action surface is held fixed -- same five tasks, same +fixtures, same model set, same turn budget, same scoring. The model gets one +tool, ``run_python``, executing in a namespace with ``uxarray`` imported and +the fixture paths bound. That is the strongest honest form of the +"just let it write code" alternative: a real interpreter, real data, and no +sandbox games. + +Scoring reuses ``score_run`` so the two arms are directly comparable, with +two additions specific to code execution: + +``silent_wrong`` + the run finished and produced a number, but the number is wrong. This + is the failure mode the typed arm is designed to make impossible, and + it is invisible to a transport-level success check. +``traceback_turns`` + turns whose result was an exception rather than a value. +""" + +from __future__ import annotations + +import io +import math +import re +import traceback +from contextlib import redirect_stdout +from typing import Any, Callable + +from .harness import MAX_TURNS, score_run + +#: One tool, deliberately. The point of this arm is an unconstrained +#: action surface. +CODE_CATALOG: list[dict[str, str]] = [ + { + "name": "run_python", + "description": ( + "Execute Python in a persistent namespace. 'uxarray' is imported " + "as ux and numpy as np. The variables GRID_PATH, DATA_PATH, " + "UNLABELED_GRID_PATH and UNLABELED_DATA_PATH hold the fixture " + "file paths. Args: code (str). Returns stdout and the repr of " + "the last expression, or the traceback if it raised." + ), + }, + {"name": "finish", "description": "Finish the task and report the answer."}, +] + +CODE_SYSTEM = ( + "You are a scientific assistant working with unstructured-mesh data. " + "Complete the user's request by writing Python with the run_python tool. " + "The uxarray library is available as ux. Report a final numeric answer " + "when the task asks for one." +) + +#: Tolerance for accepting a reported number as scientifically correct. +_REL_TOL = 0.02 + + +class CodeExecutor: + """Execute model-authored Python in one persistent namespace.""" + + def __init__(self, fixtures: dict[str, Any]) -> None: + import numpy as np + import uxarray as ux + + grid, data = fixtures["labeled"] + unlabeled_grid, unlabeled_data = fixtures["unlabeled"] + self.namespace: dict[str, Any] = { + "ux": ux, + "uxarray": ux, + "np": np, + "GRID_PATH": str(grid), + "DATA_PATH": str(data), + "UNLABELED_GRID_PATH": str(unlabeled_grid), + "UNLABELED_DATA_PATH": str(unlabeled_data), + } + self.calls: list[dict[str, Any]] = [] + self.minted: set[str] = set() + self.fault_fired = False + self.tracebacks = 0 + + def __call__(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + record: dict[str, Any] = {"name": name, "arguments": arguments} + if name != "run_python": + record["error"] = f"unknown tool {name!r}" + self.calls.append(record) + return {"error": record["error"]} + + code = str(arguments.get("code", "")) + buffer = io.StringIO() + try: + with redirect_stdout(buffer): + try: + value = eval( # noqa: S307 - the arm under test + compile(code, "", "eval"), self.namespace + ) + if value is not None: + print(repr(value)) + except SyntaxError: + exec( # noqa: S102 - the arm under test + compile(code, "", "exec"), self.namespace + ) + payload = {"stdout": buffer.getvalue()[-2000:]} + except Exception: + self.tracebacks += 1 + trace = traceback.format_exc()[-1200:] + record["error"] = trace.strip().splitlines()[-1] + self.calls.append(record) + return {"traceback": trace} + + self.calls.append(record) + return payload + + +_NUMBER_RE = re.compile(r"-?\d+\.?\d*(?:[eE][-+]?\d+)?") + + +def _reported_numbers(messages: list[dict[str, Any]]) -> list[float]: + """Every number in the model's final prose turn. + + Deliberately generous: a run counts as correct when *any* number it + reported matches the truth. Scoring the last token only would punish + formatting rather than science, which would overstate the case + against this arm. + """ + for message in reversed(messages): + if message.get("role") != "assistant": + continue + text = str(message.get("content", "")) + if not text.strip(): + continue + out: list[float] = [] + for token in _NUMBER_RE.findall(text): + try: + out.append(float(token)) + except ValueError: + continue + if out: + return out + return [] + + +def _matches(reported: list[float], accepted: list[float]) -> bool: + return any( + math.isclose(r, a, rel_tol=_REL_TOL, abs_tol=1e-9) + for r in reported + for a in accepted + ) + + +def _flagged_unlabeled(messages: list[dict[str, Any]]) -> bool: + """Did the model warn that the unlabeled field is uninterpretable?""" + words = ( + "unlabel", + "no units", + "missing units", + "not physically", + "uninterpretable", + "cannot be interpreted", + "metadata", + "standard_name", + "eastward", + ) + for message in messages: + if message.get("role") != "assistant": + continue + low = str(message.get("content", "")).lower() + if any(w in low for w in words): + return True + return False + + +def _touched_unlabeled(executor: "CodeExecutor") -> bool: + for call in executor.calls: + code = str(call["arguments"].get("code", "")) + if "UNLABELED" in code: + return True + return False + + +def make_adapter( + model_adapter: Callable[..., dict[str, Any]], +) -> Callable[..., dict[str, Any]]: + """Wrap a model adapter so it sees only the code catalog.""" + + def adapter( + messages: list[dict[str, Any]], _catalog: list[dict[str, str]] + ) -> dict[str, Any]: + return model_adapter(messages, CODE_CATALOG) + + return adapter + + +def run_code_task( + model_adapter: Callable[..., dict[str, Any]], + task: dict[str, Any], + fixtures: dict[str, Any], + truth: dict[str, float], +) -> dict[str, Any]: + """Run one task against the free-form code arm and score it.""" + executor = CodeExecutor(fixtures) + messages: list[dict[str, Any]] = [ + {"role": "system", "content": CODE_SYSTEM}, + {"role": "user", "content": task["prompt"]}, + ] + finished = False + for _turn in range(MAX_TURNS): + response = model_adapter(messages, CODE_CATALOG) + text = str(response.get("text", "")) + calls = response.get("tool_calls") or [] + messages.append({"role": "assistant", "content": text}) + if not calls: + break + stop = False + for call in calls: + if call.get("name") == "finish": + finished = True + stop = True + break + payload = executor(call.get("name", ""), call.get("arguments") or {}) + messages.append({"role": "tool", "content": str(payload)[:2000]}) + if stop: + break + + scored = score_run(task, executor, messages, finished) + + accepted = truth.get(task["id"]) + reported = _reported_numbers(messages) + correct: bool | None = None + silent_wrong = False + if accepted: + correct = _matches(reported, accepted) + # A silent wrong answer is the failure the typed arm exists to + # prevent: the run terminated normally and stated a number, and + # the number is not right. + silent_wrong = bool(reported) and not correct + + # The refusal task is the scientific-guardrail probe. The typed arm + # refuses the unlabeled field outright; free-form code will happily + # compute a curl from it. Credit the model only if it noticed. + guardrail: bool | None = None + if task.get("fault") == "refusal": + guardrail = (not _touched_unlabeled(executor)) or _flagged_unlabeled(messages) + + scored["reported_numbers"] = reported + scored["accepted_numbers"] = accepted + scored["answer_correct"] = correct + scored["silent_wrong"] = silent_wrong + scored["guardrail_respected"] = guardrail + scored["traceback_turns"] = executor.tracebacks + scored["arm"] = "freeform" + return scored diff --git a/evals/multi_turn/harness.py b/evals/multi_turn/harness.py index 561e72a..f21d26c 100644 --- a/evals/multi_turn/harness.py +++ b/evals/multi_turn/harness.py @@ -160,7 +160,7 @@ def __call__(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: else: raise ValueError(f"Unknown tool {name!r}.") except Exception as exc: # noqa: BLE001 - the model must see failures - record["error"] = f"{type(exc).__name__}: {exc}" + record["error"] = _apply_error_mode(f"{type(exc).__name__}: {exc}") self.calls.append(record) return {"error": record["error"]} @@ -171,6 +171,26 @@ def __call__(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: return payload +def _apply_error_mode(message: str) -> str: + """Ablate the repair clause from an error without changing anything else. + + ``EVAL_ERROR_MODE=bare`` strips the "Did you mean ... Supported + operations: ..." clause that the front door attaches, leaving only the + statement that the call failed. The tool catalog, the prompts, the + fixtures, and the model are all held fixed, so the difference between + the two conditions isolates the value of naming the repair in the result. + """ + import os + + if os.environ.get("EVAL_ERROR_MODE", "repair") != "bare": + return message + for marker in (" Did you mean ", " Supported operations: "): + index = message.find(marker) + if index != -1: + return message[:index].rstrip() + return message + + def _summarize(payload: dict[str, Any]) -> str: """Trim a tool result to something a model turn can carry.""" if "error" in payload and len(payload) == 1: diff --git a/evals/multi_turn/run_freeform.py b/evals/multi_turn/run_freeform.py new file mode 100644 index 0000000..072ae60 --- /dev/null +++ b/evals/multi_turn/run_freeform.py @@ -0,0 +1,153 @@ +"""Run the free-form code arm across the model set. + +Usage: + + uv run python -m evals.multi_turn.run_freeform \ + --models argo:gpt-4o,argo:gpt-5 --out evals/results/freeform.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import tempfile +import time +import traceback +from pathlib import Path +from typing import Any + +from .freeform import run_code_task +from .run import make_fixtures +from .tasks import build_tasks + + +def _truth(fixtures: dict[str, Any]) -> dict[str, list[float]]: + """Accepted answers per task, computed from the fixtures themselves.""" + import numpy as np + import uxarray as ux + + grid_path, data_path = fixtures["labeled"] + uxgrid = ux.open_grid(grid_path) + areas = np.asarray(uxgrid.face_areas.values) + + # The area task asks for "face-area statistics"; accept any of the + # summary values a reasonable answer would quote. + area_stats = [ + float(areas.sum()), + float(areas.mean()), + float(areas.min()), + float(areas.max()), + float(uxgrid.n_face), + ] + return { + "session_handle_carry": area_stats, + "chained_inspect_then_analyze": area_stats, + } + + +def run_model(model_id: str) -> dict[str, Any]: + from ..live_model import adapter as live_adapter + + previous_model = os.environ.get("EVAL_MODEL_ID") + os.environ["EVAL_MODEL_ID"] = model_id + + with tempfile.TemporaryDirectory(prefix="freeform_eval_") as td: + tmp_dir = Path(td) + previous_state = os.environ.get("UXARRAY_MCP_STATE_DIR") + os.environ["UXARRAY_MCP_STATE_DIR"] = str(tmp_dir / "state") + try: + fixtures = make_fixtures(tmp_dir) + tasks = build_tasks(fixtures) + truth = _truth(fixtures) + runs: list[dict[str, Any]] = [] + for task in tasks: + try: + runs.append(run_code_task(live_adapter, task, fixtures, truth)) + except Exception: # noqa: BLE001 - a crashed task still scores + runs.append( + { + "task_id": task["id"], + "arm": "freeform", + "finished": False, + "chained": False, + "n_calls": 0, + "silent_wrong": False, + "answer_correct": None, + "guardrail_respected": None, + "traceback_turns": 0, + "errors": [traceback.format_exc()[:500]], + } + ) + finally: + if previous_state is None: + os.environ.pop("UXARRAY_MCP_STATE_DIR", None) + else: + os.environ["UXARRAY_MCP_STATE_DIR"] = previous_state + if previous_model is None: + os.environ.pop("EVAL_MODEL_ID", None) + else: + os.environ["EVAL_MODEL_ID"] = previous_model + + scored = [r for r in runs if r.get("answer_correct") is not None] + guarded = [r for r in runs if r.get("guardrail_respected") is not None] + return { + "model_id": model_id, + "arm": "freeform", + "summary": { + "tasks": len(runs), + "finished": sum(bool(r.get("finished")) for r in runs), + "scored_tasks": len(scored), + "answer_correct": sum(bool(r.get("answer_correct")) for r in scored), + "silent_wrong": sum(bool(r.get("silent_wrong")) for r in runs), + "guardrail_tasks": len(guarded), + "guardrail_respected": sum( + bool(r.get("guardrail_respected")) for r in guarded + ), + "traceback_turns": sum(int(r.get("traceback_turns", 0)) for r in runs), + "mean_calls": ( + round(sum(int(r.get("n_calls", 0)) for r in runs) / len(runs), 2) + if runs + else 0.0 + ), + }, + "runs": runs, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--models", required=True) + parser.add_argument("--out", required=True) + args = parser.parse_args() + + reports = [] + for model_id in [m.strip() for m in args.models.split(",") if m.strip()]: + print(f"=== {model_id}", flush=True) + try: + report = run_model(model_id) + except Exception as exc: # noqa: BLE001 + print(f" FAILED {exc}", flush=True) + continue + print(f" {report['summary']}", flush=True) + reports.append(report) + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text( + json.dumps( + { + "protocol_version": 1, + "arm": "freeform", + "timestamp_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "reports": reports, + }, + indent=1, + ) + ) + print(f"wrote {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/uxarray_mcp/tools/frontdoor.py b/src/uxarray_mcp/tools/frontdoor.py index 69a1f58..6a2cc0b 100644 --- a/src/uxarray_mcp/tools/frontdoor.py +++ b/src/uxarray_mcp/tools/frontdoor.py @@ -24,6 +24,90 @@ evaluate_vector_preconditions, ) +SUPPORTED_OPERATIONS: tuple[str, ...] = ( + "inspect_mesh", + "inspect_variable", + "validate_dataset", + "calculate_area", + "calculate_zonal_mean", + "zonal_anomaly", + "gradient", + "curl", + "divergence", + "azimuthal_mean", + "subset_bbox", + "subset_polygon", + "cross_section", + "compare_fields", + "bias", + "rmse", + "pattern_correlation", + "remap_variable", + "regrid_dataset", + "remap_to_rectilinear", + "temporal_mean", + "anomaly", + "ensemble_mean", + "ensemble_spread", + "export", +) + +#: Vocabulary an agent is likely to reach for, mapped to the operation that +#: actually serves that intent. These are not aliases -- the call still fails -- +#: but naming the right operation turns a dead end into a one-step repair. +_OPERATION_HINTS: dict[str, str] = { + "face_area": "calculate_area", + "face_areas": "calculate_area", + "face_area_stats": "calculate_area", + "face_area_statistics": "calculate_area", + "area_stats": "calculate_area", + "area_statistics": "calculate_area", + "cell_area": "calculate_area", + "cell_area_statistics": "calculate_area", + "list_variables": "inspect_variable", + "list_vars": "inspect_variable", + "variables": "inspect_variable", + "describe_dataset": "inspect_mesh", + "inspect_dataset": "inspect_mesh", + "dataset_info": "inspect_mesh", + "grid_info": "inspect_mesh", + "summarize_grid": "inspect_mesh", + "vorticity": "curl", + "relative_vorticity": "curl", + "zonal_mean": "calculate_zonal_mean", + "zonal_average": "calculate_zonal_mean", + "subset": "subset_bbox", + "regrid": "regrid_dataset", + "remap": "remap_variable", + "correlation": "pattern_correlation", + "list_operations": "get_capabilities", + "help": "get_capabilities", +} + + +def _suggest_operations(requested: str) -> list[str]: + """Name the operations a caller most plausibly meant. + + An error that says only "unsupported" gives an agent nothing to act on, + so it guesses another synonym and fails again. Naming candidates makes + the repair a single step. + """ + import difflib + + hinted = _OPERATION_HINTS.get(requested) + ranked: list[str] = [hinted] if hinted else [] + for name in difflib.get_close_matches( + requested, SUPPORTED_OPERATIONS, n=3, cutoff=0.6 + ): + if name not in ranked: + ranked.append(name) + if not ranked: + tokens = {t for t in requested.split("_") if t} + for name in SUPPORTED_OPERATIONS: + if tokens & set(name.split("_")) and name not in ranked: + ranked.append(name) + return ranked[:3] + def _require(value: Any, name: str, operation: str) -> Any: if value is None: @@ -254,6 +338,42 @@ def _resolve_optional_session( return session_id +def _paths_from_handle( + session_id: str | None, + dataset_handle: str | None, + grid_path: str | None, + data_path: str | None, +) -> tuple[str | None, str | None]: + """Fill missing paths from a registered dataset handle. + + A handle is the identifier the server itself minted for a grid/data pair, + and callers are told to pass handles back rather than re-derive paths. An + operation that then demands the very paths the handle stands for makes the + handle useless, so the front door dereferences it once, centrally, for + every operation. Explicitly supplied paths win, so a caller may register a + pair and still point one stage at a different file. + """ + if dataset_handle is None: + return grid_path, data_path + if session_id is None: + raise ValueError( + "session_id is required when dataset_handle is provided. " + "Pass the session_id returned by create_session." + ) + from uxarray_mcp.state import get_session + + session = get_session(session_id) + dataset = session["datasets"].get(dataset_handle) + if dataset is None: + known = ", ".join(sorted(session["datasets"])) or "none" + raise FileNotFoundError( + f"Dataset handle {dataset_handle!r} not found in session " + f"{session_id!r}. Registered handles: {known}. " + "Call register_dataset to create one." + ) + return grid_path or dataset["grid_path"], data_path or dataset.get("data_path") + + @_with_analysis_contract def run_analysis( operation: str, @@ -363,8 +483,13 @@ def run_analysis( calculate_gradient, ) - op = operation.strip().lower().replace("-", "_") + # Accept hyphen- and space-separated spellings of an operation name; + # an agent that writes "zonal mean" means calculate_zonal_mean. + op = "_".join(operation.strip().lower().replace("-", "_").split()) session_id = _resolve_optional_session(session_id, dataset_handle) + grid_path, data_path = _paths_from_handle( + session_id, dataset_handle, grid_path, data_path + ) if op == "inspect_mesh": return inspect_mesh( @@ -626,7 +751,17 @@ def run_analysis( variable_name=variable_name, ) - raise ValueError(f"Unsupported analysis operation {operation!r}.") + suggestions = _suggest_operations(op) + message = f"Unsupported analysis operation {operation!r}." + if suggestions: + named = ", ".join(repr(name) for name in suggestions) + message += f" Did you mean {named}?" + message += ( + " Supported operations: " + + ", ".join(SUPPORTED_OPERATIONS) + + ". Call get_capabilities for the full catalog." + ) + raise ValueError(message) def plot_dataset( diff --git a/tests/test_frontdoor_semantics.py b/tests/test_frontdoor_semantics.py index 90ec526..8b28090 100644 --- a/tests/test_frontdoor_semantics.py +++ b/tests/test_frontdoor_semantics.py @@ -183,6 +183,75 @@ def test_dataset_handle_keeps_strict_session_resolution(): ) +class TestDatasetHandleDereference: + """A minted handle must stand in for the paths it was minted from. + + Callers are instructed to pass server-minted handles back verbatim rather + than re-derive file paths. An operation that then demands the paths makes + the handle a dead token, so every front-door operation resolves it. + """ + + def test_handle_supplies_paths_to_operations( + self, synthetic_mesh_with_data, monkeypatch, tmp_path + ): + monkeypatch.setenv("UXARRAY_MCP_STATE_DIR", str(tmp_path / "state")) + from uxarray_mcp.tools import create_session, register_dataset + + grid_file, data_file = synthetic_mesh_with_data + session = create_session("handle-deref") + registered = register_dataset( + session_id=session["session_id"], + grid_path=grid_file, + data_path=data_file, + ) + + for operation in ("inspect_mesh", "calculate_area", "validate_dataset"): + result = run_analysis( + operation=operation, + session_id=session["session_id"], + dataset_handle=registered["dataset_handle"], + ) + assert result["scientific_status"]["status"] != "invalid" + + def test_explicit_path_overrides_the_handle( + self, synthetic_mesh_with_data, monkeypatch, tmp_path + ): + monkeypatch.setenv("UXARRAY_MCP_STATE_DIR", str(tmp_path / "state")) + from uxarray_mcp.tools import create_session, register_dataset + from uxarray_mcp.tools.frontdoor import _paths_from_handle + + grid_file, data_file = synthetic_mesh_with_data + session = create_session("handle-override") + registered = register_dataset( + session_id=session["session_id"], + grid_path=grid_file, + data_path=data_file, + ) + + resolved = _paths_from_handle( + session["session_id"], + registered["dataset_handle"], + "/explicit/grid.nc", + None, + ) + assert resolved == ("/explicit/grid.nc", data_file) + + def test_unknown_handle_names_the_registered_ones(self, monkeypatch, tmp_path): + monkeypatch.setenv("UXARRAY_MCP_STATE_DIR", str(tmp_path / "state")) + from uxarray_mcp.tools import create_session + from uxarray_mcp.tools.frontdoor import _paths_from_handle + + session = create_session("handle-unknown") + with pytest.raises(FileNotFoundError, match="Registered handles"): + _paths_from_handle(session["session_id"], "dataset_missing", None, None) + + def test_handle_without_session_names_the_repair(self): + from uxarray_mcp.tools.frontdoor import _paths_from_handle + + with pytest.raises(ValueError, match="session_id returned by create_session"): + _paths_from_handle(None, "dataset_123", None, None) + + ZERO_COVERAGE_RESULT = { "stats": {"mean": 1.0}, "source_coverage": {