diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cb59c8..039d30b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to ReadyAgents Core. ## Unreleased +### Added + +- Approve what you saw: an approval pause stores `prompt_sha256` (sha256 of the + rendered gate prompt) in the run file and the `paused` audit line. An approve + on resume/decide re-renders and compares; on a change it is refused with + `ReapprovalRequired`, CLI exit **3**, reason `reapprove_required` (stderr and + `--json`), a `decision_refused` audit line, and the run re-pauses on the new + prompt. Reject is never blocked. Runs paused before this release have no digest + and resume as before with a warning. Exit codes 0/1/2 are unchanged. + ## 2.0.10 — 2026-09-22 ### Added diff --git a/docs/cli.md b/docs/cli.md index ce5d06e..793bec3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -404,7 +404,7 @@ readyagents run examples/calc_pipeline.yaml --stream --json Signing proves origin, not safety. See [supply-chain.md](supply-chain.md). -Exit code `1` on validation or execution errors, including a missing workflow file (`ConfigError`). Exit code `2` is reserved for an **approval** node pausing for a decision. The CLI prints `ErrorClass: message` rather than a full traceback. Logs include `run=` and `node=`. +Exit code `1` on validation or execution errors, including a missing workflow file (`ConfigError`). Exit code `2` is reserved for an **approval** node pausing for a decision. Exit code `3` means `resume --approve` / `decide` was refused because the approval prompt changed since the pause (`reapprove_required`; `--json` carries `reason`, `expected_sha256`, `actual_sha256`); the run stays paused on the new prompt, so review and approve again. The CLI prints `ErrorClass: message` rather than a full traceback. Logs include `run=` and `node=`. Failed runs print the **node timeline**, `run_id`, and a `readyagents resume RUN_ID` hint (same idea as approval pauses). `--json` is an envelope with additive `ok` and `command` plus existing keys (`run_id`, `error`, `message`, `run`). Pause is exit 2 and still includes `run_id`. `resume` and `runs replay` accept `--json` too. JSON is written without Rich markup, so values like `[dry-run]` stay intact. diff --git a/docs/stability.md b/docs/stability.md index c92461c..b7fb69f 100644 --- a/docs/stability.md +++ b/docs/stability.md @@ -158,7 +158,7 @@ Everything else, including `readyagents.replay` internals, is not stable. - Workflow file format v1 (`schemas/workflow-v1.json`, schema `$id`). - Run-record format `record_version: 1`. A 0.9-era record without that field still loads. -- CLI command names and exit codes (approval pause remains **2**). +- CLI command names and exit codes (approval pause remains **2**; **3** is `reapprove_required`, added in 2.0.11). - Documented `--json` envelope keys `ok` and `command` are stable; other keys are additive. - Pack protocol (`get_pack()`) and tool/node type names. diff --git a/docs/workflows.md b/docs/workflows.md index e2d6eb3..01fcef0 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -188,6 +188,8 @@ The approval YAML is unchanged. The optional page is a foreground loopback UI, n Optional enterprise fields (all inert unless set): `approvals_required`, `distinct_actors`, `deny_actor`, `approver_roles`, `require`, `expires_in`, `on_expire` (`reject` / `escalate` / `fail`; **`approve` is refused**), `escalate_to`, `require_reason`, `reject_short_circuit`, `recommendation`, `notify`. Core starts **no timer** — expiry is lazy. Full model: [approvals.md](approvals.md). +**Approve what you saw.** At the pause, the engine stores the sha256 of the rendered prompt in `pending.prompt_sha256` (run file and `paused` audit line). `resume --approve` / `decide --decision approve` re-renders the prompt and compares. If it changed (workflow edited, upstream state differs), the approve is refused: CLI exit **3**, reason `reapprove_required` (stderr and `--json` `reason`, plus `expected_sha256` / `actual_sha256`), a `decision_refused` audit line, and the run stays **paused** on the new prompt. Review it and approve again. Reject is never blocked. Runs paused before 2.0.11 have no digest and resume as before with a warning. + `then` is the approve path; `else` is the reject path. `next` is used when approved if `then` is omitted. Multiple gates in one graph are allowed. Each needs its own `--approve NODE` (or `--reject`), or an injected JSON decision (`readyagents decide` / `--decision-file`). See `examples/multi_gate.yaml` (in a checkout; `new --from-example multi_gate` from PyPI). diff --git a/src/readyagents/cli/_common.py b/src/readyagents/cli/_common.py index ff45ca1..0bf1a7a 100644 --- a/src/readyagents/cli/_common.py +++ b/src/readyagents/cli/_common.py @@ -10,11 +10,13 @@ from rich.table import Table from readyagents.errors import ( + REAPPROVE_EXIT_CODE, ApprovalRequired, CassetteMiss, ConverseRequired, EnvRefused, ReadyAgentsError, + ReapprovalRequired, WaitingRequired, ) from readyagents.workflow.state import ( @@ -130,6 +132,29 @@ def _emit_run_exception( err_console.print(f"[yellow]waiting[/yellow] {escape(str(exc))}") raise typer.Exit(code=2) from exc + if isinstance(exc, ReapprovalRequired): + state = _state_from_exc(exc) + if as_json: + payload = { + "error": type(exc).__name__, + "reason": exc.reason, + "message": str(exc), + "run_id": exc.run_id, + "node_id": exc.node_id, + "prompt": exc.prompt, + "expected_sha256": exc.expected_sha256, + "actual_sha256": exc.actual_sha256, + "status": "paused", + } + if state is not None: + payload["run"] = state.to_record() + _print_json(_json_envelope(command, ok=False, **payload)) + else: + err_console.print(f"[red]{exc.reason}[/red] {escape(str(exc))}", highlight=False) + if exc.prompt: + console.print(Panel(escape(exc.prompt), title=f"Approval: {exc.node_id}")) + raise typer.Exit(code=REAPPROVE_EXIT_CODE) from exc + if isinstance(exc, (ApprovalRequired, ConverseRequired)): state = _state_from_exc(exc) if as_json: diff --git a/src/readyagents/errors.py b/src/readyagents/errors.py index 4a4bf1c..1473144 100644 --- a/src/readyagents/errors.py +++ b/src/readyagents/errors.py @@ -730,6 +730,39 @@ def __init__( ) +class ReapprovalRequired(ApprovalRequired): + """``--approve`` was refused: the gate prompt changed since the run paused. + + The run stays paused with the new prompt and digest, so the approver must + review it and approve again. CLI exit code ``REAPPROVE_EXIT_CODE``. + """ + + reason = "reapprove_required" + + def __init__( + self, + node_id: str, + run_id: str, + prompt: str, + *, + expected_sha256: str, + actual_sha256: str, + state: object | None = None, + pause: dict | None = None, + ) -> None: + super().__init__(node_id, run_id, prompt, state=state, pause=pause) + self.expected_sha256 = expected_sha256 + self.actual_sha256 = actual_sha256 + self.args = ( + f"reapprove_required: approval prompt for node '{node_id}' (run {run_id}) " + f"changed since the pause (sha256 {expected_sha256[:12]} -> {actual_sha256[:12]}). " + f"Review it, then: readyagents resume {run_id} --approve {node_id}", + ) + + +REAPPROVE_EXIT_CODE = 3 + + class BudgetExceeded(ReadyAgentsError): """An LLM call was blocked because the run is over its token or cost budget.""" diff --git a/src/readyagents/workflow/engine.py b/src/readyagents/workflow/engine.py index e1f57b9..063656b 100644 --- a/src/readyagents/workflow/engine.py +++ b/src/readyagents/workflow/engine.py @@ -25,6 +25,7 @@ from readyagents.logging import get_logger, log_event from readyagents.workflow.nodes import ( ExecutionContext, + approval_prompt_digest, evaluate_condition, execute_node_with_policy, ) @@ -209,6 +210,7 @@ def run_workflow( "node_id": exc.node_id, "type": "approval", "prompt": exc.prompt, + "prompt_sha256": approval_prompt_digest(exc.prompt or ""), "then": getattr(paused, "then", None), "else": getattr(paused, "else_", None), "resume": f"readyagents resume {state.run_id} --approve {exc.node_id}", @@ -231,6 +233,7 @@ def run_workflow( run_id=state.run_id, node_id=exc.node_id, actor=ctx.actor, + prompt_sha256=state.pending["prompt_sha256"], ) _notify_pause(ctx, exc, state) _observe( diff --git a/src/readyagents/workflow/nodes.py b/src/readyagents/workflow/nodes.py index 417d816..2e4a9ed 100644 --- a/src/readyagents/workflow/nodes.py +++ b/src/readyagents/workflow/nodes.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import threading from collections.abc import Callable, Iterator, Mapping @@ -29,6 +30,7 @@ NodeError, PolicyDenied, ReadyAgentsError, + ReapprovalRequired, RouteBudgetExceeded, RoutingError, RunawayGuard, @@ -1046,6 +1048,53 @@ def _run_condition(node: NodeSpec, state: RunState, ctx: ExecutionContext) -> di return {"matched": matched, "next": nxt} +def approval_prompt_digest(prompt: str) -> str: + """sha256 hex of the rendered approval prompt the approver is shown.""" + return hashlib.sha256(prompt.encode("utf-8")).hexdigest() + + +def _check_approval_digest( + node: NodeSpec, state: RunState, ctx: ExecutionContext, prompt: str +) -> None: + """Refuse ``--approve`` when the prompt differs from the one paused on.""" + # Peek: decision_for() consumes MCP decisions, the gate below must still see it. + if ctx.decisions.get(node.id) not in _APPROVE_VALUES: + return + pending = state.pending if isinstance(state.pending, dict) else {} + if pending.get("node_id") != node.id or pending.get("type") != "approval": + return + expected = pending.get("prompt_sha256") + if not expected: + log.warning( + "approval %s: run record has no prompt_sha256 (paused before 2.0.11); " + "approving without a prompt check", + node.id, + extra={"run_id": state.run_id, "node_id": node.id}, + ) + return + actual = approval_prompt_digest(prompt) + if actual == expected: + return + if ctx.auditor is not None: + ctx.auditor( + "decision_refused", + run_id=state.run_id, + node_id=node.id, + actor=ctx.actor, + reason=ReapprovalRequired.reason, + expected_sha256=expected, + actual_sha256=actual, + ) + raise ReapprovalRequired( + node.id, + state.run_id, + prompt, + expected_sha256=str(expected), + actual_sha256=actual, + state=state, + ) + + def _run_approval(node: NodeSpec, state: RunState, ctx: ExecutionContext) -> dict[str, Any]: from readyagents.approvals.gate import ( Vote, @@ -1065,6 +1114,7 @@ def _run_approval(node: NodeSpec, state: RunState, ctx: ExecutionContext) -> dic ns = state.mapping() prompt = interpolate(_prompt_text(node, ctx) or f"Approve node '{node.id}'?", ns) + _check_approval_digest(node, state, ctx, prompt) pending = state.pending if isinstance(state.pending, dict) else {} use_enterprise = enterprise_fields_set(node) or ( isinstance(pending, dict) diff --git a/tests/test_approval_digest.py b/tests/test_approval_digest.py new file mode 100644 index 0000000..9a6f825 --- /dev/null +++ b/tests/test_approval_digest.py @@ -0,0 +1,156 @@ +"""Approval prompt digest: approve-then-edit refusal (reapprove_required).""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from readyagents.cli import app +from readyagents.config import clear_settings_cache +from readyagents.errors import REAPPROVE_EXIT_CODE, ApprovalRequired, ReapprovalRequired +from readyagents.workflow.nodes import approval_prompt_digest +from readyagents.workflow.runner import resume_run, run_workflow_file + +runner = CliRunner() + +WF = """ +name: dg +start: g +nodes: + - id: g + type: approval + prompt: "{prompt}" + then: ok + else: denied + - id: ok + type: transform + template: "ok" + output_key: summary + - id: denied + type: transform + template: "denied" + output_key: summary +""" + + +def _write(path: Path, prompt: str) -> Path: + path.write_text(WF.format(prompt=prompt), encoding="utf-8") + return path + + +def _run_file(home: Path, run_id: str) -> Path: + matches = list(home.rglob(f"runs/{run_id}.json")) + assert len(matches) == 1, matches + return matches[0] + + +def _audit_events(home: Path, run_id: str) -> list[dict]: + lines = [] + for path in home.rglob(f"audit/{run_id}.jsonl"): + lines += [json.loads(x) for x in path.read_text(encoding="utf-8").splitlines() if x] + return lines + + +def test_pause_stores_digest_in_run_file_and_audit(tmp_settings) -> None: + path = _write(tmp_settings.workspace_path() / "dg.yaml", "Pay 42?") + with pytest.raises(ApprovalRequired) as paused: + run_workflow_file(path, settings=tmp_settings, persist=True) + run_id = paused.value.run_id + want = approval_prompt_digest("Pay 42?") + record = json.loads(_run_file(tmp_settings.home, run_id).read_text(encoding="utf-8")) + assert record["pending"]["prompt_sha256"] == want + events = [e for e in _audit_events(tmp_settings.home, run_id) if e["event"] == "paused"] + assert events and events[-1]["prompt_sha256"] == want + + +def test_matching_digest_resumes(tmp_settings) -> None: + path = _write(tmp_settings.workspace_path() / "dg.yaml", "Pay 42?") + with pytest.raises(ApprovalRequired) as paused: + run_workflow_file(path, settings=tmp_settings, persist=True) + state = resume_run(paused.value.run_id, settings=tmp_settings, decisions={"g": "approve"}) + assert state.status == "succeeded" + assert state.output_keys["summary"] == "ok" + + +def test_changed_prompt_refuses_then_reapproves(tmp_settings) -> None: + path = _write(tmp_settings.workspace_path() / "dg.yaml", "Pay 42?") + with pytest.raises(ApprovalRequired) as paused: + run_workflow_file(path, settings=tmp_settings, persist=True) + run_id = paused.value.run_id + _write(path, "Pay 4200?") + with pytest.raises(ReapprovalRequired) as refused: + resume_run(run_id, settings=tmp_settings, decisions={"g": "approve"}) + exc = refused.value + assert exc.reason == "reapprove_required" + assert exc.expected_sha256 == approval_prompt_digest("Pay 42?") + assert exc.actual_sha256 == approval_prompt_digest("Pay 4200?") + assert exc.state.status == "paused" + assert exc.state.pending["prompt_sha256"] == exc.actual_sha256 + refusals = [ + e for e in _audit_events(tmp_settings.home, run_id) if e["event"] == "decision_refused" + ] + assert refusals and refusals[-1]["reason"] == "reapprove_required" + # Approving the prompt now on screen goes through. + state = resume_run(run_id, settings=tmp_settings, decisions={"g": "approve"}) + assert state.status == "succeeded" + + +def test_reject_is_never_blocked_by_digest(tmp_settings) -> None: + path = _write(tmp_settings.workspace_path() / "dg.yaml", "Pay 42?") + with pytest.raises(ApprovalRequired) as paused: + run_workflow_file(path, settings=tmp_settings, persist=True) + _write(path, "Pay 4200?") + state = resume_run(paused.value.run_id, settings=tmp_settings, decisions={"g": "reject"}) + assert state.status == "succeeded" + assert state.output_keys["summary"] == "denied" + + +def test_legacy_run_without_digest_resumes_with_warning( + tmp_settings, caplog: pytest.LogCaptureFixture +) -> None: + path = _write(tmp_settings.workspace_path() / "dg.yaml", "Pay 42?") + with pytest.raises(ApprovalRequired) as paused: + run_workflow_file(path, settings=tmp_settings, persist=True) + run_id = paused.value.run_id + run_file = _run_file(tmp_settings.home, run_id) + record = json.loads(run_file.read_text(encoding="utf-8")) + del record["pending"]["prompt_sha256"] + run_file.write_text(json.dumps(record), encoding="utf-8") + with caplog.at_level(logging.WARNING): + state = resume_run(run_id, settings=tmp_settings, decisions={"g": "approve"}) + assert state.status == "succeeded" + assert "paused before 2.0.11" in caplog.text + + +def test_cli_mismatch_exit_code_and_reason(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + home = tmp_path / ".readyagents" + monkeypatch.setenv("READYAGENTS_HOME", str(home)) + monkeypatch.chdir(tmp_path) + clear_settings_cache() + path = _write(tmp_path / "dg.yaml", "Pay 42?") + paused = runner.invoke(app, ["run", str(path), "--json"]) + assert paused.exit_code == 2, paused.stdout + paused.stderr + run_id = json.loads(paused.stdout)["run_id"] + _write(path, "Pay 4200?") + refused = runner.invoke(app, ["resume", run_id, "--approve", "g", "--json"]) + assert REAPPROVE_EXIT_CODE == 3 + assert refused.exit_code == REAPPROVE_EXIT_CODE, refused.stdout + refused.stderr + payload = json.loads(refused.stdout) + assert payload["ok"] is False + assert payload["error"] == "ReapprovalRequired" + assert payload["reason"] == "reapprove_required" + assert payload["status"] == "paused" + assert payload["prompt"] == "Pay 4200?" + assert payload["expected_sha256"] == approval_prompt_digest("Pay 42?") + # Text mode: reason on stderr, same exit code. Re-edit so it mismatches again. + _write(path, "Pay 9?") + text = runner.invoke(app, ["resume", run_id, "--approve", "g"]) + assert text.exit_code == REAPPROVE_EXIT_CODE + assert "reapprove_required" in text.stderr + ok = runner.invoke(app, ["resume", run_id, "--approve", "g"]) + assert ok.exit_code == 0, ok.stdout + ok.stderr + clear_settings_cache()