diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..5423e735 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,48 @@ +name: Docs + +# Build the MkDocs Material site and publish it to the `gh-pages` branch via +# `mkdocs gh-deploy` — the same mechanism UiPath/uipath-python uses. This workflow +# only pushes a branch (needs `contents: write`); it never calls the Pages API, so +# it succeeds even before Pages is switched on. +# +# One-time setup (org owner): the UiPath org blocks Pages *creation*, so enable +# Pages for this repo once — Settings → Pages → Build and deployment → +# Source: "Deploy from a branch" → branch `gh-pages` / `/ (root)`. That's the same +# (legacy, branch-based) configuration uipath-python already runs on. +on: + push: + branches: [main] + paths: + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: docs-gh-pages + cancel-in-progress: true + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Install social-card system libraries + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libcairo2 libfreetype6 libjpeg-turbo8 libpng16-16 pngquant + - name: Install MkDocs Material + run: pip install "mkdocs-material[imaging]>=9.5,<10" + - name: Configure git identity for gh-pages commits + run: | + git config user.name "coder-eval" + git config user.email "coder-eval@uipath.com" + - name: Build strictly and publish to gh-pages + run: mkdocs gh-deploy --force --strict diff --git a/.gitignore b/.gitignore index 3ae7ab95..949f035d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# MkDocs build output +site/ +.cache/ + # Python-generated files __pycache__/ *.py[oc] diff --git a/README.md b/README.md index 5784bdf4..06e72a0b 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,33 @@ -# coder_eval — evaluate AI coding agents & their skills +# Coder Eval — evaluate & benchmark AI coding agents and Claude Code skills [![PyPI](https://img.shields.io/pypi/v/coder-eval.svg)](https://pypi.org/project/coder-eval/) +[![Docs](https://img.shields.io/badge/docs-uipath.github.io%2Fcoder__eval-1f6feb.svg)](https://uipath.github.io/coder_eval/) [![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE) [![Python 3.13+](https://img.shields.io/badge/python-3.13%2B-blue.svg)](https://www.python.org/downloads/) [![CI](https://github.com/UiPath/coder_eval/actions/workflows/pr-checks.yml/badge.svg)](https://github.com/UiPath/coder_eval/actions/workflows/pr-checks.yml) [![Code style: Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) -A framework for evaluating AI coding agents **and their skills** — built for CLI +**Coder Eval** (`pip install coder-eval` / `uv tool install coder-eval`) is an open-source framework for +**evaluating and benchmarking AI coding agents and their skills** — built for CLI and skill builders — with sandboxing, reproducibility, and data-driven analysis. -Not an "agentic coding" benchmark: it measures how effective your CLI and skills -are when used by coding agents. +It runs a real agent (**Claude Code**, **Codex**, or **Google Antigravity / +Gemini**) in a sandbox against declarative YAML tasks, then scores the files and +commands it actually produced. Not an "agentic coding" benchmark: it measures how +effective your CLI and skills are when used by coding agents. + +Reach for it when you want to **test whether a Claude Code skill triggers**, +**A/B-test Claude Code vs. Codex vs. Gemini** (or model vs. model, prompt vs. +prompt), or **gate CI on coding-agent quality**. Unlike fixed datasets (SWE-bench, +SkillsBench) that rank models on a shared leaderboard, Coder Eval evaluates the +tasks, skills, and workflows *you* ship — with weighted 0.0–1.0 criteria, a +`skill_triggered` activation check, an A/B experiment layer, and per-tool cost +telemetry. See [How it compares](https://uipath.github.io/coder_eval/comparison/). +📚 **Full docs:** **[uipath.github.io/coder_eval](https://uipath.github.io/coder_eval/)**.

- coder_eval running the hello_date task: a sandboxed agent writes and runs a script from a YAML task, then the scored result is browsed in evalboard + Coder Eval running the hello_date task: a sandboxed agent writes and runs a script from a YAML task, then the scored result is browsed in evalboard

-> **The Coding Agents Gym.** A sandboxed, reproducible framework to evaluate, -> benchmark, and A/B-test AI coding agents — Claude Code, Codex, and Google -> Antigravity (Gemini) today, any agent via a plugin SPI — with declarative -> YAML tasks and weighted scoring. - - **Declarative YAML tasks** with pinned dependencies and clear success criteria - **Sandboxed execution** in isolated environments with resource limits - **Weighted, continuous scoring** (0.0–1.0) with fractional credit and thresholds @@ -30,14 +38,14 @@ are when used by coding agents. ## What you can do with it -- **Benchmark coding agents** — score an agent across a suite of tasks with weighted, pass/fail thresholds +- **Benchmark coding agents** — score an agent across a suite of tasks with weighted scoring and pass/fail thresholds - **Compare models & configs** — A/B-test Claude vs. Codex vs. Gemini, model vs. model, tool-on vs. tool-off, prompt vs. prompt - **Evaluate skills** — verify an agent actually engages a target skill (`skill_triggered`) and score skill-driven suites (SkillsBench-style) - **Keep skills up to date in CI** — re-validate your skills on every change or on a schedule; catch silent regressions when models, prompts, or the skills themselves drift - **Gate CI on agent quality** — run the suite in GitHub Actions and fail the build on regressions - **Bring your own dataset** — fan one task out over many rows for larger benchmark suites -> **Keeping skills fresh?** Run coder_eval as a scheduled GitHub Actions job so your +> **Keeping skills fresh?** Run Coder Eval as a scheduled GitHub Actions job so your > skills are continuously re-evaluated against the latest model — a skill that quietly > stops triggering surfaces as a failing criterion before your users hit it. See > **[Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md)**. @@ -53,7 +61,9 @@ git clone https://github.com/UiPath/coder_eval.git cd coder_eval uv sync --extra dev # install core + dev tools -cp .env.example .env # then set ANTHROPIC_API_KEY +cp .env.example .env # then set ANTHROPIC_API_KEY — or skip that: an + # existing Claude Code login (`claude login`) is + # picked up automatically uv run coder-eval plan tasks/hello_date.yaml # validate (no tokens spent) uv run coder-eval run tasks/hello_date.yaml # run your first evaluation @@ -67,11 +77,23 @@ The optional `[uipath]` extra (`uv sync --extra dev --extra uipath`) adds the in required). Without it the framework runs end-to-end; uipath-dependent features fail at dispatch with a clear hint. -> **Using coder_eval in CI or another project?** Install the published package: -> `pip install coder-eval` (or `uv add coder-eval`; extras install the same way — -> `pip install "coder-eval[codex,antigravity]"`). In a real CI gate, pin to a -> specific released version so a harness upgrade can't silently move your results. -> See [Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md) for the full setup. +**Using Coder Eval in CI or another project?** Install the published package +instead of cloning: + +```bash +uv tool install coder-eval # puts the `coder-eval` CLI on your PATH, + # in its own isolated environment + +uv tool install "coder-eval[codex,antigravity]" # same, with agent extras +coder-eval --version # verify the install +``` + +To add it as a project dependency instead: `uv add coder-eval` or +`pip install coder-eval`. In a real CI gate, pin to a specific released version +so a harness upgrade can't silently move your results. (The example `tasks/` +live in this repo — clone it or point the CLI at your own task files.) See +[Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md) for +the full setup. ## Telemetry @@ -99,17 +121,18 @@ at dispatch with a clear hint. ## How it compares -- **vs. SWE-bench and fixed benchmarks** — SWE-bench is a fixed dataset of GitHub - issues; coder_eval is a *framework* for authoring your own tasks in declarative - YAML, so you evaluate the skills and workflows you care about (and can still wrap - a fixed dataset via [Bring Your Own Dataset](docs/BYOD.md)). -- **vs. LLM-output eval harnesses (e.g. OpenAI Evals)** — those grade a model's text; - coder_eval runs a full **agent** in a **sandbox** with real tool use and multi-turn - dialog, then scores the files and commands it actually produced (continuous - 0.0–1.0) — not just a judge over a string. +- **vs. fixed benchmarks (SWE-bench, SkillsBench)** — they score a canonical dataset; + Coder Eval scores *your* tasks with continuous 0.0–1.0 weighted criteria (and can + still wrap a fixed dataset via [Bring Your Own Dataset](docs/BYOD.md)). +- **vs. large-scale / RL harnesses (Harbor)** — Harbor targets scale and RL rollouts; + Coder Eval targets weighted, skill-aware suites gated in CI. +- **vs. model-output eval tools (OpenAI Evals)** — they grade model text; Coder Eval + runs a full agent in a sandbox and scores the files and commands it produced. - **vs. hand-rolled scripts** — reproducible sandboxes, weighted criteria, cost/token telemetry, A/B experiments, and CI-ready pass/fail gates out of the box. +See the full [comparison — with sources](https://uipath.github.io/coder_eval/comparison/). + ## Task Definition A task is a YAML file: a prompt, the agent config, a sandbox, and success criteria. @@ -159,12 +182,12 @@ extension points (new criteria, new agents). ## Known limits & non-goals -- **Not a fixed benchmark or leaderboard** — coder_eval scores *your* tasks and ships +- **Not a fixed benchmark or leaderboard** — Coder Eval scores *your* tasks and ships example tasks, not a canonical scored dataset. - **Tasks execute real code** — run untrusted tasks only under the container driver (see [Docker Isolation](docs/DOCKER_ISOLATION.md)); the `tempdir` driver is not a security boundary. -- **Bring your own model credentials** — Anthropic, Bedrock, or Gemini keys; coder_eval +- **Bring your own model credentials** — Anthropic, Bedrock, or Gemini keys; Coder Eval does not proxy or supply model access. - **Python 3.13+ only.** diff --git a/docs/AB_EXPERIMENTS.md b/docs/AB_EXPERIMENTS.md index 9bc0da09..52c56269 100644 --- a/docs/AB_EXPERIMENTS.md +++ b/docs/AB_EXPERIMENTS.md @@ -1,3 +1,10 @@ +--- +description: >- + A/B-test AI coding agents with coder_eval's experiment layer — Claude Code vs. + Codex vs. Gemini, model vs. model, skill on vs. off, prompt vs. prompt — on + identical tasks. +--- + # A/B Experiment Guide How to run the same tasks across multiple configuration variants ("arms") and diff --git a/docs/BYOD.md b/docs/BYOD.md index f2d461d1..5284d9cd 100644 --- a/docs/BYOD.md +++ b/docs/BYOD.md @@ -1,3 +1,10 @@ +--- +description: >- + Use a custom Docker image with coder_eval — extend the base coder-eval-agent + image with your own dependencies and tools, then point task configuration at + it. +--- + # Bring Your Own Docker (BYOD) The BYOD feature allows customers to use custom Docker images that extend the base `coder-eval-agent` image, enabling them to add custom dependencies and tools while maintaining the latest coder-eval codebase. diff --git a/docs/CODEX_AGENT_GUIDE.md b/docs/CODEX_AGENT_GUIDE.md index e1629b98..eb2cc160 100644 --- a/docs/CODEX_AGENT_GUIDE.md +++ b/docs/CODEX_AGENT_GUIDE.md @@ -1,8 +1,15 @@ -# Codex Agent Implementation +--- +description: >- + Run OpenAI Codex as the agent under evaluation in coder_eval — installation, + authentication, task configuration, and how Codex telemetry maps to sandboxed, + weighted scoring. +--- + +# Running OpenAI Codex in coder_eval ## Overview -A new `CodexAgent` has been added to coder_eval that integrates OpenAI's Codex SDK. The implementation mirrors the structure of `ClaudeCodeAgent` and provides seamless integration with the evaluation framework. +coder_eval can run OpenAI's Codex as the agent under evaluation, via the official Codex SDK. The `CodexAgent` mirrors the structure of `ClaudeCodeAgent` and plugs into the same sandbox, scoring, and telemetry pipeline — set `agent.type: codex` in a task and the rest of the framework works unchanged. ## Setup diff --git a/docs/DOCKER_ISOLATION.md b/docs/DOCKER_ISOLATION.md index 0cdc600d..6deba4a5 100644 --- a/docs/DOCKER_ISOLATION.md +++ b/docs/DOCKER_ISOLATION.md @@ -1,9 +1,14 @@ +--- +description: >- + Run each coder_eval task in its own fresh Docker container — strong host + isolation, a pinned reproducible agent runtime, and custom images for + task-specific dependencies. +--- + # Docker Isolation Run each evaluation task inside its own fresh container. Strong host isolation and a pinned, reproducible agent runtime. -> Supersedes the agent-side FS perimeter flag from #199 (reverted in 9fe4320). The container boundary subsumes what that flag tried to do at the agent level. - ## When to use Set `sandbox.driver: docker` on a task (or pass `--driver docker` on the CLI — diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 48e93ff1..e86c3c02 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -1,3 +1,10 @@ +--- +description: >- + Full schema reference for coder_eval task YAML — agent config, sandboxes, run + limits, dataset fan-out, and all 14 success criterion types with weighted + 0.0–1.0 scoring. +--- + # Task Definition Guide Complete reference for defining evaluation tasks in coder_eval. @@ -128,6 +135,8 @@ an error. - `codex` — OpenAI Codex agent (requires `[codex]` extra; set `CODEX_API_KEY` and optional `CODEX_BASE_URL` environment variables). - `none` — No-op agent: no coding agent runs and no model API call is made. See [No-op / System Tasks](#no-op--system-tasks-type-none) below. + + ### No-op / System Tasks (`type: none`) Set `agent: {type: none}` to run a task with **no coding agent** — "coder-eval @@ -159,7 +168,7 @@ Contract (enforced at load): a `type: none` task must declare no `initial_prompt every criterion must be agent-independent — criteria that inspect the agent trajectory (`command_executed`, `skill_triggered`, `reference_comparison`, `commands_efficiency`) are rejected. A worked example lives at -[`tasks/agentless_smoke_test.yaml`](../tasks/agentless_smoke_test.yaml). +[`tasks/agentless_smoke_test.yaml`](https://github.com/UiPath/coder_eval/blob/main/tasks/agentless_smoke_test.yaml). ### `max_turns`, `task_timeout`, `turn_timeout` location @@ -590,7 +599,7 @@ Checks whether the agent executed specific tools/commands during evaluation. Ins Evaluates a UiPath agent against a named evaluation set. **Fractional scoring:** metrics passed / total metrics. -> The `uipath` CLI must be available **inside the sandbox** (typically declared in the task's own Python deps). This is independent of the host's optional `coder-eval[uipath]` extra — see the install matrix in [README.md](../README.md#installation). +> The `uipath` CLI must be available **inside the sandbox** (typically declared in the task's own Python deps). This is independent of the host's optional `coder-eval[uipath]` extra — see the install matrix in [README.md](https://github.com/UiPath/coder_eval/blob/main/README.md#quick-start). ```yaml - type: "uipath_eval" diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 670c2395..844c040a 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -1,3 +1,10 @@ +--- +description: >- + Complete coder_eval reference — every CLI command and flag, configuration + layers and -D overrides, environment variables, run outputs, and reports for + evaluating AI coding agents. +--- + # coder_eval User Guide The full command, configuration, and output reference. For a gentle introduction @@ -81,6 +88,8 @@ in `.claude/commands/`, available when using Claude Code in this repository: | `/coder-eval-run-analysis ` | Analyze evaluation runs and suggest improvements to tasks, config, and prompts. Works at task, variant, or run scope. | | `/coder-eval-task-create` | Create evaluation task YAML files from a natural language description. | + + ## API Routing & Benchmarking `coder-eval` supports two API routing modes, selected via `--backend` or the diff --git a/docs/comparison.md b/docs/comparison.md new file mode 100644 index 00000000..f46b39cc --- /dev/null +++ b/docs/comparison.md @@ -0,0 +1,153 @@ +--- +title: "Coder Eval vs. SWE-bench, SkillsBench, Harbor & OpenAI Evals — how it compares" +description: >- + How Coder Eval compares to SWE-bench, SkillsBench, Harbor, and OpenAI Evals for + evaluating AI coding agents and Claude Code skills — with when to choose each. + Grounded in each project's own docs. +--- + +# How Coder Eval compares + +**Coder Eval** is a framework for evaluating AI coding agents and their skills. It +runs an agent (Claude Code, Codex, Gemini) in a sandbox against declarative YAML +tasks and scores the files and commands it produces on a weighted 0.0–1.0 scale, +with cost/token telemetry, an A/B experiment layer, skill-activation checks, and CI +pass/fail gates. + +The tools below overlap with parts of that scope. This page describes what each is +designed for and when to choose it. Every claim about another project is grounded +in its own documentation; see [Sources](#sources). + +## At a glance + +| | Coder Eval | SWE-bench | SkillsBench | Harbor | OpenAI Evals | Hand-rolled | +| --- | --- | --- | --- | --- | --- | --- | +| **What it grades** | Files + commands the agent produced | Whether a patch passes the repo's tests | Skill value on a fixed task set | Agent task success in sandboxes | Model text output | Whatever you wire up | +| **Task source** | Your own (YAML) | Fixed benchmark (+ collection script) | Fixed (87 tasks / 8 domains) | Your own (framework) + Terminal-Bench 2.0 | Your own (YAML/JSON) + registry | Manual | +| **Runs a real agent + tools** | ✅ Claude Code, Codex, Gemini | Runs your patch/scaffold | ✅ multi-harness | ✅ Claude Code, OpenHands, Codex | ❌ grades model output | ❌ | +| **Sandboxed & reproducible** | ✅ tempdir / Docker | ✅ Docker | ✅ deterministic verifiers | ✅ Docker + cloud (Daytona/Modal) | — (grades text) | ❌ | +| **Scoring** | Weighted 0.0–1.0 + thresholds | Pass/fail (tests) | Pass/fail (verifiers) | Task-level | Match / model-graded | ❌ | +| **A/B experiments (model / tool / prompt)** | ✅ built-in | ❌ | Skills on/off only | — | Compare model versions | ❌ | +| **Cost / token telemetry** | ✅ per tool call | ❌ | ❌ | — | limited | ❌ | +| **Skill-activation testing** | ✅ `skill_triggered` | ❌ | Measures effect, not trigger | ❌ | ❌ | ❌ | +| **CI pass/fail gate** | ✅ built-in | ❌ | ❌ | ❌ | ❌ | ❌ | +| **Results dashboard** | ✅ evalboard | ❌ | ❌ | ❌ | ❌ | ❌ | +| **Primary interface** | Python + YAML | Python harness + HF dataset | Research benchmark | Python CLI | Python + YAML/JSON | Your choice | + +## vs. SWE-bench and fixed benchmarks + +[SWE-bench](https://www.swebench.com/) is a fixed benchmark of real GitHub issues. +A model generates a patch, which is graded by running the repository's tests in +Docker; SWE-bench Verified is a human-validated subset of 500 problems, and there +are additional variants (Lite, Multimodal, Multilingual). It includes a +data-collection procedure for building tasks from other repositories, but its +primary use is a shared leaderboard on a canonical dataset. + +Coder Eval is a framework for authoring tasks in YAML, with continuous 0.0–1.0 +scoring and cost telemetry rather than a single pass/fail. A fixed dataset can be +wrapped as tasks via [Bring Your Own Dataset](BYOD.md). + +**Choose SWE-bench** for a standardized model leaderboard on patch-and-test tasks. +**Choose Coder Eval** to evaluate your own tasks, agents, and skills and to gate CI +on them. + +## vs. SkillsBench and skill benchmarks + +[SkillsBench](https://arxiv.org/abs/2602.12670) is a fixed research benchmark for +agent skills: 87 tasks across 8 domains, each paired with curated skills and +deterministic verifiers. It compares agents with and without the curated skills +across 18 model-harness configurations and reports a 33.9% → 50.5% average +pass-rate change. It measures whether skills improve performance on that task set. + +Coder Eval addresses a related but distinct question about a specific set of skills: + +- **Activation vs. effectiveness.** Coder Eval's `skill_triggered` criterion checks + whether the agent engaged the skill (Claude's `Skill` tool call, or Codex reading + the skill's files). SkillsBench measures the downstream effect on task success. + The two are different signals: a skill that does not activate cannot contribute. +- **Reusable in CI.** A skill-on/off comparison can be built with the + [experiment layer](AB_EXPERIMENTS.md), gated on each change, and scheduled to + detect regressions over time. + +**Choose SkillsBench** for a standardized measure of skill value across domains. +**Choose Coder Eval** to author, activation-test, and regression-check your own +skills. + +## vs. Harbor + +[Harbor](https://www.harborframework.com/) is a Python framework from the +Terminal-Bench team for evaluating and optimizing agents and language models. It +runs agents (Claude Code, OpenHands, Codex CLI) in sandboxes using Docker and cloud +providers (Daytona, Modal, LangSmith), is the official harness for Terminal-Bench +2.0, can run across large numbers of environments in parallel, and can generate +rollouts for RL optimization. Its emphasis is scale and optimization. + +Coder Eval focuses on scoring and workflow for coding-agent and skill suites: + +- 14 weighted criterion types (file/regex/JSON checks, `run_command`, AST/token + similarity, LLM- and agent-judges) on a 0.0–1.0 scale with per-criterion + thresholds. +- An [experiment layer](AB_EXPERIMENTS.md) for model/tool/prompt A/Bs, + `skill_triggered` activation checks, per-tool cost/token telemetry, CI pass/fail + gates, and the evalboard dashboard. + +**Choose Harbor** for large-scale agent evaluation and RL-optimization rollouts, +particularly around Terminal-Bench. **Choose Coder Eval** for weighted, skill-aware +task suites gated in CI. + +## vs. OpenAI Evals + +[OpenAI Evals](https://github.com/openai/evals) is a framework and registry for +evaluating model output. Data is provided in JSON and parameters in YAML, and +grading uses exact/pattern match or model-graded templates. It evaluates model +output rather than running an autonomous coding agent with tool use in a sandbox, +and contributions of custom-code evals are restricted (model-graded YAML evals are +the accepted path). + +Coder Eval runs the full agent and scores the files and commands it produces, with +per-tool cost/token telemetry and skill-activation checks. + +**Choose OpenAI Evals** to grade model output and compare model versions. **Choose +Coder Eval** to evaluate agent behavior — the tools it calls, the files it writes, +the commands it runs, and the skills it triggers. + +## vs. hand-rolled scripts + +A `claude -p` loop is quick to start, but a durable harness requires implementing +sandboxing, weighted criteria, retries, cost/token accounting, A/B configuration, a +results view, and CI reporting. Coder Eval provides these components, so work can +focus on tasks and criteria rather than harness infrastructure. + +## When to choose Coder Eval + +- Evaluating how well agents use a CLI or Claude Code skills. +- A/B-testing models, tools, or prompts on the same tasks (Claude Code vs. Codex vs. Gemini). +- Detecting skill regressions when a model or prompt changes. +- Gating CI on coding-agent quality. +- Producing reproducible, sandboxed runs scored per file and per command, with a dashboard for review. + +## Next steps + +- [Tutorial 01 — Your First Evaluation](tutorials/01-first-evaluation.md) +- [Tutorial 02 — Running coder_eval in CI](tutorials/02-ci-pipeline.md) +- [A/B Experiments](AB_EXPERIMENTS.md) +- [Bring Your Own Dataset](BYOD.md) + +## Sources + +Claims about other projects are grounded in their own documentation, retrieved +July 2026: + +- SWE-bench — [swebench.com](https://www.swebench.com/) and the + [SWE-bench repository](https://github.com/SWE-bench/SWE-bench) (real GitHub + issues, Docker-reproducible test-based evaluation, Verified = 500 problems). +- SkillsBench — [*Benchmarking How Well Agent Skills Work Across Diverse Tasks*](https://arxiv.org/abs/2602.12670) + (87 tasks / 8 domains, deterministic verifiers, 18 model-harness configs, + 33.9% → 50.5% pass-rate lift). +- Harbor — [harborframework.com](https://www.harborframework.com/) and the + [harbor-framework/harbor repository](https://github.com/harbor-framework/harbor) + (Python framework from the Terminal-Bench creators; Docker + multi-cloud sandboxes; + official Terminal-Bench 2.0 harness; parallel across thousands of environments; + RL-optimization rollouts). +- OpenAI Evals — [openai/evals](https://github.com/openai/evals) (framework + + registry; JSON data + YAML params; match and model-graded evals). diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..5eca53e6 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,100 @@ +--- +title: "Evaluate AI coding agents & Claude Code skills — Coder Eval" +description: >- + Coder Eval is an open-source framework to evaluate, benchmark, and A/B-test AI + coding agents and Claude Code skills in a sandbox — declarative YAML tasks, + weighted scoring, cost/token telemetry, and CI gates for Claude Code, Codex, + and Gemini. +--- + +# Evaluate AI coding agents & Claude Code skills — Coder Eval + +**Coder Eval** (`pip install coder-eval` / `uv tool install coder-eval`) is an open-source framework for +**evaluating AI coding agents and their skills** — built for CLI and skill +builders — with sandboxing, reproducibility, and data-driven analysis. It is not +an "agentic coding" benchmark: it measures how effective *your* CLI and skills +are when used by coding agents such as **Claude Code**, **Codex**, and **Google +Antigravity (Gemini)**. + +If you have ever asked *"how do I test whether my Claude Code skill actually +triggers?"*, *"how do I benchmark Claude Code vs. Codex on my own tasks?"*, or +*"how do I gate CI on coding-agent quality?"* — this is the framework for that. + +

+ Coder Eval running a sandboxed coding-agent evaluation from a YAML task and browsing the scored result in evalboard +

+ +## What Coder Eval does + +- **Declarative YAML tasks** with pinned dependencies and clear success criteria +- **Sandboxed execution** in isolated environments with resource limits +- **Weighted, continuous scoring** (0.0–1.0) with fractional credit and thresholds +- **Many criterion types** — from file checks to code similarity and LLM-graded rubrics +- **Agent abstraction** — Claude Code, Codex, and Antigravity (Gemini) today, extensible via a plugin SPI +- **Experiment layer** — A/B agent configs (models, tools, prompts) side-by-side +- **Full telemetry** — every tool call, token counts, and cost, with real-time streaming + +## Use cases + +- **Benchmark coding agents** — score an agent across a suite of tasks with weighted pass/fail thresholds +- **Compare models & configs** — A/B-test Claude Code vs. Codex vs. Gemini, model vs. model, tool-on vs. tool-off, prompt vs. prompt +- **Test whether a Claude Code skill triggers** — verify an agent actually engages a + target skill (`skill_triggered`) and score skill-driven suites (SkillsBench-style) +- **Keep skills fresh in CI** — re-validate skills on every change or on a schedule; catch silent regressions when models, prompts, or the skills themselves drift +- **Gate CI on agent quality** — run the suite in GitHub Actions and fail the build on regressions +- **Bring your own dataset** — fan one task out over many rows for larger benchmark suites + +## Quick start + +```bash +# 1. Install the coder-eval CLI on your PATH (isolated environment) +uv tool install coder-eval + +# 2. Grab the runnable example tasks +git clone https://github.com/UiPath/coder_eval.git +cd coder_eval + +# 3. Credentials — optional if you're already logged in to Claude Code +# (`claude login`, reused automatically); otherwise set ANTHROPIC_API_KEY +cp .env.example .env + +# 4. Validate, run, and read your first evaluation +coder-eval plan tasks/hello_date.yaml # validate (no tokens spent) +coder-eval run tasks/hello_date.yaml # run your first evaluation +coder-eval report runs/latest # view the result +``` + +Prefer Coder Eval as a project dependency instead of a CLI tool? `uv add +coder-eval` or `pip install coder-eval`. Hacking on Coder Eval itself? Clone, +`uv sync --extra dev`, and prefix commands with `uv run`. + +New here? Start with **[Tutorial 01 — Your First Evaluation](tutorials/01-first-evaluation.md)**. + +## Where to go next + +| Guide | What's in it | +| --- | --- | +| [Tutorials](tutorials/README.md) | Step-by-step walkthroughs — start here | +| [User Guide](USER_GUIDE.md) | Full CLI, configuration, output, and environment-variable reference | +| [Task Definition Guide](TASK_DEFINITION_GUIDE.md) | The task-file schema — all criterion types, scoring, templates | +| [A/B Experiments](AB_EXPERIMENTS.md) | Compare models / tools / prompts across the same tasks | +| [Bring Your Own Dataset](BYOD.md) | Fan a single task out over a dataset | +| [Codex Agent Guide](CODEX_AGENT_GUIDE.md) | Running the Codex agent | +| [Docker Isolation](DOCKER_ISOLATION.md) | The container sandbox driver | +| [How it compares](comparison.md) | Coder Eval vs. SWE-bench, SkillsBench, Harbor, OpenAI Evals, and hand-rolled harnesses | + +## How Coder Eval compares + +- **vs. SWE-bench and fixed benchmarks** — SWE-bench is a fixed dataset; Coder Eval + is a *framework* for authoring your own tasks in YAML, so you evaluate the skills + and workflows you care about (and can still wrap a fixed dataset via + [Bring Your Own Dataset](BYOD.md)). +- **vs. other agent-eval frameworks (e.g. Harbor) and LLM-eval tools (OpenAI Evals)** — + OpenAI Evals grades model text; Harbor targets large-scale agent eval and RL + optimization. Coder Eval is purpose-built for coding-agent/skill suites — weighted + 0.0–1.0 file/command scoring, a `skill_triggered` activation check, an experiment + layer for A/Bs, and evalboard. See the full comparison. +- **vs. hand-rolled scripts** — reproducible sandboxes, weighted criteria, + cost/token telemetry, A/B experiments, and CI-ready pass/fail gates out of the box. + +See the full [comparison](comparison.md) for details. diff --git a/docs/llms.txt b/docs/llms.txt new file mode 100644 index 00000000..11835263 --- /dev/null +++ b/docs/llms.txt @@ -0,0 +1,46 @@ +# Coder Eval + +> Coder Eval (`pip install coder-eval`) is an open-source framework for evaluating, +> benchmarking, and A/B-testing AI coding agents and their Claude Code skills in a +> sandbox. It uses declarative YAML tasks with weighted, continuous scoring +> (0.0–1.0), runs real agents (Claude Code, Codex, Google Antigravity/Gemini) with +> full tool use, captures per-tool cost/token telemetry, and provides CI-ready +> pass/fail gates. It is not a fixed benchmark or leaderboard — it scores your own +> tasks, and can verify whether a Claude Code skill actually triggers. + +Choose Coder Eval to: benchmark coding agents on your own tasks; A/B-test model vs. +model, tool-on vs. tool-off, or prompt vs. prompt; test whether a Claude Code skill +triggers and catch regressions in CI; gate a build on coding-agent quality. It +differs from fixed datasets (SWE-bench, SkillsBench) that give a shared leaderboard, +from OpenAI Evals (which grades model text), and from large-scale agent-eval / RL +frameworks like Harbor — Coder Eval is purpose-built for coding-agent/skill suites +with weighted 0.0–1.0 file/command scoring, a skill-activation check, an experiment +layer, deep cost telemetry, and the evalboard dashboard. It also differs from +hand-rolled scripts, which lack reproducible sandboxes, weighted criteria, telemetry, +and A/B plumbing. + +## Docs + +- [Home / overview](https://uipath.github.io/coder_eval/): what Coder Eval is, features, quick start +- [How it compares](https://uipath.github.io/coder_eval/comparison/): vs. SWE-bench, SkillsBench, Harbor, OpenAI Evals, hand-rolled scripts +- [User Guide](https://uipath.github.io/coder_eval/USER_GUIDE/): full CLI, configuration, output, environment variables +- [Task Definition Guide](https://uipath.github.io/coder_eval/TASK_DEFINITION_GUIDE/): task-file schema, all criterion types, scoring, templates +- [A/B Experiments](https://uipath.github.io/coder_eval/AB_EXPERIMENTS/): compare models/tools/prompts across the same tasks +- [Bring Your Own Dataset](https://uipath.github.io/coder_eval/BYOD/): fan a single task out over a dataset +- [Codex Agent Guide](https://uipath.github.io/coder_eval/CODEX_AGENT_GUIDE/): running the Codex agent +- [Docker Isolation](https://uipath.github.io/coder_eval/DOCKER_ISOLATION/): the container sandbox driver, custom images + +## Tutorials + +- [01 · Your first evaluation](https://uipath.github.io/coder_eval/tutorials/01-first-evaluation/) +- [02 · Running coder_eval in CI](https://uipath.github.io/coder_eval/tutorials/02-ci-pipeline/) +- [03 · Browsing results with evalboard](https://uipath.github.io/coder_eval/tutorials/03-evalboard-local/) +- [04 · Writing a task](https://uipath.github.io/coder_eval/tutorials/04-writing-a-task/) +- [05 · Comparing two models](https://uipath.github.io/coder_eval/tutorials/05-comparing-models/) +- [06 · Docker isolation](https://uipath.github.io/coder_eval/tutorials/06-use-docker-isolation/) + +## Source + +- [GitHub repository](https://github.com/UiPath/coder_eval) +- [PyPI package](https://pypi.org/project/coder-eval/) +- [Changelog](https://github.com/UiPath/coder_eval/blob/main/CHANGELOG.md) diff --git a/docs/tutorials/01-first-evaluation.md b/docs/tutorials/01-first-evaluation.md index 0315ded4..e03d02e8 100644 --- a/docs/tutorials/01-first-evaluation.md +++ b/docs/tutorials/01-first-evaluation.md @@ -1,3 +1,10 @@ +--- +description: >- + Install coder-eval, point it at an API key or your Claude Code login, run a + built-in task in a sandbox, and read your first scored coding-agent result — + in about 5 minutes. +--- + # Tutorial 01 — Your First Evaluation By the end you'll have installed `coder-eval`, pointed it at an API key, run a @@ -18,6 +25,11 @@ cd coder_eval uv sync --extra dev ``` +> **Just want the CLI?** `uv tool install coder-eval` installs the released +> `coder-eval` command on your PATH in its own isolated environment — then run +> every command below without the `uv run` prefix. Keep the clone anyway: the +> example `tasks/` used in this tutorial live in the repo. + `uv sync` creates a `.venv/`. **Activate it** to run `coder-eval` directly and drop the `uv run` prefix used throughout this tutorial: diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index c6527814..7dc04eea 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -1,3 +1,10 @@ +--- +description: >- + Run coder_eval in GitHub Actions — install the CLI, execute your coding-agent + evaluation suite on demand or on a schedule, and gate merges on a pass/fail + verdict. +--- + # Tutorial 02 — Running coder_eval in CI Run your evaluation suite automatically in GitHub Actions: on demand, on a diff --git a/docs/tutorials/03-evalboard-local.md b/docs/tutorials/03-evalboard-local.md index 3560afa0..9392e997 100644 --- a/docs/tutorials/03-evalboard-local.md +++ b/docs/tutorials/03-evalboard-local.md @@ -1,3 +1,10 @@ +--- +description: >- + Browse coder_eval runs in evalboard, a local web UI — pass rates, per-task + detail, tool and message timelines, token costs, and artifact downloads, + straight from your filesystem. +--- + # Tutorial 03 — Browsing Results Locally with evalboard `coder-eval` writes results as JSON, markdown, and logs under `runs/`. **evalboard** diff --git a/docs/tutorials/04-writing-a-task.md b/docs/tutorials/04-writing-a-task.md index bba05de5..cdbfc514 100644 --- a/docs/tutorials/04-writing-a-task.md +++ b/docs/tutorials/04-writing-a-task.md @@ -1,3 +1,10 @@ +--- +description: >- + Write a coder_eval task YAML from scratch — prompt, isolated sandbox, turn + budget, and three success criteria — validate it without spending tokens, then + run and score it. +--- + # Tutorial 04 — Writing a Task By the end you'll have authored a task YAML from scratch — a prompt, an isolated diff --git a/docs/tutorials/05-comparing-models.md b/docs/tutorials/05-comparing-models.md index 7b891f23..ec3539c0 100644 --- a/docs/tutorials/05-comparing-models.md +++ b/docs/tutorials/05-comparing-models.md @@ -1,3 +1,10 @@ +--- +description: >- + A/B-test two Claude models on the same task with coder_eval's experiment layer + and read the cross-variant report — a fair, like-for-like coding-agent model + comparison. +--- + # Tutorial 05 — Comparing Two Models By the end you'll have A/B-tested two Claude models on the same task through the diff --git a/docs/tutorials/06-use-docker-isolation.md b/docs/tutorials/06-use-docker-isolation.md index c65bb1c4..9f0931e0 100644 --- a/docs/tutorials/06-use-docker-isolation.md +++ b/docs/tutorials/06-use-docker-isolation.md @@ -1,4 +1,11 @@ -# Tutorial 04 — Running Tasks in Docker Isolation +--- +description: >- + Switch a coder_eval task from the temp-dir sandbox to a fresh Docker container + per task — strong host isolation, a pinned toolchain, and task-specific + dependencies. +--- + +# Tutorial 06 — Running Tasks in Docker Isolation By default a task runs in a temp-dir sandbox on your host. This tutorial switches that to a fresh **Docker container per task** — strong host isolation and a @@ -190,7 +197,7 @@ COPY input/ /root/input/ # build context = this Dockerfile's parent d > tagged by `make docker-image`, so either works in the `FROM`.) Tasks whose > Dockerfile brings its own base — e.g. Fedora/`dnf` — use > inject-mode instead; see -> [Docker Isolation](../DOCKER_ISOLATION.md#tasks-that-bring-their-own-base-the-runtime-kit-coder-eval-runtime). +> [Docker Isolation](../DOCKER_ISOLATION.md#tasks-that-bring-their-own-base-image-the-runtime-kit-coder-eval-runtime). ## Next steps diff --git a/docs/tutorials/README.md b/docs/tutorials/README.md index ffad1bbb..22d95a43 100644 --- a/docs/tutorials/README.md +++ b/docs/tutorials/README.md @@ -1,3 +1,10 @@ +--- +description: >- + Step-by-step coder_eval tutorials — run your first AI coding-agent evaluation, + wire it into CI, browse results, write tasks, compare models, and isolate runs + in Docker. +--- + # Tutorials Step-by-step, task-oriented walkthroughs. Start here if you're new; for the diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..9952af4c --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,94 @@ +site_name: Coder Eval +site_description: >- + Evaluate & benchmark AI coding agents and Claude Code skills — a sandboxed, + reproducible framework with declarative YAML eval suites for Claude Code, + Codex & Gemini, A/B experiments, weighted scoring, and CI gates. +site_url: https://uipath.github.io/coder_eval/ +repo_url: https://github.com/UiPath/coder_eval +repo_name: UiPath/coder_eval +edit_uri: edit/main/docs/ +copyright: © 2026 UiPath — Apache License 2.0 + +theme: + name: material + features: + - navigation.instant + - navigation.tracking + - navigation.sections + - navigation.top + - search.suggest + - search.highlight + - content.code.copy + - content.action.edit + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: deep orange + accent: deep orange + toggle: + icon: material/weather-night + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: deep orange + accent: deep orange + toggle: + icon: material/weather-sunny + name: Switch to light mode + icon: + repo: fontawesome/brands/github + +# IDEAS.md is internal engineering backlog (links to source files) — not for the +# public docs site. +exclude_docs: | + IDEAS.md + +# The social plugin generates per-page Open Graph / Twitter card images (and +# the matching og:/twitter: meta tags) so shared links unfurl properly. It +# needs Cairo system libraries, so it is enabled only in CI (the Docs workflow +# installs the deps); local builds skip it. +plugins: + - search + - social: + enabled: !ENV [CI, false] + +markdown_extensions: + - admonition + - attr_list + - md_in_html + - tables + - toc: + permalink: true + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.superfences + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.tabbed: + alternate_style: true + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/UiPath/coder_eval + - icon: fontawesome/brands/python + link: https://pypi.org/project/coder-eval/ + +nav: + - Home: index.md + - Tutorials: + - Overview: tutorials/README.md + - 01 · Your first evaluation: tutorials/01-first-evaluation.md + - 02 · Running coder_eval in CI: tutorials/02-ci-pipeline.md + - 03 · Browsing results with evalboard: tutorials/03-evalboard-local.md + - 04 · Writing a task: tutorials/04-writing-a-task.md + - 05 · Comparing two models: tutorials/05-comparing-models.md + - 06 · Docker isolation: tutorials/06-use-docker-isolation.md + - Guides: + - User Guide: USER_GUIDE.md + - Task Definition Guide: TASK_DEFINITION_GUIDE.md + - A/B Experiments: AB_EXPERIMENTS.md + - Bring Your Own Dataset: BYOD.md + - Codex Agent Guide: CODEX_AGENT_GUIDE.md + - Docker Isolation: DOCKER_ISOLATION.md + - How it compares: comparison.md diff --git a/pyproject.toml b/pyproject.toml index d66c4bf3..24448327 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,8 +10,9 @@ keywords = [ "ai", "llm", "agent", "coding-agent", "evaluation", "eval", "evals", "benchmark", "swe-bench", "claude", "claude-code", "codex", "anthropic", "gemini", "antigravity", "sandbox", "code-generation", "agent-evaluation", - "llm-evaluation", "skills-evaluation", "claude-skills", "agent-skills", - "skillsbench", + "llm-evaluation", "llm-eval", "ai-evaluation", "skills-evaluation", + "claude-skills", "claude-code-skills", "agent-skills", "skillsbench", + "agent-testing", "llmops", ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/uv.lock b/uv.lock index a79a6d0a..b4616959 100644 --- a/uv.lock +++ b/uv.lock @@ -1684,11 +1684,11 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.3" +version = "0.6.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, ] [[package]]