feat: packaged CI gate — JUnit XML output + composite GitHub Action

.github/workflows/pr-checks.yml

@@ -831,3 +831,61 @@ jobs:
           name: byoa-live-output
           path: tmp/
           retention-days: 7
+
+  action-dogfood:
+    name: Action Dogfood (composite action, real API)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    # Skip on fork PRs where secrets aren't available (matches e2e-smoke).
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+
+      # The composite action is agent-agnostic and does NOT install a coding-agent
+      # runtime. The dogfood task uses the default claude-code agent, so provide
+      # Node + the Claude CLI here (as e2e-smoke does), before invoking the action.
+      - name: Set up Node.js 20
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
+        with:
+          node-version: "20"
+      - name: Install Claude CLI
+        run: npm install -g @anthropic-ai/claude-code
+
+      - name: Run coder-eval via local action
+        id: dogfood
+        uses: ./
+        with:
+          version: local
+          tasks: tasks/hello_date.yaml
+          model: claude-haiku-4-5-20251001
+          run-dir: runs/ci-action-dogfood
+          junit-path: runs/ci-action-dogfood/junit.xml
+          # Credentials go through the generic env passthrough (the only channel);
+          # ANTHROPIC_API_KEY reaching the run is proven by the API-backed task
+          # succeeding. A floor of 0.0 passes for any produced score (exercises
+          # the gate path green in CI without flakiness); the second line
+          # exercises multi-line env parsing.
+          minimum-task-score: "0.0"
+          env: |
+            ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
+            CE_DOGFOOD_MARKER=1
+
+      - name: Verify outputs and JUnit file
+        env:
+          JUNIT: ${{ steps.dogfood.outputs.junit-path }}
+          RUNDIR: ${{ steps.dogfood.outputs.run-dir }}
+        run: |
+          set -euo pipefail
+          test -n "$JUNIT" && test -f "$JUNIT" || { echo "junit output missing"; exit 1; }
+          # Well-formedness check on a file this job just generated (trusted input;
+          # our writer emits no DTDs/entities) — stdlib ET is fine here.
+          python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT"
+          test -f "$RUNDIR/run.json" || { echo "run.json missing"; exit 1; }
+
+      - name: Upload dogfood run on failure
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: action-dogfood-runs
+          path: runs/ci-action-dogfood/
+          retention-days: 7

.github/workflows/release.yml

@@ -182,23 +182,50 @@ jobs:
           echo "version=$V" >> "$GITHUB_OUTPUT"
           echo "Publishing version: $V"
 
-      - name: Regenerate uv.lock and amend release commit
+      - name: Regenerate uv.lock, bump action.yml pin, and amend release commit
         if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != ''
+        env:
+          # Passed via env (not interpolated into the script) per GitHub's
+          # injection guidance.
+          VERSION: ${{ steps.release.outputs.version }}
         run: |
+          set -euo pipefail
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git config user.name "github-actions[bot]"
+          # Bump the composite action's default `version:` pin to the just-released
+          # version so `UiPath/coder_eval@vX.Y.Z` installs `coder-eval==X.Y.Z`. The
+          # anchor is indentation-tolerant and keyed on the unique trailing
+          # "# <-- kept in sync" comment; the grep guard fails the release loudly
+          # if a reformat ever detaches it (rather than shipping a stale pin).
+          sed -i -E 's/^([[:space:]]*default: ")[0-9]+\.[0-9]+\.[0-9]+("   # <-- kept in sync)/\1'"${VERSION}"'\2/' action.yml
+          grep -q "default: \"${VERSION}\"" action.yml || { echo "action.yml version bump failed"; exit 1; }
+          git add action.yml
+          # Regenerate the lock too; stage it (a no-op if unchanged).
           uv lock
-          if ! git diff --quiet uv.lock; then
-            git config user.email "github-actions[bot]@users.noreply.github.com"
-            git config user.name "github-actions[bot]"
-            git add uv.lock
+          git add uv.lock
+          # Amend only if action.yml/uv.lock actually changed the tree.
+          if ! git diff --cached --quiet; then
             git commit --amend --no-edit
             # Amend replaced the commit the tag points at; re-point it before pushing.
-            git tag -f "v${{ steps.release.outputs.version }}"
+            git tag -f "v${VERSION}"
           fi
 
       - name: Push release commit and tags
         if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != ''
         run: git push origin main "v${{ steps.release.outputs.version }}"
 
+      - name: Move major action tag (vN -> this release)
+        if: steps.release.outputs.version != ''
+        env:
+          VERSION: ${{ steps.release.outputs.version }}
+        run: |
+          set -euo pipefail
+          # Consumers pin `UiPath/coder_eval@v0` (becomes `@v1` at 1.0.0). Force-move
+          # the moving major tag to this release. Force on a missing tag creates it.
+          MAJOR="v${VERSION%%.*}"
+          git tag -f "$MAJOR" "v${VERSION}"
+          git push -f origin "$MAJOR"
+
       - name: Build wheel + sdist
         if: steps.ver.outputs.version != ''
         run: uv build

CLAUDE.md

@@ -20,6 +20,7 @@ coder_eval/
 ├── orchestrator.py                # Main evaluation loop
 ├── reports.py                     # Markdown/JSON report generation (run-level + per-suite rollup via write_suite_rollups)
 ├── reports_experiment.py          # Experiment/cross-variant report generation
+├── reports_junit.py               # JUnit XML report from a finalized run dir (run.json spine; for CI test-report ingestion)
 ├── analysis.py                    # Command statistics aggregation
 ├── logging_config.py              # Structured logging setup
 ├── path_utils.py                  # Run ID generation, path utilities
@@ -120,6 +121,7 @@ tasks/                             # Task definition YAML files
 tests/                             # Test suite
 docs/                              # Documentation
 templates/                         # Sandbox template directories
+action.yml                         # Published composite GitHub Action (coder-eval as a CI gate). release.yml maintains its `version:` default + the moving `v<major>` tag.
 ```
 
 ## Key Architectural Patterns

README.md

@@ -96,6 +96,74 @@ live in this repo — clone it or point the CLI at your own task files.) See
 [Tutorial 02 — Running coder_eval in CI](docs/tutorials/02-ci-pipeline.md) for
 the full setup.
 
+## Use as a GitHub Action
+
+A composite action at the repo root runs `coder-eval` as a CI gate — it installs
+the pinned CLI, runs your tasks, writes a JUnit XML report, appends `run.md` to
+the job summary, and fails the step on any task/gate failure:
+
+```yaml
+- uses: UiPath/coder_eval@v0    # becomes @v1 once 1.0.0 ships; @vX.Y.Z pins exactly
+  with:
+    tasks: tests/tasks/**/*.yaml
+    model: claude-sonnet-5
+    env: |
+      ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
+```
+
+| Input | Default | Purpose |
+| --- | --- | --- |
+| `tasks` | *(all `tasks/`)* | Task YAML path(s)/glob |
+| `tags` | — | `--tags` filter |
+| `model` | — | `--model` override |
+| `extra-args` | — | Verbatim extra args (`--experiment`, `-D …`, …) |
+| `version` | pinned release | PyPI version, or `local` to install from the checkout |
+| `run-dir` | `runs/ci` | Run directory |
+| `junit-path` | `coder-eval-junit.xml` | Where to write the JUnit report |
+| `step-summary` | `true` | Append `run.md` to the job summary |
+| `env` | — | Credentials/backend passthrough: newline-separated `NAME=VALUE` pairs, exported for the run step only |
+| `minimum-task-score` | *(off)* | Strict floor (0.0–1.0): fail the step if any task's `weighted_score` is below it |
+
+Outputs: `run-dir` and `junit-path`. Feed the JUnit file to your platform's
+test-report renderer — e.g. on GitHub Actions with
+[`mikepenz/action-junit-report`](https://github.com/mikepenz/action-junit-report):
+
+```yaml
+- uses: mikepenz/action-junit-report@v5
+  if: always()
+  with:
+    report_paths: coder-eval-junit.xml
+```
+
+**Credentials and backend config** are the sole responsibility of `env` — a
+passthrough exported for the run step only (never written to `$GITHUB_ENV`, so
+it can't leak into later steps). Set whatever the run needs, Anthropic or not:
+
+```yaml
+- uses: UiPath/coder_eval@v0
+  with:
+    tasks: tests/tasks/**/*.yaml
+    minimum-task-score: "0.8"   # fail the build if any task scores below 0.8
+    env: |
+      CODER_EVAL_API_BACKEND=bedrock
+      AWS_BEARER_TOKEN_BEDROCK=${{ secrets.BEDROCK_TOKEN }}
+```
+
+`minimum-task-score` is a strict floor **on top of** coder-eval's own exit
+code: the step fails if *either* coder-eval exits non-zero *or* any task's
+`weighted_score` falls below the floor. Leave it unset to gate on the exit code
+alone.
+
+> **Agent runtime is the caller's responsibility.** The action is agent-agnostic —
+> it installs `coder-eval` but no coding-agent runtime. Tasks using the default
+> `claude-code` agent need the `claude` CLI on `PATH` (`actions/setup-node` +
+> `npm install -g @anthropic-ai/claude-code`) in the job before the action runs.
+
+> **Security.** Evaluated tasks execute agent-generated code. Do **not** run this
+> action under `pull_request_target` with secrets exposed to untrusted fork PRs —
+> use `pull_request` and gate on the same-repo condition, as this repo's own
+> dogfood job does.
+
 ## Telemetry
 
 > 📊 **Usage telemetry is on by default.** `coder-eval` sends **anonymous** usage