[WIP] Test repair agent: Part1

agents/test-repair/Dockerfile

@@ -0,0 +1,45 @@
+FROM python:3.12 AS builder
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+ENV UV_PROJECT_ENVIRONMENT=/opt/venv
+
+WORKDIR /app
+
+# Install external deps without building workspace members.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=VERSION,target=VERSION \
+    uv sync --frozen --no-dev --no-install-workspace --package hackbot-agent-test-repair
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,target=/app,rw \
+    uv sync --locked --no-dev --no-editable --package hackbot-agent-test-repair
+
+FROM python:3.12 AS base
+
+COPY --from=builder /opt/venv /opt/venv
+WORKDIR /app
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PATH="/opt/venv/bin:$PATH"
+
+FROM base AS agent
+
+# hackbot.toml lives at the agent root (not inside the package), so copy it into
+# the working dir; the runtime discovers it there (cwd) at startup.
+COPY agents/test-repair/hackbot.toml /app/hackbot.toml
+
+RUN useradd --create-home --shell /bin/bash agent \
+    && mkdir -p /workspace \
+    && chown agent:agent /workspace
+
+# `mach bootstrap` installs the toolchain here at runtime; put it on PATH so the
+# agent's own `./mach build` (and the build_firefox tool) find rustc/clang.
+ENV PATH="/home/agent/.cargo/bin:/home/agent/.mozbuild/clang/bin:${PATH}"
+
+USER agent
+
+CMD ["python", "-m", "hackbot_agents.test_repair"]

agents/test-repair/README.md

@@ -0,0 +1,65 @@
+# Test Repair Agent
+
+Two-stage Claude agent that finds the commit which regressed a failing Firefox CI
+test and proposes a fix. Agent logic in `hackbot_agents/test_repair/`.
+
+The pulse listener only forwards failures that already passed its regression and
+flakiness filters, so a regression is the prior, but the agent still reports the
+classification it reaches. Its only input is a Taskcluster task id.
+
+Run the Docker command below from the repo root, with secrets in a local `.env`
+(`ANTHROPIC_API_KEY`; `BUGZILLA_API_KEY` is optional).
+
+## Deterministic prep
+
+Before Claude is invoked, `resolve.py` turns the task id into everything the
+investigation needs (no log parsing):
+
+1. Project + hg revision from the Taskcluster task.
+2. The failing test groups, via mozci.
+3. The revision at which the group was last green, by walking mozci push
+   ancestors.
+4. The git range that landed since then, from the hg pushlog + lando. Only the
+   range endpoints are mapped to git; the commit count sizes the clone and is
+   capped so an old last-green can't produce an unbounded one.
+
+The agent gets the range (`base..head`), not a list of shas, and enumerates and
+narrows it itself with `git log`. When the range isn't known to reach a green run
+it is passed as `HEAD~N..HEAD` and the prompt stops asserting the culprit is in
+it.
+
+`SOURCE_REF` / `SOURCE_DEPTH` pin the shallow clone to the failure commit, deep
+enough to walk the range. The task's full and sanitized logs are written to files
+for the agent to search.
+
+## Input
+
+- `FAILURE_TASKS` - a dictionary of failed Taskcluster test tasks
+  `{task_name: taskcluster_task_id}`. Everything else is resolved from the first
+  task id.
+
+## Output
+
+First stage - analysis (read-only):
+
+- `summary.md` - a short verdict
+- `analysis.md` - detailed reasoning, with evidence from the logs and diffs
+- `verdict.json` - `classification` (`regression` / `intermittent`),
+  `culprit_commit`, `culprit_bug`, `intermittent_bug`, `recommendation`
+  (`backout` / `land_fix` / `do_not_backout`) and `confidence`
+
+Second stage - fixing (only when a culprit was identified):
+
+- A patch in Hackbot format
+
+The result reports the `culprit_commit` so the caller can attribute the
+regression to a developer.
+
+## Test the agent
+
+```sh
+FAILURE_TASKS='{"test-linux1804-64/opt-xpcshell-1":"XyU4b_BIRdO_IeK6z_kcQg"}' \
+  docker compose up test-repair-agent --build
+```
+
+Artifacts are written to `~/hackbot/artifacts/`.

agents/test-repair/compose.yml

@@ -0,0 +1,24 @@
+services:
+  test-repair-agent:
+    build:
+      context: ../..
+      dockerfile: agents/test-repair/Dockerfile
+      target: agent
+    # Per-run inputs are interpolated by the listener/hackbot-api as env; pydantic
+    # AgentInputs validates them at runtime. BUGZILLA_MCP_URL is optional (Bugzilla
+    # is supplementary context) and left unset here.
+    environment:
+      - RUN_ID
+      - FAILURE_TASKS=${FAILURE_TASKS:-}
+      - SOURCE_REPO=/workspace/firefox
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:?error}
+      - WANDB_API_KEY=${WANDB_API_KEY:-}
+      # No uploader locally: summary/logs/artifacts are written under
+      # /artifacts/<run_id>, bind-mounted to the host's ~/hackbot/artifacts.
+      - ARTIFACTS_DIR=/artifacts
+    volumes:
+      - workspace:/workspace
+      - ${HOME}/hackbot/artifacts:/artifacts
+
+volumes:
+  workspace:

agents/test-repair/hackbot.toml

@@ -0,0 +1,10 @@
+[source]
+repo_url = "https://github.com/mozilla-firefox/firefox.git"
+checkout_path = "/workspace/firefox"
+# The agent resolves the failure commit and candidate range from the task id at
+# startup, then sets SOURCE_REF (the head/failure commit) and SOURCE_DEPTH (deep
+# enough to reach the last-green commit) before the runtime prepares the tree.
+
+[firefox]
+enabled = true
+objdir = "objdir-test-repair"

agents/test-repair/hackbot_agents/test_repair/__main__.py

@@ -0,0 +1,85 @@
+import logging
+import os
+import tempfile
+from pathlib import Path
+
+from hackbot_runtime import HackbotContext, run_async
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from .agent import TestRepairResult
+from .logs import download_failure_logs
+from .resolve import Investigation, resolve_investigation
+
+logger = logging.getLogger(__name__)
+
+
+class AgentInputs(BaseSettings):
+    # Failing Taskcluster test tasks {task_name: task_id}. The agent resolves the
+    # push, last-green revision and candidate commit range from the task id.
+    failure_tasks: dict[str, str]
+    bugzilla_mcp_url: str = ""
+    model: str | None = None
+    max_turns: int | None = None
+
+    # Compose passes unset per-run inputs as empty strings; treat those as absent.
+    model_config = SettingsConfigDict(extra="ignore", env_ignore_empty=True)
+
+
+def _pin_checkout(investigation: Investigation) -> None:
+    """Pin the shallow clone to the failure commit, deep enough for the range.
+
+    Read by the runtime when it prepares the source tree
+    (HackbotContext.source_repo).
+    """
+    os.environ.setdefault("SOURCE_REF", investigation.failure_commit)
+    os.environ.setdefault("SOURCE_DEPTH", str(investigation.commit_range.span + 1))
+    logger.info(
+        "Pinning checkout to %s with depth %s",
+        os.environ["SOURCE_REF"],
+        os.environ["SOURCE_DEPTH"],
+    )
+
+
+async def main(ctx: HackbotContext) -> TestRepairResult:
+    from .agent import run_test_repair
+
+    inputs = AgentInputs()
+    if not inputs.failure_tasks:
+        raise ValueError("failure_tasks must contain at least one task")
+
+    task_id = next(iter(inputs.failure_tasks.values()))
+    logger.info("Starting test-repair for task %s", task_id)
+    investigation: Investigation = resolve_investigation(task_id)
+    _pin_checkout(investigation)
+
+    scratch_dir = Path(tempfile.mkdtemp(prefix="test-repair-"))
+    scratch_in = scratch_dir / "in"
+    scratch_out = scratch_dir / "out"
+    scratch_in.mkdir(parents=True, exist_ok=True)
+    scratch_out.mkdir(parents=True, exist_ok=True)
+
+    logger.info("Downloading failure logs for %d task(s)", len(inputs.failure_tasks))
+    task_logs = await download_failure_logs(inputs.failure_tasks, scratch_in)
+
+    bugzilla_mcp_server = (
+        {"type": "http", "url": inputs.bugzilla_mcp_url}
+        if inputs.bugzilla_mcp_url
+        else None
+    )
+    return await run_test_repair(
+        bugzilla_mcp_server=bugzilla_mcp_server,
+        source_repo=ctx.source_repo,
+        fx_ctx=ctx.firefox,
+        investigation=investigation,
+        task_logs=task_logs,
+        scratch_out=scratch_out,
+        model=inputs.model,
+        max_turns=inputs.max_turns,
+        log=ctx.log_path,
+        verbose=True,
+        publish_file=ctx.publish_file,
+    )
+
+
+if __name__ == "__main__":
+    run_async(main)