fix: weight:0 un-gates criteria + configurable retry policy

CLAUDE.md

@@ -35,7 +35,7 @@ coder_eval/
 │   ├── criteria.py                # 14 success criterion types + base + union
 │   ├── experiment.py              # ExperimentDefinition, ExperimentVariant, ResolvedTask, result models
 │   ├── judge_defaults.py          # DEFAULT_JUDGE_MODEL constant (cycle-free leaf)
-│   ├── mutations.py               # PromptMutation variants (prefix/suffix/replace/template/rephrase)
+│   ├── mutations.py               # PromptMutation variants (prefix/suffix/replace/template)
 │   ├── results.py                 # CriterionResult (+ ClassificationCriterionResult), TurnRecord, EvaluationResult, EarlyStopInfo/EarlyStopReason, CriterionAggregate, ThresholdCheck, SuiteRollup
 │   ├── routing.py                 # ApiRoute (DirectRoute/BedrockRoute)
 │   ├── sandbox.py                 # SandboxConfig, ResourceLimits
@@ -294,7 +294,7 @@ Tasks are YAML files. See [docs/TASK_DEFINITION_GUIDE.md](docs/TASK_DEFINITION_G
 
 **Runtime (always)**: pydantic, pydantic-settings, pyyaml, typer, rich, python-dotenv, anthropic, claude-agent-sdk, anyio, radon, tqdm, jmespath, jsonschema
 
-**Runtime (optional, `[uipath]` extra)**: uipath — the in-host `uipath` SDK (handy for local sandbox parity with tasks that invoke `uv run uipath eval ...`). Base installs without this extra still run end-to-end; UiPath-dependent paths fail at dispatch with a clear `pip install 'coder-eval[uipath]'` hint. The LLM judge and the `rephrase` mutation no longer use the LLM Gateway client — they route through the run's backend (Bedrock / Anthropic), so `uipath-llmgw-client` is no longer a dependency.
+**Runtime (optional, `[uipath]` extra)**: uipath — the in-host `uipath` SDK (handy for local sandbox parity with tasks that invoke `uv run uipath eval ...`). Base installs without this extra still run end-to-end; UiPath-dependent paths fail at dispatch with a clear `pip install 'coder-eval[uipath]'` hint. The LLM judge no longer uses the LLM Gateway client — it routes through the run's backend (Bedrock / Anthropic), so `uipath-llmgw-client` is no longer a dependency.
 
 **Dev**: pytest, pytest-asyncio, pytest-mock, pytest-cov, ruff, pyright, pip-audit, bandit, pre-commit, mcp
 

docs/AB_EXPERIMENTS.md

@@ -227,8 +227,8 @@ variants:
         text: "\n\nThink step by step and validate your work before finishing."
 ```
 
-The full mutation catalog (prefix / suffix / replace / template / rephrase) is
-defined in `coder_eval/models/mutations.py`.
+The full mutation catalog (prefix / suffix / replace / template) is defined in
+`coder_eval/models/mutations.py`.
 
 ## Recipe: Smoke vs. e2e Flavors (Early Stop)
 

docs/TASK_DEFINITION_GUIDE.md

@@ -102,6 +102,10 @@ run_limits:
   expected_turns: 8                   # Optional: SOFT efficiency budget (visible turns) — not a cap
   turn_timeout: 300                   # Optional: per-communicate() timeout in seconds
   task_timeout: 600                   # Optional: wall-clock cap across all iterations
+  retry:                              # Optional: override the built-in retry policy
+    max_retries: 2                    #   attempts per retryable error category (0 = fail fast)
+    initial_delay: 5.0                #   seconds before the first retry
+    backoff_multiplier: 2.0           #   exponential backoff factor
 
 agent:
   type: "claude-code"                 # Agent type — optional if supplied via experiment / --type
@@ -273,11 +277,17 @@ sandbox:
     - "!node_modules"
     - "*.bak"                         #   bare entry adds an extra pattern
   limits:                             # Optional: resource limits
-    timeout: 300                       # Enforced via subprocess timeout
-    max_memory_mb: 512                 # NOT enforced (reserved for future use)
-    max_disk_mb: 1024                  # NOT enforced (reserved for future use)
+    timeout: 300                       # Enforced via subprocess timeout (both drivers)
+    max_memory_mb: 512                 # driver:docker -> `--memory`; ignored under tempdir
+    max_cpus: 2                        # driver:docker -> `--cpus`; ignored under tempdir
+    max_pids: 512                      # driver:docker -> `--pids-limit`; ignored under tempdir
+    max_disk_mb: 1024                  # NOT enforced (reserved: no portable docker knob)
 ```
 
+Under `driver: tempdir` only `timeout` is enforced — the agent can consume
+arbitrary host memory, CPU, and PIDs. Use `driver: docker` when you need the
+container limits above to actually bind.
+
 ## Template Sources
 
 Tasks can start with preset files instead of an empty sandbox. Multiple sources are applied sequentially (last wins for conflicts).
@@ -381,7 +391,7 @@ All criteria share these fields:
 | Field | Default | Description |
 |-------|---------|-------------|
 | `description` | — | Human-readable description (required) |
-| `weight` | 1.0 | Relative importance for weighted score |
+| `weight` | 1.0 | Relative importance for weighted score. `0` = **informational**: excluded from both the score and the pass/fail gate |
 | `pass_threshold` | 0.9 | Minimum score (0.0–1.0) to pass |
 | `stop_when` | `null` | Arms this criterion for early stop (`pass`/`fail`/`decided`); requires `run_limits.stop_early: true` and an observable criterion type (`skill_triggered`, `command_executed`). See [`stop_early`](#stop_early-opt-in-early-stop). |
 
@@ -390,7 +400,12 @@ All criteria share these fields:
 - **Fractional** (0.0–1.0): `file_contains`, `file_check`, `json_check`, `command_executed`, `uipath_eval`
 - **Continuous** (0.0–1.0): `reference_comparison`, `llm_judge`, `agent_judge`
 
-**Task success:** ALL criteria must score >= their `pass_threshold`.
+**Task success:** all *gating* criteria must score >= their `pass_threshold`. A
+criterion with `weight: 0` is informational — it is still checked, stored, and
+rendered in reports, but it neither contributes to the score nor fails the task.
+(A `weight: 0` criterion may not set `stop_when` or `suite_thresholds`: arming a
+non-gating criterion for the early-stop or suite gate would let an
+"informational" check flip a run to failure.)
 
 **Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` — calculated regardless for quality assessment.
 

docs/tutorials/03-evalboard-local.md

@@ -56,6 +56,7 @@ Then open **http://localhost:3030**.
 | `/runs/<run-id>` | Run summary — pass rate, cost, duration — plus one row per task (green = pass, red = fail). "Download run (.zip)" bundles the whole folder |
 | `/runs/<run-id>/<task-id>` | Per-task detail: success-criteria cards, the tool timeline, the message timeline (per-message tokens, expandable into thinking/tool/text), artifact downloads, and the tail of `task.log` |
 | `/trends` | Per-task pass rate and average duration/cost/turns across recent runs |
+| `/watchlist` | Tasks that need attention — newly broken, flaky, or persistently failing across recent runs |
 
 `<task-id>` matches the `task_id` in the results and the subdirectory name under
 `<run-id>/<variant>/` — the same layout described in the

evalboard/app/layout.tsx

@@ -40,31 +40,34 @@ export default function RootLayout({ children }: { children: ReactNode }) {
                             <SearchBox className="w-full max-w-xl" />
                         </Suspense>
                     </div>
-                    {/* Watchlist + Trends are internal-only surfaces; the public
-                        OSS edition hides these nav links (see lib/edition.ts). The
-                        routes themselves are left intact. */}
-                    {isInternal && (
-                        <div className="ml-auto sm:ml-0 flex items-center gap-4 text-sm shrink-0">
+                    {/* Watchlist + Trends are edition-agnostic: both are built
+                        purely from the run artifacts any coder-eval install
+                        produces (see lib/trends.ts, lib/watchlist.ts), so the OSS
+                        edition ships them too. Only Path to GA stays internal —
+                        it reports on the UiPath-internal `path-to-ga` task tag,
+                        which is meaningless in a public clone. */}
+                    <div className="ml-auto sm:ml-0 flex items-center gap-4 text-sm shrink-0">
+                        {isInternal && (
                             <a
                                 href="/path-to-ga"
                                 className="text-gray-700 hover:text-studio-blue"
                             >
                                 Path to GA
                             </a>
-                            <a
-                                href="/watchlist"
-                                className="text-gray-700 hover:text-studio-blue"
-                            >
-                                Watchlist
-                            </a>
-                            <a
-                                href="/trends"
-                                className="text-gray-700 hover:text-studio-blue"
-                            >
-                                Trends
-                            </a>
-                        </div>
-                    )}
+                        )}
+                        <a
+                            href="/watchlist"
+                            className="text-gray-700 hover:text-studio-blue"
+                        >
+                            Watchlist
+                        </a>
+                        <a
+                            href="/trends"
+                            className="text-gray-700 hover:text-studio-blue"
+                        >
+                            Trends
+                        </a>
+                    </div>
                 </header>
                 <main className="px-4 sm:px-6 lg:px-8 py-6 max-w-[1400px] mx-auto">
                     {children}

evalboard/lib/edition.ts

@@ -1,7 +1,9 @@
 // Build edition gate.
 //
 // "internal" enables UiPath-internal-only surfaces (the hosted dashboard:
-// recent-runs index, watchlist, trends, activation, and UiPath branding).
+// front-page analytics, harness column, path-to-ga, activation, and UiPath
+// branding). Surfaces derived purely from run artifacts — notably /trends and
+// /watchlist — are edition-agnostic and ship in both editions.
 // Anything else — including unset — is the public OSS edition, so a bare clone
 // runs in OSS mode with no configuration. The hosted UiPath deploy opts in by
 // setting EVALBOARD_EDITION=internal.

src/coder_eval/cli/evaluate_command.py

@@ -130,7 +130,12 @@ async def _setup_and_run() -> EvaluationResult:
         raise typer.Exit(1)
 
     for criterion, cr in zip(task.success_criteria, criteria_results, strict=True):
-        status = "[green]✓[/green]" if cr.score >= criterion.pass_threshold else "[red]✗[/red]"
+        if not criterion.is_gating:
+            # weight=0 is informational: it cannot pass/fail the task, so don't
+            # render it as ✓/✗ (that would contradict the gate and the exit code).
+            status = "[dim]○[/dim]"
+        else:
+            status = "[green]✓[/green]" if cr.score >= criterion.pass_threshold else "[red]✗[/red]"
         console.print(f"{status} {cr.criterion_type}")
         console.print(f"  [dim]{cr.description}[/dim]")
         console.print(f"  [dim]Score: {cr.score:.2f}[/dim]")
@@ -140,15 +145,19 @@ async def _setup_and_run() -> EvaluationResult:
             console.print(f"  [red]Error: {cr.error}[/red]")
         console.print()
 
-    passed = sum(
-        1 for cr, c in zip(criteria_results, task.success_criteria, strict=True) if cr.score >= c.pass_threshold
-    )
-    total = len(task.success_criteria)
+    # Gate over gating criteria only (weight=0 is informational and cannot fail
+    # the task) so this summary + the exit code below match final_status.
+    gating = [(cr, c) for cr, c in zip(criteria_results, task.success_criteria, strict=True) if c.is_gating]
+    passed = sum(1 for cr, c in gating if cr.score >= c.pass_threshold)
+    total = len(gating)
     failed = total - passed
+    informational = len(task.success_criteria) - total
 
     console.print("[bold]Summary:[/bold]")
     console.print(f"  Passed: {passed}/{total}")
     console.print(f"  Failed: {failed}/{total}")
+    if informational:
+        console.print(f"  [dim]Informational (weight=0, not gated): {informational}[/dim]")
     console.print(f"\n[dim]Run directory: {prepared_run_dir}[/dim]")
     if result.sandbox_path:
         console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]")

src/coder_eval/errors/executor.py

@@ -5,9 +5,10 @@
 from collections.abc import Awaitable, Callable
 from typing import Any
 
-from .categories import RETRY_CONFIG, RetryConfig
+from coder_eval.models import RetryPolicy
+
 from .categorization import categorize_error
-from .retry import get_error_tip, get_retry_delay, should_retry
+from .retry import get_error_tip, get_retry_delay, resolve_retry_config, should_retry
 
 
 logger = logging.getLogger(__name__)
@@ -19,6 +20,7 @@ async def execute_with_retry(
     context: dict[str, Any],
     max_attempts: int | None = None,
     on_attempt_error: Callable[[Exception, int], Awaitable[None]] | None = None,
+    retry_policy: RetryPolicy | None = None,
 ) -> Any:
     """Execute an operation with automatic retry on transient errors.
 
@@ -41,6 +43,8 @@ async def execute_with_retry(
             invoked after every failed attempt (including the final non-retryable one),
             for draining ``agent.pending_turn`` and calling ``agent.discard_pending_turn()``.
             Callback exceptions are logged and swallowed so they cannot mask the original.
+        retry_policy: Per-run overrides for the built-in per-category retry policy
+            (``run_limits.retry``). None = built-in defaults.
 
     Returns:
         Result from operation
@@ -92,10 +96,10 @@ async def execute_with_retry(
 
             # Categorize error
             category = categorize_error(e, context)
-            config = RETRY_CONFIG.get(category, RetryConfig())
+            config = resolve_retry_config(category, retry_policy)
 
             # Check if we should retry (handles both non-retryable categories and exhausted attempts)
-            if not should_retry(category, attempt):
+            if not should_retry(category, attempt, retry_policy):
                 if config.max_retries == 0:
                     logger.error(f"[{task_id}] {operation_name} failed (non-retryable): {category.value} - {e}")
                 else:
@@ -105,7 +109,7 @@ async def execute_with_retry(
                 raise
 
             # Calculate backoff with jitter
-            delay = get_retry_delay(category, attempt)
+            delay = get_retry_delay(category, attempt, retry_policy)
 
             # Enhanced warning with actionable tip
             tip = get_error_tip(category)

src/coder_eval/errors/retry.py

@@ -10,25 +10,44 @@
 from datetime import datetime
 from typing import Any
 
+from coder_eval.models import RetryPolicy
+
 from .categories import ERROR_TIPS, RETRY_CONFIG, ErrorCategory, ErrorContext, RetryConfig
 from .categorization import categorize_error
 
 
 logger = logging.getLogger(__name__)
 
 
-def should_retry(category: ErrorCategory, attempt: int) -> bool:
+def resolve_retry_config(category: ErrorCategory, policy: RetryPolicy | None = None) -> RetryConfig:
+    """Look up the effective retry config for ``category``.
+
+    Single seam between the built-in per-category table (``RETRY_CONFIG``) and
+    the run's optional ``run_limits.retry`` overrides — every retry decision
+    (should we? how long?) resolves through here so the two can't disagree.
+
+    A non-retryable category (``max_retries == 0``) is returned untouched:
+    whether an error class is worth retrying is a property of the error, not a
+    per-run knob (see ``RetryPolicy``).
+    """
+    config = RETRY_CONFIG.get(category, RetryConfig())
+    if policy is None or config.max_retries == 0:
+        return config
+    return config.model_copy(update=policy.overrides())
+
+
+def should_retry(category: ErrorCategory, attempt: int, policy: RetryPolicy | None = None) -> bool:
     """Determine if an error should be retried.
 
     Args:
         category: Error category
         attempt: Current attempt number (0-indexed)
+        policy: Optional per-run overrides (run_limits.retry)
 
     Returns:
         True if retry should be attempted
     """
-    config = RETRY_CONFIG.get(category, RetryConfig())
-    return attempt < config.max_retries
+    return attempt < resolve_retry_config(category, policy).max_retries
 
 
 def compute_backoff(config: RetryConfig, attempt: int) -> float:
@@ -52,15 +71,16 @@ def compute_backoff(config: RetryConfig, attempt: int) -> float:
     return base_delay + jitter
 
 
-def get_retry_delay(category: ErrorCategory, attempt: int) -> float:
+def get_retry_delay(category: ErrorCategory, attempt: int, policy: RetryPolicy | None = None) -> float:
     """Calculate delay before retry with exponential backoff and jitter.
 
-    Convenience wrapper around ``compute_backoff`` that looks up the
+    Convenience wrapper around ``compute_backoff`` that resolves the effective
     ``RetryConfig`` for the given error category.
 
     Args:
         category: Error category
         attempt: Current attempt number (0-indexed)
+        policy: Optional per-run overrides (run_limits.retry)
 
     Returns:
         Delay in seconds
@@ -71,8 +91,7 @@ def get_retry_delay(category: ErrorCategory, attempt: int) -> float:
         >>> get_retry_delay(ErrorCategory.AGENT_API_ERROR, 1)  # doctest: +SKIP
         10.8  # 10.0 base + 0.8 jitter
     """
-    config = RETRY_CONFIG.get(category, RetryConfig())
-    return compute_backoff(config, attempt)
+    return compute_backoff(resolve_retry_config(category, policy), attempt)
 
 
 def get_error_tip(category: ErrorCategory) -> str:
@@ -140,6 +159,7 @@ def create_error_context(
     attempt: int,
     component: str | None = None,
     agent_name: str | None = None,
+    retry_policy: RetryPolicy | None = None,
     **kwargs: Any,
 ) -> dict[str, Any]:
     """Create comprehensive error context for reporting.
@@ -153,6 +173,9 @@ def create_error_context(
         attempt: Attempt number (1-indexed)
         component: Component that failed (agent/sandbox/evaluator)
         agent_name: Agent name (optional)
+        retry_policy: Per-run overrides (run_limits.retry). Must be the SAME policy the
+            executor ran under, so the persisted is_retryable / retry_delay_seconds
+            cannot contradict the retry decision that was actually taken.
         **kwargs: Additional context fields:
             - disk_usage_gb: Disk usage in GB
             - memory_usage_gb: Memory usage in GB
@@ -198,8 +221,12 @@ def create_error_context(
         component=component,
         agent_name=agent_name,
         attempt_number=attempt,
-        is_retryable=should_retry(category, attempt_index),
-        retry_delay_seconds=get_retry_delay(category, attempt_index) if should_retry(category, attempt_index) else 0.0,
+        is_retryable=should_retry(category, attempt_index, retry_policy),
+        retry_delay_seconds=(
+            get_retry_delay(category, attempt_index, retry_policy)
+            if should_retry(category, attempt_index, retry_policy)
+            else 0.0
+        ),
         disk_usage_gb=kwargs.get("disk_usage_gb"),
         memory_usage_gb=kwargs.get("memory_usage_gb"),
         agent_stdout=agent_stdout,

src/coder_eval/models/__init__.py

@@ -77,7 +77,7 @@
 from coder_eval.models.judge_defaults import DEFAULT_JUDGE_MODEL
 
 # Limits
-from coder_eval.models.limits import RunLimits
+from coder_eval.models.limits import RetryPolicy, RunLimits
 
 # Merge strategy
 from coder_eval.models.merge_strategy import (
@@ -292,6 +292,7 @@
     # Judge
     "JudgeVerdict",
     # Limits
+    "RetryPolicy",
     "RunLimits",
     # Merge strategy
     "MERGE_STRATEGY_KEY",