diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index 9f9935b7..cafe9e4c 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -15,13 +15,14 @@ from coder_eval.path_utils import replicate_subdir_name from coder_eval.reports import resolve_agent_settings from coder_eval.reports_stats import ( + VariantSeries, bootstrap_mean_ci, - cohens_d, + collect_variant_series, describe_prompt_config, fmt_mean_sd, fmt_p, load_variant_eval_results, - paired_bootstrap_diff_ci, + paired_comparison, stddev, welch_t_test, wilson_interval, @@ -216,27 +217,6 @@ def _prompt_config_lines(result: ExperimentResult, experiment: ExperimentDefinit lines.append(f"- **{vid}**: {desc}") return lines - @staticmethod - def _collect_variant_series( - result: ExperimentResult, - ) -> tuple[dict[str, list[float]], dict[str, list[float]], dict[str, list[float]], dict[str, list[float]]]: - """Per-variant (scores, durations, tokens, assistant-turns) series collected - across all task summaries — the raw inputs to the Aggregate Metrics stat rows.""" - variant_scores: dict[str, list[float]] = {vid: [] for vid in result.variant_ids} - variant_durations: dict[str, list[float]] = {vid: [] for vid in result.variant_ids} - variant_tokens: dict[str, list[float]] = {vid: [] for vid in result.variant_ids} - variant_asst_turns: dict[str, list[float]] = {vid: [] for vid in result.variant_ids} - - for ts in result.task_summaries: - for vr in ts.variant_results: - variant_scores[vr.variant_id].append(vr.weighted_score) - variant_durations[vr.variant_id].append(vr.duration_seconds / vr.replicate_count) - if vr.total_tokens is not None: - variant_tokens[vr.variant_id].append(float(vr.total_tokens)) - if vr.total_assistant_turns is not None: - variant_asst_turns[vr.variant_id].append(float(vr.total_assistant_turns)) - return variant_scores, variant_durations, variant_tokens, variant_asst_turns - @staticmethod def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list[str]: """The integer-aggregate rows of the Aggregate Metrics table: Tasks Run, @@ -312,10 +292,7 @@ def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list @staticmethod def _aggregate_stat_rows( result: ExperimentResult, - variant_scores: dict[str, list[float]], - variant_durations: dict[str, list[float]], - variant_tokens: dict[str, list[float]], - variant_asst_turns: dict[str, list[float]], + series: dict[str, VariantSeries], show_p_values: bool, vid_a: str, vid_b: str, @@ -328,38 +305,38 @@ def _aggregate_stat_rows( # Row: Score (mean ± stddev, p-value) row = "| Score" for vid in result.variant_ids: - row += f" | {fmt_mean_sd(variant_scores[vid])}" + row += f" | {fmt_mean_sd(series[vid].scores)}" if show_p_values: - p = welch_t_test(variant_scores[vid_a], variant_scores[vid_b]) + p = welch_t_test(series[vid_a].scores, series[vid_b].scores) row += f" | {fmt_p(p)}" lines.append(row + " |") # Row: Duration row = "| Avg Duration (s)" for vid in result.variant_ids: - row += f" | {fmt_mean_sd(variant_durations[vid], '.1f')}" + row += f" | {fmt_mean_sd(series[vid].durations, '.1f')}" if show_p_values: - p = welch_t_test(variant_durations[vid_a], variant_durations[vid_b]) + p = welch_t_test(series[vid_a].durations, series[vid_b].durations) row += f" | {fmt_p(p)}" lines.append(row + " |") # Row: Assistant Turns (if data available) - if any(variant_asst_turns[vid] for vid in result.variant_ids): + if any(series[vid].asst_turns for vid in result.variant_ids): row = "| Assistant Turns" for vid in result.variant_ids: - row += f" | {fmt_mean_sd(variant_asst_turns[vid], '.1f')}" + row += f" | {fmt_mean_sd(series[vid].asst_turns, '.1f')}" if show_p_values: - p = welch_t_test(variant_asst_turns[vid_a], variant_asst_turns[vid_b]) + p = welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns) row += f" | {fmt_p(p)}" lines.append(row + " |") # Row: Tokens (if data available) - if any(variant_tokens[vid] for vid in result.variant_ids): + if any(series[vid].tokens for vid in result.variant_ids): row = "| Tokens" for vid in result.variant_ids: - row += f" | {fmt_mean_sd(variant_tokens[vid], ',.0f')}" + row += f" | {fmt_mean_sd(series[vid].tokens, ',.0f')}" if show_p_values: - p = welch_t_test(variant_tokens[vid_a], variant_tokens[vid_b]) + p = welch_t_test(series[vid_a].tokens, series[vid_b].tokens) row += f" | {fmt_p(p)}" lines.append(row + " |") @@ -371,9 +348,7 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]: columns). The p-value column + Welch t-tests appear only for exactly 2 variants; ``vid_a``/``vid_b`` stay local so the 3+-variant path never indexes them.""" # ── Aggregate Metrics (vertical: metrics as rows, variants as columns) ── - variant_scores, variant_durations, variant_tokens, variant_asst_turns = ( - ExperimentReportGenerator._collect_variant_series(result) - ) + series = collect_variant_series(result) show_p_values = len(result.variant_ids) == 2 vid_a, vid_b = (result.variant_ids[0], result.variant_ids[1]) if show_p_values else ("", "") @@ -389,9 +364,7 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]: lines = ["", "## Aggregate Metrics", "", header, sep] lines += ExperimentReportGenerator._aggregate_count_rows(result, show_p_values) - lines += ExperimentReportGenerator._aggregate_stat_rows( - result, variant_scores, variant_durations, variant_tokens, variant_asst_turns, show_p_values, vid_a, vid_b - ) + lines += ExperimentReportGenerator._aggregate_stat_rows(result, series, show_p_values, vid_a, vid_b) # Row: Replicates/task (if any variant ran >1 replicate) if any(result.variant_aggregates[vid].replicate_count > 1 for vid in result.variant_ids): @@ -465,8 +438,7 @@ def _win_loss_lines(result: ExperimentResult) -> list[str]: @staticmethod def _replicate_stats_lines(result: ExperimentResult) -> list[str]: """The ``## Replicate Statistics`` block: per-variant bootstrap-CI / Wilson - pass-rate table + the 2-variant paired-bootstrap comparison. Returns ``[]`` - when no variant ran more than one replicate.""" + pass-rate table. Returns ``[]`` when no variant ran more than one replicate.""" # ── Replicate Statistics (only when any variant ran >1 replicate) ── if not any(ts.replicate_count > 1 for ts in result.task_summaries): return [] @@ -488,45 +460,41 @@ def _replicate_stats_lines(result: ExperimentResult) -> list[str]: + f" | {passes}/{len(all_scores)} [{wlo:.2f}, {whi:.2f}] |" ) - # Paired comparison for 2-variant experiments - if len(result.variant_ids) == 2: - vid_a, vid_b = result.variant_ids[0], result.variant_ids[1] - per_rep_a = result.per_replicate_scores.get(vid_a, {}) - per_rep_b = result.per_replicate_scores.get(vid_b, {}) - common_tasks = sorted(set(per_rep_a) & set(per_rep_b)) - a_scores: list[float] = [] - b_scores: list[float] = [] - skipped_tasks: list[str] = [] - for task_id in common_tasks: - rep_a = per_rep_a[task_id] - rep_b = per_rep_b[task_id] - if len(rep_a) == len(rep_b): - a_scores.extend(rep_a) - b_scores.extend(rep_b) - else: - skipped_tasks.append(task_id) - diff = paired_bootstrap_diff_ci(a_scores, b_scores) - if diff is not None: - mean_diff, d_lo, d_hi = diff - d_val = cohens_d(a_scores, b_scores) - d_str = f"{d_val:.2f}" if d_val is not None else "n/a" - suffix = f" ({len(skipped_tasks)} task(s) excluded: unequal replicate counts)" if skipped_tasks else "" - lines.extend( - [ - "", - f"**Paired mean diff ({vid_a} - {vid_b})**: {mean_diff:+.3f}" - + f" [95% CI {d_lo:+.3f}, {d_hi:+.3f}], Cohen's d = {d_str}{suffix}", - ] - ) - else: - lines.extend( - [ - "", - f"*Paired statistics skipped — unequal replicate counts between {vid_a} and {vid_b}.*", - ] - ) return lines + @staticmethod + def _paired_comparison_lines(result: ExperimentResult) -> list[str]: + """The ``## Paired Comparison`` block for 2-variant experiments. + + Renders :func:`coder_eval.reports_stats.paired_comparison`, which the HTML + reporter renders too. Returns ``[]`` only when the two variants have no + scored task in common; when they have exactly one, the section explains why + no paired result is shown. + """ + pc = paired_comparison(result) + if pc is None: + return [] + + header = ["", "## Paired Comparison", ""] + if pc.mean_diff is None or pc.ci_low is None or pc.ci_high is None: + return [ + *header, + f"*A paired comparison needs at least 2 tasks common to {pc.vid_a} and {pc.vid_b};" + + f" found {pc.task_count}.*", + ] + + d_str = f"{pc.effect_size:.2f}" if pc.effect_size is not None else "n/a" + excluded = f" ({pc.excluded_count} task(s) excluded — not scored by both variants)" if pc.excluded_count else "" + return [ + *header, + f"*Paired over the per-task mean score of {pc.task_count} task(s) common to both variants" + + excluded + + " — pairing cancels between-task difficulty, which the pooled Welch test above cannot.*", + f"**Paired mean diff ({pc.vid_a} - {pc.vid_b})**: {pc.mean_diff:+.3f}" + + f" [95% CI {pc.ci_low:+.3f}, {pc.ci_high:+.3f}], Cohen's d = {d_str}" + + f", p = {fmt_p(pc.p_value)}", + ] + @staticmethod def generate_experiment_report( result: ExperimentResult, @@ -549,6 +517,7 @@ def generate_experiment_report( lines += ExperimentReportGenerator._aggregate_metrics_lines(result) lines += ExperimentReportGenerator._win_loss_lines(result) lines += ExperimentReportGenerator._replicate_stats_lines(result) + lines += ExperimentReportGenerator._paired_comparison_lines(result) return "\n".join(lines) @staticmethod diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 7f0e9cad..a6f74991 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -1245,28 +1245,52 @@ def _experiment_prompt_config(experiment: ExperimentDefinition | None, variant_i """ -def _collect_variant_series(result: ExperimentResult) -> dict[str, dict[str, list[float]]]: - """Group per-variant numeric series (scores, durations, etc.).""" - series: dict[str, dict[str, list[float]]] = { - vid: {"scores": [], "durations": [], "asst_turns": [], "tokens": []} for vid in result.variant_ids - } - for ts in result.task_summaries: - for vr in ts.variant_results: - s = series.get(vr.variant_id) - if s is None: - continue - s["scores"].append(vr.weighted_score) - s["durations"].append(vr.duration_seconds) - if vr.total_tokens is not None: - s["tokens"].append(float(vr.total_tokens)) - if vr.total_assistant_turns is not None: - s["asst_turns"].append(float(vr.total_assistant_turns)) - return series +def _experiment_paired_comparison(result: ExperimentResult) -> str: + """Render the Paired Comparison section — the HTML twin of the markdown one. + + Both render the same ``reports_stats.paired_comparison`` result, so the two + reports can never disagree about the paired numbers. + """ + from .reports_stats import fmt_p, paired_comparison + + pc = paired_comparison(result) + if pc is None: + return "" + + if pc.mean_diff is None or pc.ci_low is None or pc.ci_high is None: + body = _esc( + f"A paired comparison needs at least 2 tasks common to {pc.vid_a} and {pc.vid_b}; found {pc.task_count}." + ) + return f""" +

Paired Comparison

+
+

{body}

+
+""" + + d_str = f"{pc.effect_size:.2f}" if pc.effect_size is not None else "n/a" + excluded = f" ({pc.excluded_count} task(s) excluded — not scored by both variants)" if pc.excluded_count else "" + note = _esc( + f"Paired over the per-task mean score of {pc.task_count} task(s) common to both variants" + + excluded + + " — pairing cancels between-task difficulty, which the pooled Welch test above cannot." + ) + headline = _esc( + f"Paired mean diff ({pc.vid_a} - {pc.vid_b}): {pc.mean_diff:+.3f}" + + f" [95% CI {pc.ci_low:+.3f}, {pc.ci_high:+.3f}], Cohen's d = {d_str}, p = {fmt_p(pc.p_value)}" + ) + return f""" +

Paired Comparison

+
+

{note}

+

{headline}

+
+""" def _experiment_aggregate_metrics(result: ExperimentResult) -> str: """Render the Aggregate Metrics table (with p-values when exactly 2 variants).""" - from .reports_stats import fmt_mean_sd, fmt_p, welch_t_test + from .reports_stats import collect_variant_series, fmt_mean_sd, fmt_p, welch_t_test show_p = len(result.variant_ids) == 2 vid_a, vid_b = (result.variant_ids[0], result.variant_ids[1]) if show_p else ("", "") @@ -1276,7 +1300,7 @@ def _experiment_aggregate_metrics(result: ExperimentResult) -> str: header_cells.append("p-value") header_html = "" + "".join(header_cells) + "" - series = _collect_variant_series(result) + series = collect_variant_series(result) def _row(label: str, values: list[str], p: str | None) -> str: cells = [f"{_esc(label)}"] + [f"{_esc(v)}" for v in values] @@ -1307,31 +1331,31 @@ def _success_rate(vid: str) -> str: rows.append( _row( "Score", - [fmt_mean_sd(series[vid]["scores"]) for vid in result.variant_ids], - fmt_p(welch_t_test(series[vid_a]["scores"], series[vid_b]["scores"])) if show_p else None, + [fmt_mean_sd(series[vid].scores) for vid in result.variant_ids], + fmt_p(welch_t_test(series[vid_a].scores, series[vid_b].scores)) if show_p else None, ) ) rows.append( _row( "Duration (s)", - [fmt_mean_sd(series[vid]["durations"], ".1f") for vid in result.variant_ids], - fmt_p(welch_t_test(series[vid_a]["durations"], series[vid_b]["durations"])) if show_p else None, + [fmt_mean_sd(series[vid].durations, ".1f") for vid in result.variant_ids], + fmt_p(welch_t_test(series[vid_a].durations, series[vid_b].durations)) if show_p else None, ) ) - if any(series[vid]["asst_turns"] for vid in result.variant_ids): + if any(series[vid].asst_turns for vid in result.variant_ids): rows.append( _row( "Assistant Turns", - [fmt_mean_sd(series[vid]["asst_turns"], ".1f") for vid in result.variant_ids], - fmt_p(welch_t_test(series[vid_a]["asst_turns"], series[vid_b]["asst_turns"])) if show_p else None, + [fmt_mean_sd(series[vid].asst_turns, ".1f") for vid in result.variant_ids], + fmt_p(welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns)) if show_p else None, ) ) - if any(series[vid]["tokens"] for vid in result.variant_ids): + if any(series[vid].tokens for vid in result.variant_ids): rows.append( _row( "Tokens", - [fmt_mean_sd(series[vid]["tokens"], ",.0f") for vid in result.variant_ids], - fmt_p(welch_t_test(series[vid_a]["tokens"], series[vid_b]["tokens"])) if show_p else None, + [fmt_mean_sd(series[vid].tokens, ",.0f") for vid in result.variant_ids], + fmt_p(welch_t_test(series[vid_a].tokens, series[vid_b].tokens)) if show_p else None, ) ) @@ -1607,6 +1631,7 @@ def generate_experiment_html( """ + _experiment_prompt_config(experiment, result.variant_ids) + _experiment_aggregate_metrics(result) + + _experiment_paired_comparison(result) + _experiment_win_rates(result) + _experiment_per_task_comparison(result) + _experiment_most_divergent(result) diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 00598578..25b8c1c4 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -12,8 +12,9 @@ import random import statistics as _stats from pathlib import Path +from typing import NamedTuple -from coder_eval.models import EvaluationResult, ExperimentVariant, TaskExperimentSummary +from coder_eval.models import EvaluationResult, ExperimentResult, ExperimentVariant, TaskExperimentSummary logger = logging.getLogger(__name__) @@ -33,16 +34,94 @@ def stddev(values: list[float]) -> float: return _stats.stdev(values) if len(values) >= 2 else 0.0 +def _betacf(a: float, b: float, x: float) -> float: + """Continued fraction for the regularized incomplete beta (Lentz's method).""" + max_iterations = 200 + eps = 3e-12 + fpmin = 1e-300 + + qab, qap, qam = a + b, a + 1.0, a - 1.0 + c = 1.0 + d = 1.0 - qab * x / qap + if abs(d) < fpmin: + d = fpmin + d = 1.0 / d + h = d + for m in range(1, max_iterations + 1): + m2 = 2 * m + # Even step of the recurrence. + aa = m * (b - m) * x / ((qam + m2) * (a + m2)) + d = 1.0 + aa * d + if abs(d) < fpmin: + d = fpmin + c = 1.0 + aa / c + if abs(c) < fpmin: + c = fpmin + d = 1.0 / d + h *= d * c + # Odd step. + aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2)) + d = 1.0 + aa * d + if abs(d) < fpmin: + d = fpmin + c = 1.0 + aa / c + if abs(c) < fpmin: + c = fpmin + d = 1.0 / d + delta = d * c + h *= delta + if abs(delta - 1.0) < eps: + return h + logger.warning("Incomplete beta continued fraction did not converge for a=%r, b=%r, x=%r", a, b, x) + return h + + +def regularized_incomplete_beta(a: float, b: float, x: float) -> float: + """Regularized incomplete beta function I_x(a, b), for a, b > 0 and x in [0, 1]. + + Raises ValueError outside that domain — returning NaN would let a bad input + render as a real-looking statistic downstream. + """ + if not (math.isfinite(a) and math.isfinite(b) and math.isfinite(x)): + raise ValueError(f"a, b and x must be finite, got a={a!r}, b={b!r}, x={x!r}") + if a <= 0.0 or b <= 0.0: + raise ValueError(f"a and b must be positive, got a={a!r}, b={b!r}") + if x <= 0.0: + return 0.0 + if x >= 1.0: + return 1.0 + ln_front = math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log1p(-x) + front = math.exp(ln_front) + # Use the continued fraction directly where it converges fast, else via symmetry. + if x < (a + 1.0) / (a + b + 2.0): + return front * _betacf(a, b, x) / a + return 1.0 - front * _betacf(b, a, 1.0 - x) / b + + +def student_t_two_tailed_p(t_stat: float, df: float) -> float: + """Exact two-tailed p-value for Student's t: P(|T| >= |t|) = I_x(df/2, 1/2), x = df/(df + t^2). + + Non-finite inputs fail closed to 1.0 — garbage must never read as significant. + """ + if not math.isfinite(t_stat) or not math.isfinite(df) or df <= 0: + return 1.0 + x = df / (df + t_stat * t_stat) + return regularized_incomplete_beta(df / 2.0, 0.5, x) + + def welch_t_test(a: list[float], b: list[float]) -> float | None: - """Two-tailed p-value using Welch's t-test via stdlib NormalDist approximation. + """Two-tailed p-value from Welch's unequal-variances t-test (exact t distribution). - Uses the normal approximation for the t-distribution when df is large (>30), - and falls back to exact Welch-Satterthwaite otherwise. Returns None if either - group has fewer than 2 observations. + Degrees of freedom via Welch-Satterthwaite; the t CDF is evaluated exactly + through the regularized incomplete beta (stdlib only, no scipy). Returns + None if either group has fewer than 2 observations, or holds a non-finite + value (rendered as "—" rather than a fabricated p-value). """ n_a, n_b = len(a), len(b) if n_a < 2 or n_b < 2: return None + if not all(math.isfinite(v) for v in (*a, *b)): + return None mean_a, mean_b = _stats.mean(a), _stats.mean(b) var_a = _stats.variance(a) @@ -50,14 +129,13 @@ def welch_t_test(a: list[float], b: list[float]) -> float | None: se_sq = var_a / n_a + var_b / n_b if se_sq == 0: - return 1.0 + # Zero variance in both groups: identical constants (p=1) or a + # deterministic difference (p=0). + return 1.0 if mean_a == mean_b else 0.0 t_stat = abs(mean_a - mean_b) / math.sqrt(se_sq) - - # Conservative normal approximation: treat t as z-score. - # Overestimates p slightly for small df (heavier tails), but correct in - # direction and sufficient for display purposes (no incomplete-beta needed). - return 2.0 * _stats.NormalDist().cdf(-t_stat) + df = se_sq**2 / ((var_a / n_a) ** 2 / (n_a - 1) + (var_b / n_b) ** 2 / (n_b - 1)) + return student_t_two_tailed_p(t_stat, df) def fmt_mean_sd(values: list[float], fmt: str = ".3f") -> str: @@ -96,7 +174,15 @@ def bootstrap_mean_ci( Returns (mean, ci_low, ci_high). When ``len(values) < 2``, returns (values[0], values[0], values[0]) or (0, 0, 0) for empty input. Uses ``random.Random(seed)`` for determinism. + + Raises ValueError for a ``confidence`` outside (0, 1) or a non-positive + ``n_resamples`` — clamping those would quietly return an interval of the + wrong width, which is worse than refusing. """ + if not 0.0 < confidence < 1.0: + raise ValueError(f"confidence must be in (0, 1), got {confidence!r}") + if n_resamples < 1: + raise ValueError(f"n_resamples must be >= 1, got {n_resamples!r}") if not values: return (0.0, 0.0, 0.0) m = sum(values) / len(values) @@ -126,30 +212,192 @@ def wilson_interval(successes: int, n: int, confidence: float = 0.95) -> tuple[f return (max(0.0, center - half), min(1.0, center + half)) -def paired_bootstrap_diff_ci( - a: list[float], - b: list[float], - n_resamples: int = 1000, - confidence: float = 0.95, - seed: int = 0, -) -> tuple[float, float, float] | None: - """Paired bootstrap of mean(a_i - b_i). +def cohens_d(a: list[float], b: list[float]) -> float | None: + """Paired Cohen's d = mean(a_i - b_i) / stddev(a_i - b_i).""" + if len(a) != len(b) or len(a) < 2: + return None + diffs = [ai - bi for ai, bi in zip(a, b, strict=True)] + s = stddev(diffs) + return (sum(diffs) / len(diffs)) / s if s > 0 else None - Returns (mean_diff, ci_low, ci_high) or None if lengths differ or < 2. + +def student_t_critical(confidence: float, df: float) -> float: + """Two-tailed critical value t* with P(|T| >= t*) = 1 - confidence. + + Inverts :func:`student_t_two_tailed_p` by bisection — that p is continuous and + strictly decreasing in |t|, so a plain bracket-and-halve is exact to ~1e-12 and + needs no separate quantile expansion. + """ + if not 0.0 < confidence < 1.0: + raise ValueError(f"confidence must be in (0, 1), got {confidence!r}") + if df <= 0 or not math.isfinite(df): + return math.inf + alpha = 1.0 - confidence + lo, hi = 0.0, 1.0 + while student_t_two_tailed_p(hi, df) > alpha: + lo = hi + hi *= 2.0 + if hi > 1e12: + # Only reachable for a confidence so close to 1 that t* overflows the + # bracket. Warn rather than return a silently wrong-width interval. + logger.warning( + "student_t_critical failed to bracket t* for confidence=%r, df=%r; returning a degraded upper bound", + confidence, + df, + ) + return hi + for _ in range(200): + mid = (lo + hi) / 2.0 + if mid in (lo, hi): + break + if student_t_two_tailed_p(mid, df) > alpha: + lo = mid + else: + hi = mid + return (lo + hi) / 2.0 + + +def paired_t_ci(a: list[float], b: list[float], confidence: float = 0.95) -> tuple[float, float, float] | None: + """Student-t confidence interval for mean(a_i - b_i): mean ± t* · sd/√n. + + Returns (mean_diff, ci_low, ci_high), or None if lengths differ, n < 2, or any + value is non-finite. Shares its distribution with :func:`paired_t_test`, so the + interval and the p-value always agree about whether 0 is excluded. """ if len(a) != len(b) or len(a) < 2: return None + if not all(math.isfinite(v) for v in (*a, *b)): + return None diffs = [ai - bi for ai, bi in zip(a, b, strict=True)] - return bootstrap_mean_ci(diffs, n_resamples=n_resamples, confidence=confidence, seed=seed) + n = len(diffs) + mean_diff = sum(diffs) / n + half_width = student_t_critical(confidence, n - 1) * stddev(diffs) / math.sqrt(n) + return (mean_diff, mean_diff - half_width, mean_diff + half_width) -def cohens_d(a: list[float], b: list[float]) -> float | None: - """Paired Cohen's d = mean(a_i - b_i) / stddev(a_i - b_i).""" +def paired_t_test(a: list[float], b: list[float]) -> float | None: + """Two-tailed p-value from a paired t-test on (a_i - b_i), exact t distribution. + + Equivalent to a one-sample t-test of the differences against 0, df = n - 1. + Returns None if lengths differ, n < 2, or any value is non-finite. + """ if len(a) != len(b) or len(a) < 2: return None + if not all(math.isfinite(v) for v in (*a, *b)): + return None diffs = [ai - bi for ai, bi in zip(a, b, strict=True)] - s = stddev(diffs) - return (sum(diffs) / len(diffs)) / s if s > 0 else None + sd = stddev(diffs) + mean_diff = sum(diffs) / len(diffs) + if sd == 0: + # All diffs identical: no difference (p=1) or a deterministic shift (p=0). + return 1.0 if mean_diff == 0 else 0.0 + t_stat = abs(mean_diff) / (sd / math.sqrt(len(diffs))) + return student_t_two_tailed_p(t_stat, len(diffs) - 1) + + +# --------------------------------------------------------------------------- +# Aggregate-metric series +# --------------------------------------------------------------------------- + + +class VariantSeries(NamedTuple): + """One variant's numeric series across all tasks — the raw inputs to the + Aggregate Metrics rows and their p-values.""" + + scores: list[float] + durations: list[float] + tokens: list[float] + asst_turns: list[float] + + +def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries]: + """Per-variant (scores, durations, tokens, assistant-turns) series, keyed by variant id. + + Shared by the markdown and HTML reporters so both render the same numbers. + ``VariantResult.duration_seconds`` is *summed* across replicates, so it is + divided by ``replicate_count`` to give a per-run duration comparable across + variants that ran different replicate counts. + """ + series = {vid: VariantSeries([], [], [], []) for vid in result.variant_ids} + for ts in result.task_summaries: + for vr in ts.variant_results: + s = series.get(vr.variant_id) + if s is None: # a task result for a variant not in variant_ids + continue + s.scores.append(vr.weighted_score) + s.durations.append(vr.duration_seconds / vr.replicate_count) + if vr.total_tokens is not None: + s.tokens.append(float(vr.total_tokens)) + if vr.total_assistant_turns is not None: + s.asst_turns.append(float(vr.total_assistant_turns)) + return series + + +class PairedComparison(NamedTuple): + """A 2-variant paired comparison over per-task mean scores. + + ``task_count`` is the number of tasks both variants scored. When it is < 2 + the statistics are all ``None`` — there is nothing to compare, and the + reporters say so rather than rendering an empty section. ``excluded_count`` + is the number of tasks that appeared for at least one variant but could not + be paired (missing or empty on the other side); the reporters surface it so + a silently narrowed sample is visible. + """ + + vid_a: str + vid_b: str + task_count: int + excluded_count: int + mean_diff: float | None + ci_low: float | None + ci_high: float | None + effect_size: float | None + p_value: float | None + + +def paired_comparison(result: ExperimentResult, confidence: float = 0.95) -> PairedComparison | None: + """Pair the two variants' per-task mean scores. Returns None unless the + experiment has exactly 2 variants with at least one commonly-scored task. + + The task is the unit of analysis: replicate slots within a task share the task + effect and are not independent, so pairing them individually would understate + the standard error. Replicate counts need not match — a task's mean score is a + well-defined pair member either way. + """ + if len(result.variant_ids) != 2: + return None + vid_a, vid_b = result.variant_ids[0], result.variant_ids[1] + per_rep_a = result.per_replicate_scores.get(vid_a, {}) + per_rep_b = result.per_replicate_scores.get(vid_b, {}) + common_tasks = sorted(t for t in set(per_rep_a) & set(per_rep_b) if per_rep_a[t] and per_rep_b[t]) + if not common_tasks: + # No shared task, or per_replicate_scores absent (results from before it existed). + return None + + # Tasks seen for at least one variant but not paired (missing or empty on the + # other side) — surfaced so a silently narrowed sample doesn't go unnoticed. + excluded_count = len(set(per_rep_a) | set(per_rep_b)) - len(common_tasks) + + if len(common_tasks) < 2: + return PairedComparison(vid_a, vid_b, len(common_tasks), excluded_count, None, None, None, None, None) + + a_scores = [mean(per_rep_a[task_id]) for task_id in common_tasks] + b_scores = [mean(per_rep_b[task_id]) for task_id in common_tasks] + ci = paired_t_ci(a_scores, b_scores, confidence=confidence) + if ci is None: # non-finite scores + return PairedComparison(vid_a, vid_b, len(common_tasks), excluded_count, None, None, None, None, None) + mean_diff, ci_low, ci_high = ci + return PairedComparison( + vid_a, + vid_b, + len(common_tasks), + excluded_count, + mean_diff, + ci_low, + ci_high, + cohens_d(a_scores, b_scores), + paired_t_test(a_scores, b_scores), + ) # --------------------------------------------------------------------------- diff --git a/tests/_fixtures/report_snapshots/experiment_2variant.md b/tests/_fixtures/report_snapshots/experiment_2variant.md index c9d13502..e19649fd 100644 --- a/tests/_fixtures/report_snapshots/experiment_2variant.md +++ b/tests/_fixtures/report_snapshots/experiment_2variant.md @@ -20,10 +20,10 @@ | - Cost budget | 0 | 1 | — | | Errors | 0 | 0 | — | | Success Rate | 50.0% | 50.0% | — | -| Score | 0.700 ± 0.283 | 0.825 ± 0.177 | 0.596 | -| Avg Duration (s) | 35.0 ± 7.1 | 55.0 ± 7.1 | 0.005 | -| Assistant Turns | 5.5 ± 0.7 | 7.5 ± 0.7 | 0.005 | -| Tokens | 1,100 ± 141 | 1,900 ± 141 | <0.001 | +| Score | 0.700 ± 0.283 | 0.825 ± 0.177 | 0.658 | +| Avg Duration (s) | 35.0 ± 7.1 | 55.0 ± 7.1 | 0.106 | +| Assistant Turns | 5.5 ± 0.7 | 7.5 ± 0.7 | 0.106 | +| Tokens | 1,100 ± 141 | 1,900 ± 141 | 0.030 | ## Win Rates @@ -41,3 +41,8 @@ - **task-b**: spread=0.450, best=mutated - **task-a**: spread=0.200, best=baseline + +## Paired Comparison + +*Paired over the per-task mean score of 2 task(s) common to both variants — pairing cancels between-task difficulty, which the pooled Welch test above cannot.* +**Paired mean diff (baseline - mutated)**: -0.125 [95% CI -4.255, +4.005], Cohen's d = -0.27, p = 0.766 diff --git a/tests/_fixtures/report_snapshots/experiment_replicates.md b/tests/_fixtures/report_snapshots/experiment_replicates.md index fa1c31b8..dfcd8988 100644 --- a/tests/_fixtures/report_snapshots/experiment_replicates.md +++ b/tests/_fixtures/report_snapshots/experiment_replicates.md @@ -40,4 +40,6 @@ | a | 3 | 0.900 | [0.850, 0.933] | 2/3 [0.21, 0.94] | | b | 3 | 0.650 | [0.600, 0.683] | 0/3 [0.00, 0.56] | -**Paired mean diff (a - b)**: +0.250 [95% CI +0.200, +0.300], Cohen's d = 5.00 +## Paired Comparison + +*A paired comparison needs at least 2 tasks common to a and b; found 1.* diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index c3bc3f5e..de432278 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -748,6 +748,102 @@ def test_welch_t_test_insufficient_data(self): assert welch_t_test([1.0], [2.0]) is None + def test_welch_t_test_exact_reference_value(self): + """Exact Student-t p-value, cross-checked against scipy ttest_ind(equal_var=False).""" + from coder_eval.reports_stats import welch_t_test + + # Equal variances 2.5, n=5 each ⇒ t=1.0, Welch-Satterthwaite df=8. + p = welch_t_test([1.0, 2.0, 3.0, 4.0, 5.0], [2.0, 3.0, 4.0, 5.0, 6.0]) + assert p is not None + assert abs(p - 0.34659) < 5e-4 + + def test_welch_t_test_zero_variance_different_means(self): + """Zero variance in both groups: deterministic difference ⇒ 0.0, identical ⇒ 1.0.""" + from coder_eval.reports_stats import welch_t_test + + assert welch_t_test([1.0, 1.0], [2.0, 2.0]) == 0.0 + assert welch_t_test([1.0, 1.0], [1.0, 1.0]) == 1.0 + + def test_student_t_two_tailed_p_small_df_not_normal(self): + """Small df must use t tails, not normal ones — the regression sensor for the old bug.""" + from coder_eval.reports_stats import student_t_two_tailed_p + + p = student_t_two_tailed_p(2.5, 4.0) + assert abs(p - 0.06677) < 5e-4 + # The old normal approximation reported 0.0124 here — falsely significant. + assert p > 0.05 + + def test_student_t_two_tailed_p_critical_values(self): + """Round-trip the standard t-table critical values.""" + from coder_eval.reports_stats import student_t_two_tailed_p + + assert abs(student_t_two_tailed_p(2.776, 4.0) - 0.05) < 1e-3 + assert abs(student_t_two_tailed_p(2.228, 10.0) - 0.05) < 1e-3 + assert abs(student_t_two_tailed_p(0.0, 7.0) - 1.0) < 1e-12 + # Degenerate df is not reachable through the t-tests, but the helper stays total. + assert student_t_two_tailed_p(1.0, 0.0) == 1.0 + + def test_student_t_two_tailed_p_large_df_matches_normal(self): + """At huge df the t distribution converges to the normal.""" + import statistics + + from coder_eval.reports_stats import student_t_two_tailed_p + + normal_p = 2.0 * statistics.NormalDist().cdf(-1.96) + assert abs(student_t_two_tailed_p(1.96, 1e6) - normal_p) < 1e-5 + + def test_regularized_incomplete_beta_closed_forms(self): + """I_x(a,b) against the closed forms it has for b=1, a=1, and the symmetric point.""" + from coder_eval.reports_stats import regularized_incomplete_beta + + assert abs(regularized_incomplete_beta(2.0, 1.0, 0.3) - 0.3**2) < 1e-10 + assert abs(regularized_incomplete_beta(1.0, 3.0, 0.4) - (1.0 - 0.6**3)) < 1e-10 + assert abs(regularized_incomplete_beta(5.0, 5.0, 0.5) - 0.5) < 1e-10 + assert regularized_incomplete_beta(2.0, 3.0, 0.0) == 0.0 + assert regularized_incomplete_beta(2.0, 3.0, 1.0) == 1.0 + + def test_paired_t_test_exact_reference_value(self): + """Exact paired-t p-value, cross-checked against scipy ttest_rel.""" + from coder_eval.reports_stats import paired_t_test, welch_t_test + + a = [0.9, 0.5, 0.8, 0.7, 0.95] + b = [0.7, 0.55, 0.6, 0.72, 0.8] + p_paired = paired_t_test(a, b) + assert p_paired is not None + assert abs(p_paired - 0.15273) < 5e-4 + # On positively correlated pairs like these, pairing is the sharper test — + # but that is a property of this data, not a guarantee (it loses df). + p_welch = welch_t_test(a, b) + assert p_welch is not None + assert p_paired < p_welch + + def test_paired_t_test_constant_shift_and_identical(self): + """Zero-sd differences: deterministic shift ⇒ 0.0, identical lists ⇒ 1.0.""" + from coder_eval.reports_stats import paired_t_test + + assert paired_t_test([1.0, 2.0, 3.0, 4.0, 5.0], [2.0, 3.0, 4.0, 5.0, 6.0]) == 0.0 + assert paired_t_test([1.0, 2.0, 3.0], [1.0, 2.0, 3.0]) == 1.0 + + def test_paired_t_test_invalid_input(self): + """Length mismatch or fewer than 2 pairs returns None.""" + from coder_eval.reports_stats import paired_t_test + + assert paired_t_test([1.0, 2.0], [1.0]) is None + assert paired_t_test([1.0], [2.0]) is None + + def test_non_finite_inputs_never_report_significance(self): + """NaN/inf must not produce a fabricated p-value — fail closed, never significant.""" + from coder_eval.reports_stats import fmt_p, paired_t_test, student_t_two_tailed_p, welch_t_test + + nan, inf = float("nan"), float("inf") + assert student_t_two_tailed_p(nan, 4.0) == 1.0 + assert student_t_two_tailed_p(2.5, nan) == 1.0 + assert student_t_two_tailed_p(inf, 4.0) == 1.0 + # The list-taking helpers can say "no result", which renders as an em dash. + assert welch_t_test([1.0, nan], [2.0, 3.0]) is None + assert paired_t_test([1.0, nan], [2.0, 3.0]) is None + assert fmt_p(welch_t_test([1.0, inf], [2.0, 3.0])) == "—" + def test_mean_and_stddev(self): """Basic mean and stddev calculations.""" from coder_eval.reports_stats import mean, stddev @@ -1067,25 +1163,27 @@ def test_section_contains_per_variant_ci_columns(self): assert "Pass-rate" in md def test_paired_diff_line_for_two_variants_equal_counts(self): + # Two tasks: the paired section pairs their per-task means (n = 2 tasks). per_rep = { - "a": {"task-1": [0.9, 0.85, 0.95]}, - "b": {"task-1": [0.6, 0.65, 0.7]}, + "a": {"task-1": [0.9, 0.85, 0.95], "task-2": [0.7, 0.75, 0.8]}, + "b": {"task-1": [0.6, 0.65, 0.7], "task-2": [0.5, 0.55, 0.6]}, } result = self._make_result(replicate_count=3, per_replicate_scores=per_rep) md = ExperimentReportGenerator.generate_experiment_report(result) assert "Paired mean diff" in md assert "Cohen's d" in md - def test_paired_diff_skipped_when_unequal_counts_across_tasks(self): - # Variant a has 3 replicates for task-1, variant b has 5 → paired skipped + def test_paired_diff_needs_two_common_tasks(self): + # Unequal replicate counts no longer exclude a task, but one task is still + # a single pair — too few for a paired comparison. per_rep = { "a": {"task-1": [0.9, 0.85, 0.95]}, "b": {"task-1": [0.6, 0.65, 0.7, 0.75, 0.8]}, } result = self._make_result(replicate_count=3, per_replicate_scores=per_rep) md = ExperimentReportGenerator.generate_experiment_report(result) - assert "Paired statistics skipped" in md - assert "unequal replicate counts" in md + assert "needs at least 2 tasks" in md + assert "Paired mean diff" not in md def test_no_paired_diff_for_single_variant(self): per_rep = {"only": {"task-1": [0.7, 0.8, 0.9]}} @@ -1117,11 +1215,272 @@ def test_variant_report_no_ci_when_replicate_count_is_one(self): assert "Score 95% CI" not in md +class TestCollectVariantSeries: + """The one series collector both reporters share.""" + + @staticmethod + def _result_with_replicates() -> ExperimentResult: + """One task per variant, 3 replicates, 90s summed ⇒ 30s per run.""" + return ExperimentResult( + experiment_id="exp", + description="d", + variant_ids=["a"], + task_summaries=[ + TaskExperimentSummary( + task_id="t1", + variant_results=[ + VariantResult( + variant_id="a", + task_id="t1", + weighted_score=0.8, + final_status="SUCCESS", + duration_seconds=90.0, + replicate_count=3, + ) + ], + best_variant="a", + score_spread=0.0, + ) + ], + variant_aggregates={ + "a": VariantAggregate( + variant_id="a", + tasks_run=1, + tasks_succeeded=1, + tasks_failed=0, + tasks_error=0, + average_score=0.8, + average_duration=30.0, + replicate_count=3, + ) + }, + total_duration_seconds=90.0, + ) + + def test_duration_is_per_run_not_replicate_inflated(self): + """duration_seconds is summed across replicates, so it must be divided out.""" + from coder_eval.reports_stats import collect_variant_series + + series = collect_variant_series(self._result_with_replicates()) + assert series["a"].durations == [30.0] + + def test_html_and_markdown_report_the_same_duration(self): + """Regression: the HTML copy of this collector used the raw summed duration.""" + from coder_eval.reports_html import _experiment_aggregate_metrics + + result = self._result_with_replicates() + md = ExperimentReportGenerator.generate_experiment_report(result) + html = _experiment_aggregate_metrics(result) + assert "| Avg Duration (s) | 30.0 |" in md + assert "30.0" in html + assert "90.0" not in html + + def test_result_for_unknown_variant_is_ignored(self): + """A task result naming a variant outside variant_ids must not raise.""" + from coder_eval.reports_stats import collect_variant_series + + result = self._result_with_replicates() + result.task_summaries[0].variant_results.append( + VariantResult( + variant_id="ghost", + task_id="t1", + weighted_score=0.1, + final_status="SUCCESS", + duration_seconds=1.0, + ) + ) + series = collect_variant_series(result) + assert set(series) == {"a"} + + +class TestPairedComparisonSection: + """Tests for the ``## Paired Comparison`` markdown section.""" + + @staticmethod + def _make_result( + variant_ids: list[str], + per_replicate_scores: dict[str, dict[str, list[float]]], + ) -> ExperimentResult: + task_ids = sorted({tid for per_task in per_replicate_scores.values() for tid in per_task}) + return ExperimentResult( + experiment_id="exp", + description="d", + variant_ids=variant_ids, + task_summaries=[ + TaskExperimentSummary( + task_id=task_id, + variant_results=[ + VariantResult( + variant_id=vid, + task_id=task_id, + weighted_score=next(iter(per_replicate_scores.get(vid, {}).get(task_id, [])), 0.0), + final_status="SUCCESS", + duration_seconds=1.0, + ) + for vid in variant_ids + ], + best_variant=variant_ids[0], + score_spread=0.1, + ) + for task_id in task_ids + ], + variant_aggregates={ + vid: VariantAggregate( + variant_id=vid, + tasks_run=len(task_ids), + tasks_succeeded=len(task_ids), + tasks_failed=0, + tasks_error=0, + average_score=0.5, + average_duration=1.0, + ) + for vid in variant_ids + }, + total_duration_seconds=10.0, + per_replicate_scores=per_replicate_scores, + ) + + def test_paired_comparison_section_two_variants_single_replicate(self): + """A single-replicate 2-variant experiment gets the paired section (no replicate gate).""" + result = self._make_result( + ["a", "b"], + { + "a": {"t1": [0.9], "t2": [0.5], "t3": [0.8]}, + "b": {"t1": [0.7], "t2": [0.55], "t3": [0.6]}, + }, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "## Paired Comparison" in md + assert "**Paired mean diff (a - b)**" in md + assert ", p = " in md + assert "Cohen's d" in md + # Single replicate ⇒ no Replicate Statistics section, but paired still renders. + assert "## Replicate Statistics" not in md + + def test_paired_comparison_section_absent(self): + """3 variants, or an empty per_replicate_scores, render no paired section.""" + three = self._make_result( + ["a", "b", "c"], + {"a": {"t1": [0.9]}, "b": {"t1": [0.7]}, "c": {"t1": [0.6]}}, + ) + assert "## Paired Comparison" not in ExperimentReportGenerator.generate_experiment_report(three) + + empty = self._make_result(["a", "b"], {}) + empty.task_summaries = [ + TaskExperimentSummary( + task_id="t1", + variant_results=[ + VariantResult( + variant_id=vid, + task_id="t1", + weighted_score=0.5, + final_status="SUCCESS", + duration_seconds=1.0, + ) + for vid in ("a", "b") + ], + best_variant="a", + score_spread=0.0, + ) + ] + assert "## Paired Comparison" not in ExperimentReportGenerator.generate_experiment_report(empty) + + def test_paired_comparison_averages_replicates_per_task(self): + """Replicates are averaged per task, so n is the task count — not the slot count.""" + result = self._make_result( + ["a", "b"], + { + "a": {"t1": [0.8, 1.0], "t2": [0.4, 0.6]}, + "b": {"t1": [0.5, 0.7], "t2": [0.2, 0.4]}, + }, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "per-task mean score of 2 task(s) common to both variants" in md + # Task means: a = [0.9, 0.5], b = [0.6, 0.3] ⇒ diffs [+0.3, +0.2], mean +0.250. + assert "**Paired mean diff (a - b)**: +0.250" in md + + def test_paired_comparison_single_common_task(self): + """One common task ⇒ the section explains why there is no paired result.""" + result = self._make_result( + ["a", "b"], + {"a": {"t1": [0.9, 0.85, 0.95]}, "b": {"t1": [0.6, 0.65, 0.7]}}, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "## Paired Comparison" in md + assert "*A paired comparison needs at least 2 tasks common to a and b; found 1.*" in md + assert "Paired mean diff" not in md + + def test_paired_comparison_keeps_tasks_with_unequal_replicate_counts(self): + """Pairing per-task means needs no equal replicate counts — no task is dropped.""" + result = self._make_result( + ["a", "b"], + { + "a": {"t1": [0.5, 0.7], "t2": [0.4], "t3": [0.8]}, + "b": {"t1": [0.5], "t2": [0.45], "t3": [0.7]}, + }, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "per-task mean score of 3 task(s) common to both variants" in md + assert "excluded" not in md + # Task means: a = [0.6, 0.4, 0.8], b = [0.5, 0.45, 0.7] ⇒ diffs [+0.1, -0.05, +0.1]. + assert "**Paired mean diff (a - b)**: +0.050" in md + + def test_html_renders_the_same_paired_numbers_as_markdown(self): + """Both reporters render one shared computation, so they cannot disagree.""" + from coder_eval.reports_html import _experiment_paired_comparison + + result = self._make_result( + ["a", "b"], + {"a": {"t1": [0.9], "t2": [0.5], "t3": [0.8]}, "b": {"t1": [0.7], "t2": [0.55], "t3": [0.6]}}, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + html = _experiment_paired_comparison(result) + assert "

Paired Comparison

" in html + # Cohen's apostrophe is HTML-escaped, so match the numbers either side of it. + for fragment in ("Paired mean diff (a - b): +0.117", "d = 0.81", "p = 0.296"): + assert fragment in html + assert "**Paired mean diff (a - b)**: +0.117 [95% CI -0.242, +0.475], Cohen's d = 0.81, p = 0.296" in md + + def test_html_paired_section_absent_for_three_variants(self): + from coder_eval.reports_html import _experiment_paired_comparison + + result = self._make_result( + ["a", "b", "c"], + {"a": {"t1": [0.9]}, "b": {"t1": [0.7]}, "c": {"t1": [0.6]}}, + ) + assert _experiment_paired_comparison(result) == "" + + def test_paired_ci_agrees_with_p_value(self): + """The interval and the p-value share a distribution, so they agree about 0.""" + result = self._make_result( + ["a", "b"], + { + "a": {"t1": [0.9], "t2": [0.85], "t3": [0.95], "t4": [0.9]}, + "b": {"t1": [0.2], "t2": [0.25], "t3": [0.15], "t4": [0.3]}, + }, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + # A large, consistent gap ⇒ p well below 0.05 and a CI clear of zero. + assert "p = <0.001" in md + assert "[95% CI +0." in md + + def test_paired_comparison_ignores_tasks_with_no_scores(self): + """A task with an empty replicate list on either side is not a pair, and the + exclusion is surfaced in the rendered note.""" + result = self._make_result( + ["a", "b"], + {"a": {"t1": [0.9], "t2": [0.5], "t3": []}, "b": {"t1": [0.7], "t2": [0.6], "t3": [0.4]}}, + ) + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "per-task mean score of 2 task(s) common to both variants" in md + assert "(1 task(s) excluded — not scored by both variants)" in md + + class TestExperimentReportSnapshots: """Byte-identical characterization snapshots for generate_experiment_report — the - safety net for its decomposition. Output is deterministic: bootstrap_mean_ci / - paired_bootstrap_diff_ci in reports_stats use a fixed default seed, so no scrubbing - is needed (do not pass a varying seed).""" + safety net for its decomposition. Output is deterministic: bootstrap_mean_ci in + reports_stats uses a fixed default seed, so no scrubbing is needed (do not pass a + varying seed); the paired section is closed-form Student-t with no randomness.""" def test_experiment_report_snapshot_2variant(self): """2 variants → p-value column shown; plus prompt config, budget sub-rows, @@ -1217,6 +1576,10 @@ def test_experiment_report_snapshot_2variant(self): ), }, total_duration_seconds=180.0, + per_replicate_scores={ + "baseline": {"task-a": [0.9], "task-b": [0.5]}, + "mutated": {"task-a": [0.7], "task-b": [0.95]}, + }, ) md = ExperimentReportGenerator.generate_experiment_report(result, experiment=experiment) assert_matches_snapshot(md, "experiment_2variant.md") diff --git a/tests/test_replicate_stats.py b/tests/test_replicate_stats.py index 2d1442ee..e0c99012 100644 --- a/tests/test_replicate_stats.py +++ b/tests/test_replicate_stats.py @@ -1,9 +1,13 @@ """Unit tests for replicate statistics helpers in reports_stats.""" +import pytest + from coder_eval.reports_stats import ( bootstrap_mean_ci, cohens_d, - paired_bootstrap_diff_ci, + paired_t_ci, + paired_t_test, + student_t_critical, wilson_interval, ) @@ -15,6 +19,16 @@ def test_empty_returns_triple_zero(self): def test_single_value_returns_triple(self): assert bootstrap_mean_ci([0.7]) == (0.7, 0.7, 0.7) + @pytest.mark.parametrize("confidence", [-1.0, 0.0, 1.0, 1.1]) + def test_confidence_outside_unit_interval_raises(self, confidence): + """Out-of-domain confidence must refuse, not return a wrong-width interval.""" + with pytest.raises(ValueError, match="confidence must be in"): + bootstrap_mean_ci([0.1, 0.5, 0.9], confidence=confidence) + + def test_non_positive_n_resamples_raises(self): + with pytest.raises(ValueError, match="n_resamples must be"): + bootstrap_mean_ci([0.1, 0.5, 0.9], n_resamples=0) + def test_mean_is_correct(self): m, _lo, _hi = bootstrap_mean_ci([0.0, 1.0]) assert abs(m - 0.5) < 1e-9 @@ -82,54 +96,62 @@ def test_bounds_clamped_to_unit_interval(self): assert 0.0 <= hi <= 1.0 -class TestPairedBootstrapDiffCi: - def test_none_on_length_mismatch(self): - assert paired_bootstrap_diff_ci([1.0, 2.0], [1.0]) is None - - def test_none_on_single_pair(self): - assert paired_bootstrap_diff_ci([1.0], [0.5]) is None - - def test_none_on_empty(self): - assert paired_bootstrap_diff_ci([], []) is None - - def test_detects_positive_shift(self): - a = [1.0] * 20 - b = [0.5] * 20 - result = paired_bootstrap_diff_ci(a, b) - assert result is not None - mean_diff, lo, hi = result - assert abs(mean_diff - 0.5) < 0.01 - # CI should be very tight around 0.5 (zero variance) - assert lo > 0.45 - assert hi < 0.55 +class TestPairedTCi: + """The Student-t paired interval that replaced the percentile bootstrap — it + shares a distribution with paired_t_test, so the two always agree about 0.""" - def test_detects_negative_shift(self): - # Use non-constant inputs so CI is not degenerate - import random + def test_none_on_invalid_input(self): + assert paired_t_ci([1.0, 2.0], [1.0]) is None + assert paired_t_ci([1.0], [0.5]) is None + assert paired_t_ci([], []) is None + assert paired_t_ci([1.0, float("nan")], [0.5, 0.5]) is None - rng = random.Random(7) - a = [0.3 + rng.gauss(0, 0.05) for _ in range(20)] - b = [0.8 + rng.gauss(0, 0.05) for _ in range(20)] - result = paired_bootstrap_diff_ci(a, b) + def test_zero_variance_shift_is_a_point_interval(self): + result = paired_t_ci([1.0] * 20, [0.5] * 20) assert result is not None mean_diff, lo, hi = result - assert mean_diff < 0 - assert lo < hi - - def test_ci_contains_diff(self): - a = [float(i) / 9 for i in range(10)] - b = [float(i) / 9 + 0.1 for i in range(10)] - result = paired_bootstrap_diff_ci(a, b) + assert abs(mean_diff - 0.5) < 1e-12 + # Every diff identical ⇒ sd = 0 ⇒ the interval collapses onto the mean. + assert (lo, hi) == (0.5, 0.5) + + def test_ci_contains_diff_and_is_symmetric(self): + a = [0.3, 0.5, 0.9, 0.2, 0.7] + b = [0.2, 0.55, 0.6, 0.3, 0.5] + result = paired_t_ci(a, b) assert result is not None mean_diff, lo, hi = result assert lo <= mean_diff <= hi - - def test_is_deterministic(self): - a = [0.1 * i for i in range(1, 11)] - b = [0.1 * i + 0.05 for i in range(1, 11)] - r1 = paired_bootstrap_diff_ci(a, b) - r2 = paired_bootstrap_diff_ci(a, b) - assert r1 == r2 + assert abs((mean_diff - lo) - (hi - mean_diff)) < 1e-12 + + def test_agrees_with_paired_t_test_about_zero(self): + """The interval excludes 0 exactly when the p-value is below alpha.""" + for a, b in ( + ([0.9, 0.5, 0.8, 0.7, 0.95], [0.7, 0.55, 0.6, 0.72, 0.8]), # p = 0.153, not significant + ([0.9, 0.85, 0.95, 0.8, 0.9], [0.2, 0.25, 0.15, 0.3, 0.2]), # strongly significant + ): + ci = paired_t_ci(a, b, confidence=0.95) + p = paired_t_test(a, b) + assert ci is not None and p is not None + _, lo, hi = ci + excludes_zero = lo > 0 or hi < 0 + assert excludes_zero == (p < 0.05) + + +class TestStudentTCritical: + def test_matches_t_table(self): + assert abs(student_t_critical(0.95, 4) - 2.776) < 1e-3 + assert abs(student_t_critical(0.95, 10) - 2.228) < 1e-3 + assert abs(student_t_critical(0.99, 10) - 3.169) < 1e-3 + + def test_converges_to_normal_at_large_df(self): + assert abs(student_t_critical(0.95, 1e7) - 1.959964) < 1e-4 + + def test_rejects_confidence_outside_unit_interval(self): + with pytest.raises(ValueError, match="confidence must be in"): + student_t_critical(1.0, 5) + + def test_degenerate_df_is_infinite(self): + assert student_t_critical(0.95, 0) == float("inf") class TestCohensD: diff --git a/tests/test_reports_stats_nonfinite.py b/tests/test_reports_stats_nonfinite.py new file mode 100644 index 00000000..fe9f8ce6 --- /dev/null +++ b/tests/test_reports_stats_nonfinite.py @@ -0,0 +1,73 @@ +"""Harness: no public numeric helper in reports_stats may launder NaN/inf into a report. + +This enumerates the module rather than listing functions by hand, so a helper added +later is covered automatically — the failure mode this guards (a new statistic that +silently returns NaN, which then renders as a real-looking number) is exactly the one +that motivated the exact-t work. See review-rubric.md criterion 15. + +To add a helper that legitimately propagates non-finite input, add it to +``_PASSTHROUGH_HELPERS`` with a reason — that edit is the point, it forces the call +to be deliberate. +""" + +import inspect +import math + +import pytest + +from coder_eval import reports_stats + + +# Thin wrappers over the stdlib that intentionally propagate whatever they are given; +# they are inputs to the guarded functions below, not report-facing statistics. +_PASSTHROUGH_HELPERS = {"mean", "stddev"} + +_NAN = float("nan") +_INF = float("inf") + + +def _numeric_helpers(): + """Public reports_stats functions whose parameters are all floats / float lists.""" + for name, fn in vars(reports_stats).items(): + if name.startswith("_") or name in _PASSTHROUGH_HELPERS or not inspect.isfunction(fn): + continue + if fn.__module__ != reports_stats.__name__: + continue + hints = inspect.get_annotations(fn, eval_str=True) + params = inspect.signature(fn).parameters + annotations = [hints.get(p) for p in params] + if annotations and all(ann is float or ann == list[float] for ann in annotations): + yield name, fn, params, hints + + +def _bad_args(params, hints, bad_value): + args = [] + for name in params: + args.append([1.0, bad_value] if hints.get(name) == list[float] else bad_value) + return args + + +def _floats_in(value): + if isinstance(value, float): + yield value + elif isinstance(value, tuple): + for item in value: + yield from _floats_in(item) + + +@pytest.mark.parametrize("bad_value", [_NAN, _INF, -_INF], ids=["nan", "inf", "-inf"]) +def test_no_public_numeric_helper_returns_nan(bad_value): + """Non-finite input must yield None, a finite number, or an explicit ValueError.""" + checked = [] + for name, fn, params, hints in _numeric_helpers(): + checked.append(name) + try: + result = fn(*_bad_args(params, hints, bad_value)) + except ValueError: + continue # refusing explicitly is a valid response + for f in _floats_in(result): + assert not math.isnan(f), f"{name} returned NaN for {bad_value} input" + + # Guard the guard: if the discovery filter silently matches nothing, the test + # above would pass vacuously. + assert {"welch_t_test", "paired_t_test", "paired_t_ci", "student_t_two_tailed_p"} <= set(checked)