diff --git a/docs/source/content/contributing.md b/docs/source/content/contributing.md index 20e10f5ce1..02cadb3990 100644 --- a/docs/source/content/contributing.md +++ b/docs/source/content/contributing.md @@ -328,7 +328,7 @@ set -a; source .env; set +a uv run python -m transformer_lens.tools.model_registry.verify_models --model ``` -`verify_models` runs phases 1–4 (forward correctness vs HF, hook firing + gradients, weight processing, generation quality) and updates `data/supported_models.json` with the resulting status and per-phase scores. We recommend running `--dry-run` first to project memory and parameter count without loading the model, and verifying one model at a time — concurrent loads tend to OOM a single device. +`verify_models` runs phases 1–4 (forward correctness vs HF, hook firing + gradients, weight processing, text-generation quality) and updates `data/supported_models.json` with the resulting status and per-phase scores. We recommend running `--dry-run` first to project memory and parameter count without loading the model, and verifying one model at a time — concurrent loads tend to OOM a single device. Running with `--no-hf-reference` skips the HuggingFace numerical comparison (Phase 1 becomes structural-only). A passing run is then recorded as **provisional** (status 4), which does *not* count as verified — re-run without the flag for a real HF-compared verification. @@ -341,11 +341,11 @@ It's worth reading the per-phase scores in addition to the final status — the | 1 | 100% | — | Verification fails | | 2 | 75% | `logits_equivalence`, `loss_equivalence` | Verification fails | | 3 | 75% | `logits_equivalence`, `loss_equivalence` | Verification fails | -| 4 | 50% | — | **Non-gating** — below 50% adds `"low text quality"` to the registry `note`; never fails verification. | +| 4 | 54.5% (measured pass line, `p4_pass_threshold()`) | — | **Non-gating** — below the line adds a `"text quality poor (P4=…)"` note; never fails verification. | | 7 | 75% | `multimodal_forward` | Verification fails. A NULL score also fails. | | 8 | 75% | `audio_forward` | Verification fails. A NULL score also fails. | -Phase 4 is intentionally lenient — it's a coherence metric, not a correctness check. A sub-100% Phase-4 score on a small parity-test model can still indicate a real adapter bug that the gates don't catch (missing `preprocess_weights` fold, wrong `default_prepend_bos`, and so on); the model can pass verification overall and still be worth a manual look. +Phase 4 prompts each model with its resolved prompt profile (chat template, translation, code, own-language continuation, ...) and scores the generation against a known-good reference with one pinned multilingual judge, via the perplexity ratio `PPL(generated)/PPL(reference)`. It's intentionally lenient — a coherence metric, not a correctness check. A sub-100% Phase-4 score on a small parity-test model can still indicate a real adapter bug that the gates don't catch (missing `preprocess_weights` fold, wrong `default_prepend_bos`, and so on); the model can pass verification overall and still be worth a manual look. If verification fails by `~1e-3` or more against the HF reference, the bisection workflow lives at [Debugging Numerical Divergence](debugging_numerical_divergence.md). diff --git a/scripts/phase4_review.py b/scripts/phase4_review.py new file mode 100644 index 0000000000..3b6e98fbc9 --- /dev/null +++ b/scripts/phase4_review.py @@ -0,0 +1,72 @@ +"""Registry-wide Phase-4 review: which verified models' stored scores predate +the profile rework and deserve a re-run. + +phase4_score is a mixed-scale column: entries stamped p4_scoring_version=2 +were measured with the pinned-judge reference-ratio scoring (pass line 56); +unstamped entries carry the old GPT-2 absolute-perplexity scale (pass line 85) +and are never compared against the new line — they are re-run candidates. +Read-only. +""" + +import argparse + +from transformer_lens.benchmarks.text_quality_profiles import ( + P4_SCORING_VERSION, + resolve_profile, +) +from transformer_lens.tools.model_registry.registry_io import load_supported_models_raw + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--below", type=float, default=None, help="Only scores below this") + parser.add_argument("--limit", type=int, default=None, help="Max rows per section") + args = parser.parse_args() + + current: list = [] + stale: list = [] + for entry in load_supported_models_raw().get("models", []): + if entry.get("status") != 1 or entry.get("phase4_score") is None: + continue + score = entry["phase4_score"] + if args.below is not None and score >= args.below: + continue + profile = str( + resolve_profile( + entry["model_id"], entry.get("architecture_id"), entry.get("prompt_profile") + ) + ) + row = (profile != "continuation", score, entry["model_id"], profile) + if entry.get("p4_scoring_version") == P4_SCORING_VERSION: + current.append(row) + else: + stale.append(row) + + # Profile-changed first, then ascending score: measurement changed most. + for rows in (current, stale): + rows.sort(key=lambda r: (not r[0], r[1])) + if args.limit: + current = current[: args.limit] + stale = stale[: args.limit] + + print( + f"{len(current)} scored on the current scale (v{P4_SCORING_VERSION}); " + f"{len(stale)} on the old GPT-2 scale (re-run candidates)\n" + ) + for title, rows in ( + (f"v{P4_SCORING_VERSION} (reference-ratio scale, pass 56)", current), + ("v1 (GPT-2 scale — scores NOT comparable to the new pass line)", stale), + ): + if not rows: + continue + changed = sum(1 for r in rows if r[0]) + print(f"== {title}: {len(rows)} models, {changed} non-default profiles") + print(f"{'score':>6} {'profile':<28} model") + for is_changed, score, model_id, profile in rows: + marker = "*" if is_changed else " " + print(f"{score:6.1f}{marker} {profile:<28} {model_id}") + print() + + +if __name__ == "__main__": + main() diff --git a/scripts/text_quality_judge_bakeoff.py b/scripts/text_quality_judge_bakeoff.py new file mode 100644 index 0000000000..95aed50529 --- /dev/null +++ b/scripts/text_quality_judge_bakeoff.py @@ -0,0 +1,464 @@ +"""Bake off perplexity judges for the reworked Phase-4 text-quality benchmark. + +Scores two small causal LMs as candidate PPL judges: for each of 9 languages +(8 natural + "code"), build a fluent corpus from ``text_quality_profiles`` and +six deterministic corruptions per fluent string (shuffle/repeat/charnoise x2/ +crosslang x2). The judge that best separates fluent from corrupted text by +ROC AUC, worst-language-first, wins; its R_FAIL/R_GOOD thresholds and +per-reference PPLs are then emitted for the real benchmark to consume. + +Run: uv run python scripts/text_quality_judge_bakeoff.py + uv run python scripts/text_quality_judge_bakeoff.py --languages en,fr --models gpt2 +""" +from __future__ import annotations + +import argparse +import gc +import json +import math +import random +import statistics +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import numpy as np +import torch +from huggingface_hub import HfApi +from transformers import AutoModelForCausalLM, AutoTokenizer + +from transformer_lens.benchmarks.text_quality_profiles import ( + CHAT_PROMPTS, + CONTINUATION_PROMPTS, + PIVOT_SENTENCES, +) + +# All PIVOT_SENTENCES languages plus code, so no scored language is left +# uncalibrated (it/nl/pt/hi were absent from the original judge selection run). +DEFAULT_LANGUAGES = [ + "en", + "fr", + "es", + "de", + "it", + "nl", + "pt", + "zh", + "ar", + "ru", + "ja", + "hi", + "code", +] +DEFAULT_MODELS = ["bigscience/bloom-560m", "Qwen/Qwen2.5-0.5B"] + +NO_SPACE_LANGS = {"zh", "ja"} +CHARNOISE_RATES = (0.15, 0.4) +CROSSLANG_RATES = (0.3, 0.7) + +OUT_JSON = Path("judge_reference_ppls.json") # cwd; override with --out + +# --------------------------------------------------------------------------- +# Fluent corpus +# --------------------------------------------------------------------------- + + +def build_fluent_corpus(languages: list[str]) -> dict[str, list[str]]: + """Fluent strings per language: pivot sentences + continuation/chat references.""" + corpus: dict[str, list[str]] = {} + for lang in languages: + if lang == "code": + corpus[lang] = [p.reference for p in CONTINUATION_PROMPTS["code"]] + continue + texts = list(PIVOT_SENTENCES.get(lang, ())) + texts += [p.reference for p in CONTINUATION_PROMPTS.get(lang, ())] + texts += [p.reference for p in CHAT_PROMPTS.get(lang, ())] + corpus[lang] = texts + return corpus + + +# --------------------------------------------------------------------------- +# Corruptions (deterministic given a shared random.Random) +# --------------------------------------------------------------------------- + + +def tokenize_units(text: str, lang: str) -> list[str]: + """Words for space-delimited languages, characters for zh/ja.""" + return list(text) if lang in NO_SPACE_LANGS else text.split() + + +def join_units(units: list[str], lang: str) -> str: + return "".join(units) if lang in NO_SPACE_LANGS else " ".join(units) + + +def corrupt_shuffle(text: str, lang: str, rng: random.Random) -> str: + """Permute word (or char) order.""" + units = tokenize_units(text, lang) + rng.shuffle(units) + return join_units(units, lang) + + +def corrupt_repeat(text: str, lang: str) -> str: + """Repeat the first 3 words (5 chars for zh/ja) until the original length.""" + units = tokenize_units(text, lang) + if not units: + return text + n = 5 if lang in NO_SPACE_LANGS else 3 + seed = units[:n] or units + out = [seed[i % len(seed)] for i in range(len(units))] + return join_units(out, lang) + + +def corrupt_charnoise(text: str, rate: float, rng: random.Random) -> str: + """Swap-adjacent-or-delete at `rate` of character positions.""" + chars = list(text) + out: list[str] = [] + i = 0 + while i < len(chars): + if rng.random() < rate: + if i + 1 < len(chars) and rng.random() < 0.5: + out.append(chars[i + 1]) + out.append(chars[i]) + i += 2 + continue + i += 1 # delete + continue + out.append(chars[i]) + i += 1 + return "".join(out) + + +def corrupt_crosslang( + text: str, lang: str, rate: float, other_units: list[str], rng: random.Random +) -> str: + """Replace `rate` of tokens with tokens drawn from another language's fluent text.""" + units = tokenize_units(text, lang) + if not units or not other_units: + return text + n_replace = min(len(units), max(1, round(rate * len(units)))) + idxs = rng.sample(range(len(units)), n_replace) + out = units[:] + for idx in idxs: + out[idx] = rng.choice(other_units) + return join_units(out, lang) + + +@dataclass +class CorruptedSample: + """One corrupted variant of a fluent source string.""" + + lang: str + source_text: str + kind: str + severity: Optional[float] + text: str + + +def build_corruptions( + languages: list[str], fluent_corpus: dict[str, list[str]], seed: int = 42 +) -> dict[str, list[CorruptedSample]]: + """6 corrupted variants per fluent string, generated once and shared by both candidates.""" + rng = random.Random(seed) + other_pool: dict[str, list[str]] = {} + for i, lang in enumerate(languages): + next_lang = languages[(i + 1) % len(languages)] + other_text = " ".join(fluent_corpus.get(next_lang, [])) + other_pool[lang] = tokenize_units(other_text, lang) + + by_lang: dict[str, list[CorruptedSample]] = {lang: [] for lang in languages} + for lang in languages: + for text in fluent_corpus[lang]: + samples = by_lang[lang] + samples.append( + CorruptedSample(lang, text, "shuffle", None, corrupt_shuffle(text, lang, rng)) + ) + samples.append(CorruptedSample(lang, text, "repeat", None, corrupt_repeat(text, lang))) + for rate in CHARNOISE_RATES: + samples.append( + CorruptedSample( + lang, text, "charnoise", rate, corrupt_charnoise(text, rate, rng) + ) + ) + for rate in CROSSLANG_RATES: + samples.append( + CorruptedSample( + lang, + text, + "crosslang", + rate, + corrupt_crosslang(text, lang, rate, other_pool[lang], rng), + ) + ) + return by_lang + + +# --------------------------------------------------------------------------- +# PPL scoring +# --------------------------------------------------------------------------- + + +def _with_retry(fn, *args, **kwargs): # type: ignore[no-untyped-def] + """One retry after 60s on a 429/rate-limit error.""" + try: + return fn(*args, **kwargs) + except Exception as exc: # noqa: BLE001 + msg = str(exc) + if "429" in msg or "rate limit" in msg.lower(): + print(f"429 hit, retrying in 60s: {msg}", file=sys.stderr) + time.sleep(60) + return fn(*args, **kwargs) + raise + + +def score_text(text: str, tokenizer, model) -> Optional[dict]: # type: ignore[no-untyped-def] + """NLL-based PPL for one string; None if tokenization has <2 tokens.""" + enc = tokenizer(text, return_tensors="pt") + ids = enc["input_ids"] + n_tokens = int(ids.shape[1]) + if n_tokens < 2: + return None + t0 = time.perf_counter() + with torch.no_grad(): + out = model(input_ids=ids, labels=ids) + dt = time.perf_counter() - t0 + ppl = math.exp(out.loss.item()) + unk_id = tokenizer.unk_token_id + unk_count = int((ids == unk_id).sum().item()) if unk_id is not None else None + return {"ppl": ppl, "n_tokens": n_tokens, "unk_count": unk_count, "dt": dt} + + +# --------------------------------------------------------------------------- +# AUC (hand-rolled, rank-based Mann-Whitney) +# --------------------------------------------------------------------------- + + +def auc_score(neg_scores: list[float], pos_scores: list[float]) -> float: + """P(pos > neg) via rank-sum; positive = corrupted, negative = fluent.""" + n_pos, n_neg = len(pos_scores), len(neg_scores) + if n_pos == 0 or n_neg == 0: + return float("nan") + combined = [(s, 0) for s in neg_scores] + [(s, 1) for s in pos_scores] + combined.sort(key=lambda x: x[0]) + n = len(combined) + ranks = [0.0] * n + i = 0 + while i < n: + j = i + while j < n and combined[j][0] == combined[i][0]: + j += 1 + avg_rank = (i + 1 + j) / 2.0 # 1-indexed, averaged over the tie block + for k in range(i, j): + ranks[k] = avg_rank + i = j + rank_sum_pos = sum(r for r, (_, lbl) in zip(ranks, combined) if lbl == 1) + return (rank_sum_pos - n_pos * (n_pos + 1) / 2) / (n_pos * n_neg) + + +# --------------------------------------------------------------------------- +# Per-candidate run +# --------------------------------------------------------------------------- + + +def run_candidate( + model_id: str, + languages: list[str], + fluent_corpus: dict[str, list[str]], + corrupted_by_lang: dict[str, list[CorruptedSample]], +) -> dict: + print(f"Loading {model_id} ...") + tokenizer = _with_retry(AutoTokenizer.from_pretrained, model_id) + model = _with_retry(AutoModelForCausalLM.from_pretrained, model_id, dtype=torch.float32) + model.eval() + + fluent_ppl: dict[str, dict[str, float]] = {} + lang_stats: dict[str, dict] = {} + total_dt = 0.0 + total_forwards = 0 + + for lang in languages: + fluent_ppl[lang] = {} + ln_fluent: list[float] = [] + has_unk = tokenizer.unk_token_id is not None + unk_count = 0 + unk_total = 0 + + for text in fluent_corpus[lang]: + res = score_text(text, tokenizer, model) + if res is None: + continue + total_forwards += 1 + total_dt += res["dt"] + fluent_ppl[lang][text] = res["ppl"] + ln_fluent.append(math.log(res["ppl"])) + if has_unk: + unk_count += res["unk_count"] + unk_total += res["n_tokens"] + + ln_corrupt: list[float] = [] + ln_ratios: list[float] = [] + ratio_raw: list[float] = [] + for sample in corrupted_by_lang[lang]: + if sample.source_text not in fluent_ppl[lang]: + continue # source string itself was skipped (too short) + res = score_text(sample.text, tokenizer, model) + if res is None: + continue + total_forwards += 1 + total_dt += res["dt"] + ppl_src = fluent_ppl[lang][sample.source_text] + ln_corrupt.append(math.log(res["ppl"])) + ln_ratios.append(math.log(res["ppl"]) - math.log(ppl_src)) + ratio_raw.append(res["ppl"] / ppl_src) + if has_unk: + unk_count += res["unk_count"] + unk_total += res["n_tokens"] + + lang_stats[lang] = { + "auc": auc_score(ln_fluent, ln_corrupt), + "median_ln_ratio": float(np.median(ln_ratios)) if ln_ratios else float("nan"), + "unk_rate": (unk_count / unk_total) if has_unk and unk_total else None, + "mean_fluent_ppl": ( + float(np.mean(list(fluent_ppl[lang].values()))) + if fluent_ppl[lang] + else float("nan") + ), + "ratio_raw": ratio_raw, + } + + avg_forward_time = total_dt / total_forwards if total_forwards else float("nan") + del model + gc.collect() + return { + "fluent_ppl": fluent_ppl, + "lang_stats": lang_stats, + "avg_forward_time": avg_forward_time, + } + + +# --------------------------------------------------------------------------- +# Winner selection + constants +# --------------------------------------------------------------------------- + + +def pick_winner(results: dict[str, dict], languages: list[str]) -> tuple[str, str]: + def min_auc(name: str) -> float: + return min(results[name]["lang_stats"][l]["auc"] for l in languages) + + def spread(name: str) -> float: + vals = [results[name]["lang_stats"][l]["median_ln_ratio"] for l in languages] + return max(vals) - min(vals) + + def speed(name: str) -> float: + return results[name]["avg_forward_time"] + + names = sorted(results, key=min_auc, reverse=True) + best = min_auc(names[0]) + tied = [n for n in names if best - min_auc(n) <= 0.01] + if len(tied) == 1: + return tied[0], "highest minimum per-language AUC" + + best_spread = min(spread(n) for n in tied) + tied2 = [n for n in tied if spread(n) == best_spread] + if len(tied2) == 1: + return tied2[0], "min-AUC tie -> smaller cross-language ln-ratio spread" + + winner = min(tied2, key=speed) + return winner, "min-AUC tie -> spread tie -> faster wall-clock per forward" + + +def compute_r_fail(winner_stats: dict, languages: list[str]) -> float: + """Geo-mean over languages of the MEDIAN corrupted/fluent ratio. + + A low percentile degenerates below 1 in weak-separation languages (some + corruptions do not raise perplexity there), which would invert the log + mapping; the median is the robust "typical broken output" anchor. This is + the exact derivation of the shipped JUDGE_R_FAIL.""" + per_lang = [] + for lang in languages: + raw = winner_stats["lang_stats"][lang]["ratio_raw"] + if raw: + per_lang.append(float(np.median(raw))) + return statistics.geometric_mean(per_lang) + + +def compute_r_good(winner_stats: dict, languages: list[str]) -> float: + per_lang = [] + for lang in languages: + vals = list(winner_stats["fluent_ppl"][lang].values()) + ratios = [vals[i] / vals[j] for i in range(len(vals)) for j in range(len(vals)) if i != j] + if ratios: + per_lang.append(float(np.percentile(ratios, 90))) + return statistics.geometric_mean(per_lang) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--languages", default=",".join(DEFAULT_LANGUAGES)) + p.add_argument("--models", default=",".join(DEFAULT_MODELS)) + p.add_argument("--out", type=Path, default=OUT_JSON, help="Where to write reference PPLs") + return p.parse_args() + + +def main() -> None: + args = parse_args() + languages = [l.strip() for l in args.languages.split(",") if l.strip()] + models = [m.strip() for m in args.models.split(",") if m.strip()] + + api = HfApi() + for model_id in models: + info = _with_retry(api.model_info, model_id) + print(f"{model_id} revision sha: {info.sha}") + + fluent_corpus = build_fluent_corpus(languages) + corrupted_by_lang = build_corruptions(languages, fluent_corpus) + + results: dict[str, dict] = {} + for model_id in models: + results[model_id] = run_candidate(model_id, languages, fluent_corpus, corrupted_by_lang) + + header = f"{'candidate':28s} {'lang':6s} {'auc':>7s} {'med_ln_ratio':>13s} {'unk_rate':>9s} {'mean_ppl':>10s}" + print("\n" + header) + print("-" * len(header)) + for model_id in models: + for lang in languages: + st = results[model_id]["lang_stats"][lang] + unk_str = f"{st['unk_rate']:.4f}" if st["unk_rate"] is not None else "n/a" + print( + f"{model_id:28s} {lang:6s} {st['auc']:7.3f} {st['median_ln_ratio']:13.3f} " + f"{unk_str:>9s} {st['mean_fluent_ppl']:10.2f}" + ) + + winner, reason = pick_winner(results, languages) + print(f"\nwinner: {winner} ({reason})") + for model_id in models: + print( + f" {model_id}: avg forward wall-clock {results[model_id]['avg_forward_time']*1000:.2f} ms" + ) + + r_fail = compute_r_fail(results[winner], languages) + r_good = compute_r_good(results[winner], languages) + print(f"JUDGE_R_FAIL = {r_fail:.1f}") + print(f"JUDGE_R_GOOD = {r_good:.2f}") + print(f"pass line score(R_GOOD) = {100 - 100 * math.log(r_good) / math.log(r_fail):.1f}") + + flagged = [ + lang + for lang in languages + if all(results[m]["lang_stats"][lang]["auc"] < 0.8 for m in models) + ] + print(f"languages with AUC < 0.8 for BOTH candidates: {flagged or 'none'}") + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(results[winner]["fluent_ppl"], ensure_ascii=False, indent=2)) + print(f"\nreference PPLs written to {args.out}") + + +if __name__ == "__main__": + main() diff --git a/tests/integration/benchmarks/__init__.py b/tests/integration/benchmarks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/benchmarks/test_text_quality_profiles.py b/tests/integration/benchmarks/test_text_quality_profiles.py new file mode 100644 index 0000000000..1e1f96fa64 --- /dev/null +++ b/tests/integration/benchmarks/test_text_quality_profiles.py @@ -0,0 +1,171 @@ +"""End-to-end Phase-4 profile scoring against real models and the real judge. + +Each test guards a profile path that unit stubs cannot: the Marian test keeps +seq2seq whole-output scoring (pre-profile P4 masked a prompt "continuation" +that seq2seq output does not have and scored 0); the Florence-2 test keeps the +caption path (text-only prompts yield a bare EOS on image-conditioned models); +the judge tests pin the revision and the fluent-vs-corrupted separation the +bake-off measured. +""" + +import pytest + +pytest.importorskip("transformers") + +from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + +def _boot(model_id, **kwargs): + from transformer_lens.model_bridge import TransformerBridge + + try: + return TransformerBridge.boot_transformers(model_id, device="cpu", **kwargs) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"{model_id} unavailable offline: {exc}") + + +@pytest.fixture(scope="module") +def judge(): + from transformer_lens.benchmarks.text_quality import load_judge + + try: + return load_judge() + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"judge unavailable offline: {exc}") + + +def test_translation_profile_marian(judge): + """Seq2seq output is standalone (a translation), not a continuation of the + prompt; the translation profile must score it whole, against the pivot + reference, in the direction parsed from the model id.""" + bridge = _boot("Helsinki-NLP/opus-mt-nl-en") + assert bridge.original_model.config.is_encoder_decoder # precondition + + judge_model, judge_tokenizer = judge + result = benchmark_text_quality( + bridge, + "task:translation@nl-en", + judge_model=judge_model, + judge_tokenizer=judge_tokenizer, + ) + assert result.details is not None, result.message + assert result.details["prompt_profile"] == "task:translation@nl-en" + # A working translator of 3 short pivot sentences must land well above the + # broken floor (score 0 = judge's typical-corruption perplexity ratio). + assert result.details["score"] > 50.0, result.details + + +def test_caption_profile_florence2(judge): + """Florence-2 emits a bare EOS for text-only prompts; P4 must drive real + image-conditioned captions and score them under the caption profile.""" + pytest.importorskip("PIL") + bridge = _boot("florence-community/Florence-2-base", trust_remote_code=True) + + judge_model, judge_tokenizer = judge + result = benchmark_text_quality( + bridge, + "continuation", # deliberately wrong: the caption adjustment must win + judge_model=judge_model, + judge_tokenizer=judge_tokenizer, + ) + assert result.details is not None, result.message + assert result.details["prompt_profile"] == "caption" + assert result.details["score"] > 0.0 + + +def test_chat_profile_templates_and_scores(judge): + """Chat models are scored through their own template (prepend_bos=False — + the template supplies BOS); output must not be the template markers.""" + bridge = _boot("Qwen/Qwen2.5-0.5B-Instruct") + + judge_model, judge_tokenizer = judge + result = benchmark_text_quality( + bridge, + "chat", + judge_model=judge_model, + judge_tokenizer=judge_tokenizer, + ) + assert result.details is not None, result.message + assert result.details["prompt_profile"] == "chat" + assert "<|im_start|>" not in result.details["generated_text"] + assert result.details["score"] > 50.0, result.details + + +def test_fluent_vs_shuffled_separation_end_to_end(judge): + """The full scoring chain must separate real model output from word salad: + any break (mask slip, ratio inversion, penalty loss) collapses the gap.""" + import random + + from transformer_lens.benchmarks.text_quality import ( + _compute_repetition_penalty, + _judge_perplexity, + _ratio_to_score, + ) + from transformer_lens.benchmarks.text_quality_profiles import CONTINUATION_PROMPTS + + judge_model, judge_tokenizer = judge + entry = CONTINUATION_PROMPTS["en"][0] + # A DISTINCT on-topic fluent paraphrase, not the reference itself: ref/ref + # is identically 1 -> 100 and would pass with the judge deleted. + # Measured: ppl 8.3 vs ref 5.1 -> score 83.2 (judge conditioned on the + # relativity prompt correctly rejects off-topic fluent text). + fluent = ( + " measurements of time and distance depend on the observer's motion," + " so no single frame of reference is absolute." + ) + words = entry.reference.split() + random.Random(42).shuffle(words) + shuffled = " ".join(words) + + ref_ppl, err = _judge_perplexity(entry.reference, entry.prompt, judge_tokenizer, judge_model) + assert err is None + fluent_ppl, err = _judge_perplexity(fluent, entry.prompt, judge_tokenizer, judge_model) + assert err is None + fluent_score = _ratio_to_score(fluent_ppl / ref_ppl) * _compute_repetition_penalty(fluent) + shuf_ppl, err = _judge_perplexity(shuffled, entry.prompt, judge_tokenizer, judge_model) + assert err is None + shuffled_score = _ratio_to_score(shuf_ppl / ref_ppl) * _compute_repetition_penalty(shuffled) + + assert fluent_score >= 60.0, fluent_score + assert shuffled_score < 50.0, (shuf_ppl, ref_ppl) + assert fluent_score - shuffled_score >= 30.0 + + +def test_reference_perplexities_match_pinned_values(judge): + """Judge-revision/reference drift guard: the judge's perplexity on a few + fixed reference strings must match values measured at bake-off time + (2026-08-20, Qwen2.5-0.5B@060db649, fp32 CPU). A judge unpin or a silent + reference edit moves these.""" + judge_model, judge_tokenizer = judge + from transformer_lens.benchmarks.text_quality import _judge_perplexity + from transformer_lens.benchmarks.text_quality_profiles import PIVOT_SENTENCES + + pinned = { + ("en", 0): 30.79, + ("fr", 0): 81.85, + ("zh", 0): 53.15, + } + for (lang, idx), expected in pinned.items(): + ppl, err = _judge_perplexity(PIVOT_SENTENCES[lang][idx], "", judge_tokenizer, judge_model) + assert err is None + assert ppl == pytest.approx(expected, rel=0.15), (lang, idx, ppl) + + +def test_continuation_references_share_a_scale(judge): + """Per-language reference PPLs must sit within 3.5x of the language + median: an outlier reference makes its prompt's bar proportionally looser + (the old en[3] measured 31.3 vs median 8.9 and handed gpt2 a clamp-100 on + output worse than its 62-scoring sibling prompt).""" + judge_model, judge_tokenizer = judge + from transformer_lens.benchmarks.text_quality import _judge_perplexity + from transformer_lens.benchmarks.text_quality_profiles import CONTINUATION_PROMPTS + + for lang, prompts in CONTINUATION_PROMPTS.items(): + ppls = [] + for pp in prompts: + ppl, err = _judge_perplexity(pp.reference, pp.prompt, judge_tokenizer, judge_model) + assert err is None, (lang, err) + ppls.append(ppl) + median = sorted(ppls)[len(ppls) // 2] + for i, ppl in enumerate(ppls): + assert ppl <= 3.5 * median, (lang, i, round(ppl, 1), round(median, 1)) diff --git a/tests/integration/model_bridge/test_encdec_string_generation.py b/tests/integration/model_bridge/test_encdec_string_generation.py new file mode 100644 index 0000000000..6fa8df4bc5 --- /dev/null +++ b/tests/integration/model_bridge/test_encdec_string_generation.py @@ -0,0 +1,202 @@ +"""bridge.generate(str) on encoder-decoder models must tokenize with the +tokenizer's native recipe. to_tokens' decoder-style BOS policy injected a +stray and dropped the trailing , corrupting encoder input — m2m100 +degenerated into token loops; Marian/T5 degraded silently. + +A tiny-random M2M100 is used because its lang-code recipe genuinely differs +from to_tokens output (Marian's happens to coincide, so it cannot +discriminate); random weights are fine — greedy decoding is deterministic, so +outputs match iff the encoder input matches. +""" + +import pytest +import torch + +pytest.importorskip("transformers") + + +def test_m2m100_string_generation_matches_native_recipe(): + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + text = "Ik moet nu echt gaan slapen." + native_ids = bridge.tokenizer(text, return_tensors="pt")["input_ids"] + to_tokens_ids = bridge.to_tokens(text) + assert ( + native_ids[0].tolist() != to_tokens_ids[0].tolist() + ), "precondition: recipes must differ or this test cannot discriminate" + # Assert on the tokens generate() actually consumed (random tiny weights + # emit input-independent output, so generated text cannot discriminate). + _, fed = bridge.generate( + text, max_new_tokens=4, temperature=0.0, return_type="tokens", return_input_tokens=True + ) + assert isinstance(fed, torch.Tensor) + assert fed[0].tolist() == native_ids[0].tolist(), (fed[0].tolist(), native_ids[0].tolist()) + + +def test_m2m100_batched_list_generation_matches_native_recipe(): + """The list-input branch had the same corruption (unpatched in the first + fix): batched generate on M2M100/MBart fed to_tokens-mangled encoder + input. Both rows must match the tokenizer's own padded batch encoding.""" + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + texts = ["Ik moet nu echt gaan slapen.", "Ik kan niet zo leven."] + native = bridge.tokenizer(texts, return_tensors="pt", padding=True)["input_ids"] + _, fed = bridge.generate( + texts, max_new_tokens=4, temperature=0.0, return_type="tokens", return_input_tokens=True + ) + assert isinstance(fed, torch.Tensor) + assert fed.tolist() == native.tolist(), (fed.tolist(), native.tolist()) + + +def test_generation_config_forced_bos_applied_by_default(): + """HF's generate() applies generation_config defaults; bart-large-cnn pins + forced_bos_token_id=0 there and its summaries degrade without it. The + bridge must honor the config value when the caller passes none.""" + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + forced = 7 + bridge.original_model.generation_config.forced_bos_token_id = forced + out = bridge.generate( + "Ik moet nu echt gaan slapen.", max_new_tokens=4, temperature=0.0, return_type="tokens" + ) + assert out[0, 1].item() == forced + + +def test_generation_config_min_length_suppresses_early_eos(): + """bart-large-cnn pins min_length=56 in its generation config; HF's + generate() suppresses EOS until then. Without it the bridge loop can EOS + on step one and emit an empty summary (observed live, scored 0).""" + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + from unittest import mock + + from transformer_lens import utilities as tl_utils + + eos = bridge.original_model.config.eos_token_id + # Sample EOS whenever its logit is finite: the loop's -inf suppression is + # then the ONLY thing that can delay it, so this discriminates exactly + # that mechanism (tiny-random weights never prefer EOS on their own). + real_sample = tl_utils.sample_logits + + def eos_greedy(logits, **kwargs): + out = real_sample(logits, **kwargs) + finite = torch.isfinite(logits[:, eos]) + out[finite] = eos + return out + + bridge.original_model.generation_config.min_length = 10 + with mock.patch.object(tl_utils, "sample_logits", eos_greedy): + out = bridge.generate( + "Ik moet nu echt gaan slapen.", + max_new_tokens=16, + temperature=0.0, + return_type="tokens", + stop_at_eos=True, + ) + decoder_part = out[0, 1:].tolist() + # Without suppression EOS lands at decoder position 1; with it, no EOS + # before the floor and EOS immediately after it lifts. + assert not any(t == eos for t in decoder_part[:8]), decoder_part + assert eos in decoder_part, decoder_part + + +def test_generation_config_no_repeat_ngram_applied(): + """bart-large-cnn pins no_repeat_ngram_size=3; HF applies it by default. + Without it greedy decoding falls into a BOS attractor (observed live: + empty summary, scored 0). Force an attractor token and assert the + processor breaks the loop.""" + from unittest import mock + + from transformer_lens import utilities as tl_utils + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + attractor = 5 + real_sample = tl_utils.sample_logits + + def prefer_attractor(logits, **kwargs): + out = real_sample(logits, **kwargs) + allowed = torch.isfinite(logits[:, attractor]) + out[allowed] = attractor + return out + + bridge.original_model.generation_config.no_repeat_ngram_size = 2 + with mock.patch.object(tl_utils, "sample_logits", prefer_attractor): + out = bridge.generate( + "Ik moet nu echt gaan slapen.", + max_new_tokens=8, + temperature=0.0, + return_type="tokens", + stop_at_eos=False, + ) + seq = out[0].tolist() + runs = [seq[i] == seq[i + 1] == attractor for i in range(len(seq) - 1)] + # A (5,5) bigram may occur once, but 5,5,5 requires repeating it — banned. + assert not any( + seq[i] == seq[i + 1] == seq[i + 2] == attractor for i in range(len(seq) - 2) + ), seq + + +def test_batched_unequal_rows_match_solo_generation(): + """Id equality can't see mask handling: the batched enc-dec path fed + native ids but no attention mask, so the short row of an unequal batch + attended over pads. Greedy decoding of the short prompt must be identical + batched and solo.""" + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + short = "Ik slaap." + long = "Ik moet nu echt heel snel gaan slapen want het is al veel te laat geworden." + # Logits-level: argmax can survive unmasked pads on a tiny model, the + # step-0 distribution cannot. + solo = bridge.generate( + short, max_new_tokens=2, temperature=0.0, return_type="tokens", output_logits=True + ) + batched = bridge.generate( + [short, long], max_new_tokens=2, temperature=0.0, return_type="tokens", output_logits=True + ) + solo_step0 = solo.logits[0][0] + batched_step0_row0 = batched.logits[0][0] + assert torch.allclose(solo_step0, batched_step0_row0, atol=1e-4), float( + (solo_step0 - batched_step0_row0).abs().max() + ) diff --git a/tests/unit/benchmarks/test_text_quality_image_conditioned.py b/tests/unit/benchmarks/test_text_quality_image_conditioned.py index b6b0be4522..25938ed8b6 100644 --- a/tests/unit/benchmarks/test_text_quality_image_conditioned.py +++ b/tests/unit/benchmarks/test_text_quality_image_conditioned.py @@ -1,11 +1,6 @@ -"""Image-conditioned seq2seq (Florence-2) must score P4 from real captions. - -Florence-2 needs pixel_values to generate: given a text-only prompt its decoder -emits a 1-token EOS, so every continuation is "too short" and P4 scored 0 (a -misleading failure for a working model). The fix drives real image-conditioned -caption generation ( on synthetic test images) and scores that -grammatical output instead — a genuine quality signal, not a skip. -""" +"""Caption test images must be real, distinct RGB inputs — averaging caption +scores over identical images would be a fake sample size. (The end-to-end +Florence-2 caption test lives in tests/integration/benchmarks/.)""" import pytest @@ -21,29 +16,3 @@ def test_build_caption_test_images_are_distinct_rgb(): assert all(im.mode == "RGB" and im.size == (224, 224) for im in images) # Distinct backgrounds -> distinct pixel data (averaging over samples is real). assert len({im.tobytes() for im in images}) == 3 - - -def test_florence2_text_quality_scores_image_captions(): - from transformer_lens.benchmarks.text_quality import benchmark_text_quality - from transformer_lens.model_bridge import TransformerBridge - - try: - bridge = TransformerBridge.boot_transformers( - "florence-community/Florence-2-base-ft", device="cpu" - ) - except (OSError, ConnectionError, TimeoutError) as exc: - pytest.skip(f"florence-2 unavailable offline: {exc}") - - # Preconditions: this is the image-conditioned seq2seq path. - assert bridge.original_model.config.is_encoder_decoder - assert getattr(bridge.cfg, "is_multimodal", False) - - result = benchmark_text_quality( - bridge, "The theory of relativity explains that", max_new_tokens=50, device="cpu" - ) - # Pre-fix: "Scoring failed for all prompts" (score absent -> registry P4=0). - assert result.details is not None, result.message - assert "score" in result.details, result.message - assert result.details["score"] > 0 - # Scored the model's actual captions, not the 4 text-only prompts. - assert result.details["num_prompts"] >= 1 diff --git a/tests/unit/benchmarks/test_text_quality_scoring.py b/tests/unit/benchmarks/test_text_quality_scoring.py new file mode 100644 index 0000000000..21c368a6fa --- /dev/null +++ b/tests/unit/benchmarks/test_text_quality_scoring.py @@ -0,0 +1,712 @@ +"""Reference-ratio Phase-4 scoring: the score must be a judge-handicap-free +comparison against a reference completion, with penalties for loops and +truncation, generated via token-level slicing (string-prefix slicing breaks +under chat templates because generate() strips special tokens on decode).""" + +import math +from types import SimpleNamespace + +import pytest +import torch + +pytest.importorskip("transformers") + +from transformer_lens.benchmarks.text_quality import ( + JUDGE_R_FAIL, + _length_penalty, + _ratio_to_score, + benchmark_text_quality, +) +from transformer_lens.benchmarks.utils import BenchmarkResult, BenchmarkSeverity + + +class FakeVocabTokenizer: + """Whitespace tokenizer with a growable vocab and template-marker specials.""" + + chat_template = None + + def __init__(self): + self._vocab: list[str] = [] + self._special: set[int] = set() + self.mask_token = None + + def _id(self, word: str, special: bool = False) -> int: + if word not in self._vocab: + self._vocab.append(word) + idx = self._vocab.index(word) + if special: + self._special.add(idx) + return idx + + def encode_words(self, text: str, special: bool = False) -> list[int]: + return [self._id(w, special) for w in text.split()] + + def __call__(self, text, return_tensors=None): + # Native recipe used for encoder-decoder inputs. + ids = self.encode_words(text) + if return_tensors == "pt": + return {"input_ids": torch.tensor([ids])} + return {"input_ids": ids} + + def decode(self, ids, skip_special_tokens=True): + ids = ids.tolist() if hasattr(ids, "tolist") else list(ids) + words = [self._vocab[i] for i in ids if not (skip_special_tokens and i in self._special)] + return " ".join(words) + + def apply_chat_template(self, messages, add_generation_prompt=True, tokenize=False): + return f"<|im_start|> {messages[0]['content']} <|im_end|>" + + +class FakeBridge: + """Decoder-only bridge stub: generate() echoes prompt ids + canned continuation.""" + + def __init__( + self, continuation="the quick brown fox jumps over the lazy dog today", chat_template=None + ): + self.tokenizer = FakeVocabTokenizer() + self.tokenizer.chat_template = chat_template + self.adapter = SimpleNamespace(supports_generation=True, native_sampler=None) + self.original_model = SimpleNamespace( + config=SimpleNamespace(is_encoder_decoder=False, architectures=["FakeLM"]) + ) + self.cfg = SimpleNamespace(device="cpu", is_multimodal=False, model_name="fake") + self._continuation = continuation + self.generate_calls: list[dict] = [] + self.to_tokens_calls: list = [] + + def to_tokens(self, text, prepend_bos=None, **kwargs): + self.to_tokens_calls.append(prepend_bos) + special = text.startswith("<|im_start|>") + if special: + # Template markers become special ids that decode drops. + ids = [] + for word in text.split(): + is_marker = word.startswith("<|") + ids.append(self.tokenizer._id(word, special=is_marker)) + return torch.tensor([ids]) + return torch.tensor([self.tokenizer.encode_words(text)]) + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + cont_ids = self.tokenizer.encode_words(self._continuation) + return torch.cat([input, torch.tensor([cont_ids])], dim=1) + + +class FakeJudgeTokenizer: + """Word-level judge tokenizer sharing nothing with the bridge's.""" + + def __init__(self): + self._vocab: list[str] = [] + + def __call__(self, text, return_tensors=None): + ids = [] + for w in text.split(): + if w not in self._vocab: + self._vocab.append(w) + ids.append(self._vocab.index(w)) + if return_tensors == "pt": + return {"input_ids": torch.tensor([ids])} + return {"input_ids": ids} + + +class FakeJudge: + """Judge whose loss is a configurable function of the scored token ids. + + Records (masked_context_words, scored_words) per call so tests can assert + what the judge was conditioned on.""" + + def __init__(self, tokenizer: FakeJudgeTokenizer, loss_fn): + self._tokenizer = tokenizer + self._loss_fn = loss_fn + self.calls: list = [] + + def __call__(self, input_ids, labels=None): + pairs = list(zip(input_ids[0].tolist(), labels[0].tolist())) + scored = [int(t) for t, l in pairs if l != -100] + masked = [int(t) for t, l in pairs if l == -100] + words = " ".join(self._tokenizer._vocab[i] for i in scored) + self.calls.append((" ".join(self._tokenizer._vocab[i] for i in masked), words)) + return SimpleNamespace(loss=torch.tensor(self._loss_fn(words))) + + +def _run(bridge, profile="continuation", loss_fn=lambda text: 1.0, **kwargs): + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, loss_fn) + result = benchmark_text_quality( + bridge, profile, judge_model=judge, judge_tokenizer=judge_tokenizer, **kwargs + ) + bridge.judge_calls = judge.calls + return result + + +class TestRatioMath: + def test_ratio_one_scores_100(self): + assert _ratio_to_score(1.0) == 100.0 + + def test_ratio_r_fail_scores_zero(self): + assert _ratio_to_score(JUDGE_R_FAIL) == pytest.approx(0.0, abs=1e-9) + + def test_ratio_sqrt_r_fail_scores_50(self): + """Registry's phase-4 floor of 50 = geometric midpoint of good and broken.""" + assert _ratio_to_score(math.sqrt(JUDGE_R_FAIL)) == pytest.approx(50.0, abs=1e-9) + + def test_ratio_below_one_clamps_to_100(self): + """Beating the reference is not extra credit (loops get there trivially).""" + assert _ratio_to_score(0.2) == 100.0 + + def test_ratio_is_handicap_invariant(self): + """A judge that is k-times worse at some language multiplies BOTH sides' + perplexity, so the score must not move (the old absolute-perplexity + mapping drops by 10 ln k).""" + gen_ppl, ref_ppl, k = 40.0, 25.0, 10.0 + assert _ratio_to_score(gen_ppl / ref_ppl) == pytest.approx( + _ratio_to_score((k * gen_ppl) / (k * ref_ppl)) + ) + + +class TestLengthPenalty: + def test_neutral_at_reference_length(self): + assert _length_penalty(40, 40) == 1.0 + + def test_neutral_band_half_to_triple(self): + """Neutral in [0.5x, 3x] of reference: terse-but-complete answers are + not punished, and the old 25% floor (which never fired in four + sweeps — a contentless chat stub scored 93.6) is gone.""" + assert _length_penalty(20, 40) == 1.0 + assert _length_penalty(120, 40) == 1.0 + + def test_penalizes_below_half(self): + assert _length_penalty(10, 40) == pytest.approx(0.5) + assert _length_penalty(13, 41) == pytest.approx(13 / 20.5) + + def test_penalizes_overlength(self): + """Rambling output the repetition penalty misses: 6x the reference + pays half.""" + assert _length_penalty(240, 40) == pytest.approx(0.5) + + def test_zero_reference_is_neutral(self): + assert _length_penalty(3, 0) == 1.0 + + +class TestBenchmarkPipeline: + def test_fluent_output_scores_high(self): + # Continuation long enough to sit in the length-penalty neutral band + # for every reference; ratio 1 then clamps every prompt to 100. + fluent = "the quick brown fox jumps over the lazy dog today while the sun sets slowly behind the old hills" + result = _run(FakeBridge(continuation=fluent), loss_fn=lambda t: 1.0) + assert result.details is not None + assert result.details["score"] == 100.0 + + def test_looping_output_scores_low_despite_low_ratio(self): + """A degenerate loop has LOW judge perplexity; only the repetition + penalty catches it under ratio scoring.""" + loop = "the cat sat the cat sat the cat sat the cat sat" + result = _run(FakeBridge(continuation=loop), loss_fn=lambda t: 0.1) + assert result.details is not None + assert result.details["score"] < 50.0 + + def test_one_token_output_scores_zero(self): + """Florence-style bare-EOS output: one token is not scoreable text.""" + result = _run(FakeBridge(continuation="x"), loss_fn=lambda t: 1.0) + assert result.details is not None + assert result.details["score"] == 0.0 + + def test_generated_segment_sliced_by_token_count(self): + """Chat-template prompts are not string prefixes of decoded output + (specials are stripped); prompt words must still be excluded from the + judged text.""" + seen: list[str] = [] + + def record(text): + seen.append(text) + return 1.0 + + bridge = FakeBridge(chat_template="{{messages}}") + result = _run(bridge, profile="chat", loss_fn=record) + assert result.details is not None + gen_texts = seen[0::2] # generated, reference alternate + assert all("<|im_start|>" not in t for t in seen) + for text in gen_texts: + assert text == bridge._continuation + + def test_context_mask_does_not_swallow_first_generated_token(self): + """Tokenizing prompt+text as one string lets the tokenizer merge + across the seam, so the context mask swallows the first generated + token; the pieces must be tokenized separately.""" + seen: list[str] = [] + + def record(text): + seen.append(text) + return 1.0 + + bridge = FakeBridge(continuation="zebra jumps over seven quiet green hills today") + _run(bridge, loss_fn=record) + gen_texts = [t for t in seen if "zebra" in t or "jumps" in t] + assert gen_texts, seen + assert all(t.startswith("zebra") for t in gen_texts), gen_texts + + def test_empty_generation_is_danger(self): + result = _run(FakeBridge(continuation=""), loss_fn=lambda t: 1.0) + assert result.severity == BenchmarkSeverity.DANGER + assert result.passed is False + + def test_uncovered_profile_skips_with_coverage_instruction(self): + """A coverage gap must tell the operator to file an issue, never score.""" + result = _run(FakeBridge(), profile="task:translation@en-sw") + assert result.severity == BenchmarkSeverity.SKIPPED + assert "task:translation@en-sw" in result.message + assert "file a TransformerLens issue" in result.message + + def test_chat_without_template_downgrades_to_continuation(self): + bridge = FakeBridge(chat_template=None) + result = _run(bridge, profile="chat") + assert result.details is not None + assert result.details["prompt_profile"] == "continuation" + assert "no chat template" in result.details["profile_adjustment"] + + def test_per_prompt_seed_independent_of_order(self): + """Each prompt's sample stream restarts at the benchmark seed, so a + prompt's output cannot depend on how much RNG earlier prompts consumed.""" + + class RngBridge(FakeBridge): + def generate(self, input, **kwargs): + # Fixed RNG consumption, then a draw: identical across prompts + # only if every prompt's stream restarts at the benchmark seed. + torch.rand(5) + draw = int(torch.randint(0, 10_000, (1,)).item()) + cont = self.tokenizer.encode_words(f"gen{draw} token one two") + return torch.cat([input, torch.tensor([cont])], dim=1) + + seen: list[str] = [] + + def record(text): + seen.append(text) + return 1.0 + + _run(RngBridge(), loss_fn=record) + gen_texts = [t for t in seen if t.startswith("gen")] + assert len(gen_texts) >= 2 + assert len(set(gen_texts)) == 1, gen_texts + + +class TestProfileDataIntegrity: + def test_every_table_entry_is_scoreable(self): + from transformer_lens.benchmarks import text_quality_profiles as p + from transformer_lens.benchmarks.text_quality import _wrong_language + + tables = [ + p.CONTINUATION_PROMPTS, + p.CHAT_PROMPTS, + p.SUMMARIZATION_PROMPTS, + p.INSTRUCTION_PROMPTS, + p.DENOISE_PROMPTS, + ] + for table in tables: + for lang, entries in table.items(): + for entry in entries: + assert entry.prompt.strip() + assert entry.reference.strip() + # Content, not just shape: a reference that self-flags as + # wrong-language hard-zeros its own sample (a fr reference + # did; the shape checks missed it). + if table in (p.CONTINUATION_PROMPTS, p.CHAT_PROMPTS): + assert not _wrong_language(entry.reference, lang), ( + lang, + entry.reference[:50], + ) + + def test_pivot_sentences_index_aligned(self): + from transformer_lens.benchmarks.text_quality_profiles import PIVOT_SENTENCES + + lengths = {lang: len(rows) for lang, rows in PIVOT_SENTENCES.items()} + assert set(lengths.values()) == {3}, lengths + + def test_all_kinds_have_knobs(self): + from transformer_lens.benchmarks import text_quality_profiles as p + + assert set(p.MAX_NEW_TOKENS_BY_KIND) == set(p.PROFILE_KINDS) + + +class TestTranslationWiring: + def test_forced_bos_threaded_for_multilingual_translators(self): + """M2M100/MBart select target language via the first decoder token; + dropping the forced_bos_token_id kwarg silently translates into an + arbitrary language (and the judge would score that fluent text well).""" + + class M2M100Bridge(FakeBridge): + def __init__(self): + super().__init__(continuation="ich muss jetzt wirklich schlafen gehen heute abend") + self.original_model.config.is_encoder_decoder = True + self.tokenizer.get_lang_id = lambda lang: {"de": 777, "en": 700}.get(lang, 0) + self.tokenizer.src_lang = "en" + + def generate(self, input, **kwargs): + # Real enc-dec output shape: [decoder_start] + generated, never + # the echoed source prompt. + self.generate_calls.append(kwargs) + start = torch.tensor([[0]]) + cont_ids = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont_ids])], dim=1) + + bridge = M2M100Bridge() + result = _run(bridge, profile="task:translation@en-de") + assert result.details is not None, result.message + assert all(call.get("forced_bos_token_id") == 777 for call in bridge.generate_calls) + assert bridge.tokenizer.src_lang == "en" + + +class TestReviewGuards: + """Guards for defects found in adversarial review of the rework.""" + + def test_cjk_repetition_penalty_uses_characters(self): + """Whitespace-split n-grams see zh/ja text as one word and never fire — + exactly where the judge rewards loops with low perplexity.""" + from transformer_lens.benchmarks.text_quality import _compute_repetition_penalty + + assert _compute_repetition_penalty("的" * 20) < 0.2 + assert _compute_repetition_penalty("のの" * 10) < 0.3 + fluent_zh = "长城是中国古代伟大的防御工程，每年吸引大量游客。" + assert _compute_repetition_penalty(fluent_zh) > 0.7 + + def test_wrong_language_output_scores_zero(self): + """Ratio scoring measures fluency, not language: fluent English beats a + short German reference and clamps to 100 unless language is checked.""" + bridge = FakeBridge(continuation="the quick brown fox jumps over the lazy dog and the cat") + seen = [] + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, lambda t: (seen.append(t) or 1.0)) + from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + result = benchmark_text_quality( + bridge, "continuation@de", judge_model=judge, judge_tokenizer=judge_tokenizer + ) + assert result.details is not None + assert result.details["score"] == 0.0 + assert "not in 'de'" in result.details["per_prompt"] + + def test_wrong_language_check_passes_correct_language(self): + bridge = FakeBridge( + continuation="der alte Zug ist nicht mit einem neuen Wagen gefahren und die Leute" + ) + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, lambda t: 1.0) + from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + result = benchmark_text_quality( + bridge, "continuation@de", judge_model=judge, judge_tokenizer=judge_tokenizer + ) + assert result.details is not None + assert result.details["score"] > 0.0 + + def test_empty_output_scored_zero_not_dropped(self): + """An empty generation must drag the average down, not vanish from it.""" + + class HalfEmptyBridge(FakeBridge): + def __init__(self): + super().__init__() + self._call = 0 + + def generate(self, input, **kwargs): + self._call += 1 + if self._call % 2 == 0: + return input # no new tokens -> empty continuation + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([input, torch.tensor([cont])], dim=1) + + result = _run(HalfEmptyBridge(), loss_fn=lambda t: 1.0) + assert result.details is not None + assert result.details["num_prompts"] == 4 + assert 40.0 <= result.details["score"] <= 60.0, result.details + + def test_denoise_t5_fill_spliced_into_sentence(self): + """Bare span fragments have judge PPL in the thousands, making the + ratio vacuous; the fill must be judged inside the restored sentence.""" + seen: list[str] = [] + + class DenoiseBridge(FakeBridge): + def __init__(self): + super().__init__(continuation="played happily") + self.original_model.config.is_encoder_decoder = True + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + start = torch.tensor([[0]]) + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont])], dim=1) + + bridge = DenoiseBridge() + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, lambda t: (seen.append(t) or 1.0)) + from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + result = benchmark_text_quality( + bridge, "task:denoise", judge_model=judge, judge_tokenizer=judge_tokenizer + ) + assert result.details is not None + # The bare fill must never reach the judge; every judged text is a + # full restored sentence. + assert seen and all(len(t.split()) >= 8 for t in seen), seen + assert "The children played happily in the park until the sun went down." in seen + + def test_chat_prepend_bos_false_threaded_to_tokenizer(self): + """The chat template supplies its own BOS; to_tokens must receive + prepend_bos=False or the prompt gets a double BOS.""" + bridge = FakeBridge(chat_template="{{messages}}") + _run(bridge, profile="chat") + assert bridge.to_tokens_calls and all(v is False for v in bridge.to_tokens_calls) + + def test_translation_scored_jointly(self): + """Short pivot sentences have unstable judge PPL; the three samples + must be concatenated into one judged pair.""" + + class MarianBridge(FakeBridge): + def __init__(self): + super().__init__(continuation="ik moet nu echt gaan slapen vandaag") + self.original_model.config.is_encoder_decoder = True + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + start = torch.tensor([[0]]) + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont])], dim=1) + + bridge = MarianBridge() + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, lambda t: 1.0) + from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + result = benchmark_text_quality( + bridge, "task:translation@en-nl", judge_model=judge, judge_tokenizer=judge_tokenizer + ) + assert result.details is not None + assert result.details["num_prompts"] == 1 + assert len(bridge.generate_calls) == 3 # generation stays per-sentence + + def test_task_kinds_generate_greedily(self): + """Users run translators deterministically; sampling variance also + makes a single-sample score unstable. Task kinds must pass + temperature 0.0 while open-ended kinds keep sampling.""" + cont_bridge = FakeBridge() + _run(cont_bridge, profile="continuation") + assert all(c["temperature"] == 0.7 for c in cont_bridge.generate_calls) + + class MarianBridge(FakeBridge): + def __init__(self): + super().__init__(continuation="ik moet nu echt gaan slapen vandaag") + self.original_model.config.is_encoder_decoder = True + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + start = torch.tensor([[0]]) + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont])], dim=1) + + task_bridge = MarianBridge() + _run(task_bridge, profile="task:translation@en-nl") + assert all(c["temperature"] == 0.0 for c in task_bridge.generate_calls) + + def test_encdec_prompt_uses_native_tokenizer_recipe(self): + """Encoder input must follow the tokenizer's own recipe (lang token + + trailing ); to_tokens' BOS policy injects and drops , + which sent m2m100 into a quote-mark loop.""" + BOS, EOS = 901, 902 + + class RecipeTokenizer(FakeVocabTokenizer): + def __call__(self, text, return_tensors=None): + ids = self.encode_words(text) + [EOS] + if return_tensors == "pt": + return {"input_ids": torch.tensor([ids])} + return {"input_ids": ids} + + class RecipeBridge(FakeBridge): + def __init__(self): + super().__init__(continuation="ik moet gaan slapen vandaag echt nu") + self.tokenizer.__class__ = RecipeTokenizer + self.original_model.config.is_encoder_decoder = True + self.seen_inputs: list = [] + + def to_tokens(self, text, prepend_bos=None, **kwargs): + self.to_tokens_calls.append(prepend_bos) + return torch.tensor([[BOS] + self.tokenizer.encode_words(text)]) + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + self.seen_inputs.append(input[0].tolist()) + start = torch.tensor([[0]]) + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont])], dim=1) + + bridge = RecipeBridge() + _run(bridge, profile="task:translation@en-nl") + assert bridge.seen_inputs, "no generation happened" + for ids in bridge.seen_inputs: + assert ids[-1] == EOS, ids + assert BOS not in ids, ids + + def test_dead_encdec_denoise_scores_zero(self): + """An empty span fill must not be spliced into the prompt sentence: + the splice hands a dead enc-dec model the near-reference sentence and + a free 100 (decoder-only dead models already scored 0).""" + + class DeadT5Bridge(FakeBridge): + def __init__(self): + super().__init__() + self.original_model.config.is_encoder_decoder = True + self.tokenizer.mask_token = None + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + return torch.tensor([[0]]) + + result = _run(DeadT5Bridge(), profile="task:denoise") + assert result.details["score"] == 0.0 + assert result.severity == BenchmarkSeverity.DANGER + + +class TestRound2ReviewGuards: + """Guards for the second review round's confirmed findings.""" + + def test_curated_strings_never_self_flag(self): + """The wrong-language detector must accept every curated string in its + own language — a reference that self-flags hard-zeros its sample (a + French reference did, via 'de/et' hitting other languages' sets).""" + from transformer_lens.benchmarks.text_quality import _wrong_language + from transformer_lens.benchmarks.text_quality_profiles import ( + CHAT_PROMPTS, + CONTINUATION_PROMPTS, + PIVOT_SENTENCES, + ) + + offenders = [] + for table in (CONTINUATION_PROMPTS, CHAT_PROMPTS): + for lang, prompts in table.items(): + for pp in prompts: + for text in (pp.prompt, pp.reference): + if _wrong_language(text, lang): + offenders.append((lang, text[:50])) + for lang, sents in PIVOT_SENTENCES.items(): + for s in sents: + if _wrong_language(s, lang): + offenders.append((lang, s[:50])) + assert offenders == [] + + def test_cjk_loop_penalized_despite_space(self): + """A single space in a degenerate CJK loop restored the inert word + path (penalty 1.0 vs 0.053); char mode must key off CJK content.""" + from transformer_lens.benchmarks.text_quality import _compute_repetition_penalty + + assert _compute_repetition_penalty("的的的的的的的的的 的的的的的的的的的的") <= 0.3 + assert _compute_repetition_penalty("我该去睡觉了，因为明天有一个很重要的会议要参加。") > 0.5 + + def test_registry_floor_equals_pass_line(self): + """[floor, pass) previously got passed=False with a clean note; both + numbers must come from the same constant.""" + from transformer_lens.benchmarks.text_quality_profiles import p4_pass_threshold + from transformer_lens.tools.model_registry.verify_models import ( + _MIN_PHASE_SCORES, + ) + + assert _MIN_PHASE_SCORES[4] == p4_pass_threshold() + + def test_judge_cannot_self_score(self): + """Ratio scoring against the judge's own perplexity is self-grading.""" + from transformer_lens.benchmarks.text_quality import JUDGE_MODEL_ID + + result = _run(FakeBridge(), profile="continuation", model_name=JUDGE_MODEL_ID) + assert result.severity == BenchmarkSeverity.SKIPPED + assert result.message.startswith("P4 skipped:") + + def test_chat_judged_with_prompt_context(self): + """A fluent off-topic stub scores ~98 when chat output is judged + standalone; conditioning on the user prompt is the relevance signal.""" + from transformer_lens.benchmarks.text_quality_profiles import ( + JUDGE_CONTEXT_KINDS, + ) + + assert "chat" in JUDGE_CONTEXT_KINDS + assert "task:instruction" in JUDGE_CONTEXT_KINDS + # Unconditioned summarization scored hallucinated summaries 100 (the + # judge never saw the article); unconditioned denoise rated broken + # restorations more fluent than the reference. + assert "task:summarization" in JUDGE_CONTEXT_KINDS + assert "task:denoise" in JUDGE_CONTEXT_KINDS + bridge = FakeBridge() + bridge.tokenizer.chat_template = "{{messages}}" + _run(bridge, profile="chat") + contexts = [c for c, _t in bridge.judge_calls] + assert any(c for c in contexts), "judge never saw the user prompt as context" + + def test_p1_only_note_labels_skip_as_coverage_gap(self): + """A skipped P4 is a coverage gap; the stale score must not be + relabeled 'text quality poor'.""" + from transformer_lens.tools.model_registry.verify_models import ( + _p1_only_core_note, + ) + + skipped = BenchmarkResult( + name="text_quality", + severity=BenchmarkSeverity.SKIPPED, + message="P4 skipped: no prompts for profile 'continuation@xx' — file an issue", + ) + skipped.phase = 4 + note = _p1_only_core_note(None, [skipped]) + assert "P4 skipped" in note and "poor" not in note + assert "poor (P4=40.0)" in _p1_only_core_note(40.0, []) + assert "errored" in _p1_only_core_note(None, []) + + def test_arch_rule_keeps_stored_language(self): + """The arch rule fixes the kind; a scraped @fr of the same kind must + survive resolve->writeback or curation can never stick.""" + from transformer_lens.benchmarks.text_quality_profiles import resolve_profile + + spec = resolve_profile( + "some/pegasus-clone", + "PegasusForConditionalGeneration", + registry_profile="task:summarization@fr", + ) + assert str(spec) == "task:summarization@fr" + spec = resolve_profile( + "some/pegasus-clone", + "PegasusForConditionalGeneration", + registry_profile="continuation@fr", + ) + assert str(spec) == "task:summarization" + + +class TestForcedBosVocabCollision: + """Bare ISO codes collide with ordinary subwords (T5's 'de' id 221, + Marian's 'en' id 39) and were injected as forced decoder tokens, + corrupting every translator without a real lang-code system.""" + + def test_plain_vocab_word_is_not_a_lang_code(self): + from transformer_lens.benchmarks.text_quality import _forced_bos_for_target + + class PlainSeq2SeqTokenizer: + unk_token_id = 3 + + def convert_tokens_to_ids(self, tok): + return {"de": 221, "en": 39}.get(tok, 3) + + assert _forced_bos_for_target(PlainSeq2SeqTokenizer(), "de") is None + + def test_nllb_style_code_still_resolves(self): + from transformer_lens.benchmarks.text_quality import _forced_bos_for_target + + class NllbLikeTokenizer: + unk_token_id = 3 + + def convert_tokens_to_ids(self, tok): + return {"deu_Latn": 256042}.get(tok, 3) + + assert _forced_bos_for_target(NllbLikeTokenizer(), "de") == 256042 + + +class TestAllGenerationsCaptured: + def test_details_carry_every_prompts_generation(self): + """Only the first prompt's output was stored; the registry-wide + review needs every generation inspectable.""" + bridge = FakeBridge() + result = _run(bridge, profile="continuation") + texts = result.details["generated_texts"] + assert len(texts) == result.details["num_prompts"] + assert all(isinstance(t, str) and t for t in texts) diff --git a/tests/unit/benchmarks/test_text_quality_seq2seq.py b/tests/unit/benchmarks/test_text_quality_seq2seq.py deleted file mode 100644 index 161fa0de46..0000000000 --- a/tests/unit/benchmarks/test_text_quality_seq2seq.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Encoder-decoder text-quality scoring must score the full decoder output. - -Seq2seq models (Marian/T5/BART) emit a standalone output, not a continuation -of the prompt. Scoring it as a continuation subtracts the prompt length and -trips the "continuation too short (< 2 tokens)" guard for every prompt when the -output is ~ the prompt length (Marian nl->en on an English prompt), scoring 0. -The fix scores the whole generated sequence for encoder-decoder models. -""" - -import pytest - -pytest.importorskip("transformers") - - -def test_marian_text_quality_scores_full_output(): - from transformer_lens.benchmarks.text_quality import benchmark_text_quality - from transformer_lens.model_bridge import TransformerBridge - - try: - bridge = TransformerBridge.boot_transformers("Helsinki-NLP/opus-mt-nl-en", device="cpu") - except (OSError, ConnectionError, TimeoutError) as exc: - pytest.skip(f"marian unavailable offline: {exc}") - - assert bridge.original_model.config.is_encoder_decoder # precondition - - result = benchmark_text_quality( - bridge, "Natural language processing is", max_new_tokens=20, device="cpu" - ) - # Pre-fix this returned "Scoring failed for all prompts" (score absent). - assert result.details is not None, result.message - assert "score" in result.details, result.message - assert result.details["score"] > 0 diff --git a/tests/unit/tools/model_registry/test_clear_hf_cache.py b/tests/unit/tools/model_registry/test_clear_hf_cache.py new file mode 100644 index 0000000000..b9047a736d --- /dev/null +++ b/tests/unit/tools/model_registry/test_clear_hf_cache.py @@ -0,0 +1,30 @@ +"""_clear_hf_cache must never delete the pinned Phase-4 judge: the sweep clears +the HF cache after every model family, and re-downloading the judge each time +defeats the batch preload.""" + +import pytest + +pytest.importorskip("transformers") + + +def test_clear_hf_cache_preserves_judge_snapshot(tmp_path, monkeypatch): + from pathlib import Path + + from transformer_lens.benchmarks.text_quality import JUDGE_MODEL_ID + from transformer_lens.tools.model_registry import verify_models + + hub = tmp_path / ".cache" / "huggingface" / "hub" + judge_dir = hub / ("models--" + JUDGE_MODEL_ID.replace("/", "--")) / "blobs" + other_dir = hub / "models--someone--other-model" / "blobs" + judge_dir.mkdir(parents=True) + other_dir.mkdir(parents=True) + judge_blob = judge_dir / "aaaa" + other_blob = other_dir / "bbbb" + judge_blob.write_bytes(b"judge-weights") + other_blob.write_bytes(b"other-weights") + + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + verify_models._clear_hf_cache(quiet=True) + + assert judge_blob.exists(), "judge blob must survive the per-family cache clear" + assert not other_blob.exists(), "non-judge blobs must still be cleared" diff --git a/tests/unit/tools/model_registry/test_prompt_profiles.py b/tests/unit/tools/model_registry/test_prompt_profiles.py new file mode 100644 index 0000000000..00c4a4dcd7 --- /dev/null +++ b/tests/unit/tools/model_registry/test_prompt_profiles.py @@ -0,0 +1,231 @@ +"""Profile resolution: curation must beat unreliable Hub metadata (observed +mis-tags: mt0-base as text-generation, conversational on base models, +unordered Helsinki-NLP language tags), and gaps must fall through safely.""" + +import pytest + +pytest.importorskip("transformers") + +from transformer_lens.benchmarks.text_quality_profiles import ( + DEFAULT_PROFILE, + HFSignals, + ProfileSpec, + extract_languages, + profile_from_hf_signals, + resolve_profile, +) + + +class TestPrecedence: + def test_override_beats_architecture_rule(self, monkeypatch): + """A real override-vs-arch clash: Pegasus's arch rule says + summarization; a per-model override must still win.""" + from transformer_lens.benchmarks import text_quality_profiles as tp + + monkeypatch.setitem(tp.MODEL_PROFILE_OVERRIDES, "google/pegasus-xsum", "continuation") + spec = resolve_profile("google/pegasus-xsum", "PegasusForConditionalGeneration") + assert spec == ProfileSpec("continuation") + + def test_override_beats_signals(self): + """long-t5 override must win even when signals disagree.""" + spec = resolve_profile( + "google/long-t5-tglobal-base", + "LongT5ForConditionalGeneration", + signals=HFSignals(pipeline_tag="summarization"), + ) + assert spec == ProfileSpec("task:denoise") + + def test_architecture_rule_beats_fetched_tag(self): + """Pegasus is summarization by architecture even if the Hub tag lies.""" + spec = resolve_profile( + "google/pegasus-xsum", + "PegasusForConditionalGeneration", + signals=HFSignals(pipeline_tag="text-generation"), + ) + assert spec.kind == "task:summarization" + + def test_mt0_mistag_resolves_to_instruction(self): + """Hub tags mt0-base text-generation; the override must correct it.""" + spec = resolve_profile( + "bigscience/mt0-base", + "MT5ForConditionalGeneration", + signals=HFSignals(pipeline_tag="text-generation"), + ) + assert spec.kind == "task:instruction" + + def test_fetched_tag_fills_gap(self): + """BART has no arch rule (checkpoint-dependent); the Hub tag decides.""" + spec = resolve_profile( + "facebook/bart-large-cnn", + "BartForConditionalGeneration", + signals=HFSignals(pipeline_tag="summarization", languages=("en",)), + ) + assert spec.kind == "task:summarization" + + def test_null_pipeline_tag_falls_through_to_arch_rule(self): + """m2m100 has pipeline_tag=None on the Hub; the arch rule must hold.""" + spec = resolve_profile( + "facebook/m2m100_418M", + "M2M100ForConditionalGeneration", + signals=HFSignals(pipeline_tag=None), + ) + assert spec.kind == "task:translation" + + def test_stored_registry_value_used_when_no_signals(self): + spec = resolve_profile("some/model", "GPT2LMHeadModel", "continuation@fr") + assert spec == ProfileSpec("continuation", "fr") + + def test_unknown_seq2seq_defaults_to_denoise_not_continuation(self): + """An unlabelled seq2seq cannot continue text; denoising is its only prompt.""" + spec = resolve_profile("someone/random-t5", "T5GemmaForConditionalGeneration") + assert spec.kind == "task:denoise" + + def test_unknown_causal_lm_defaults_to_continuation(self): + assert resolve_profile("someone/random-lm", "LlamaForCausalLM") == DEFAULT_PROFILE + + +class TestHubSignals: + def test_conversational_tag_alone_is_not_chat(self): + """HF adds `conversational` to ANY repo shipping a chat template, base + models included (observed on Qwen/Qwen2.5-0.5B).""" + spec = profile_from_hf_signals( + "Qwen/Qwen2.5-0.5B", + "Qwen2ForCausalLM", + HFSignals(pipeline_tag="text-generation", tags=("conversational",)), + ) + assert spec is not None and spec.kind == "continuation" + + def test_code_tag_maps_to_code_continuation(self): + spec = profile_from_hf_signals( + "bigcode/some-model", "GPTBigCodeForCausalLM", HFSignals(tags=("code",)) + ) + assert spec == ProfileSpec("continuation", "code") + + def test_marian_direction_from_model_id_not_tag_order(self): + """Helsinki-NLP language tags are unordered; only opus-mt-{src}-{tgt} + carries the direction.""" + spec = resolve_profile( + "Helsinki-NLP/opus-mt-nl-en", + "MarianMTModel", + signals=HFSignals(languages=("en", "nl")), # tag order is wrong on purpose + ) + assert (spec.src, spec.lang) == ("nl", "en") + + def test_translation_tag_without_direction_returns_none(self): + """Tag lists are unordered: guessing a pair risks a reversed or + identity direction, so signals alone must abstain (resolution then + falls through to overrides/arch rules — t5-small still lands on + en-de via its override).""" + spec = profile_from_hf_signals( + "google-t5/t5-small", + "T5ForConditionalGeneration", + HFSignals(pipeline_tag="translation"), + ) + assert spec is None + resolved = resolve_profile( + "google-t5/t5-small", + "T5ForConditionalGeneration", + signals=HFSignals(pipeline_tag="translation"), + ) + assert (resolved.src, resolved.lang) == ("en", "de") + + def test_translation_tag_with_en_and_target_infers_pair(self): + spec = profile_from_hf_signals( + "someone/en-fr-translator", + "BartForConditionalGeneration", + HFSignals(pipeline_tag="translation", languages=("en", "fr")), + ) + assert spec is not None and (spec.src, spec.lang) == ("en", "fr") + + +class TestLanguageExtraction: + def test_handles_str_and_list(self): + assert extract_languages("fr", []) == ("fr",) + assert extract_languages(["de", "en"], []) == ("de", "en") + + def test_merges_iso_tags_and_drops_noise(self): + langs = extract_languages( + None, ["pytorch", "transformers", "nl", "marian", "safetensors", "en"] + ) + assert langs == ("nl", "en") + + def test_caps_at_eight(self): + many = ["fr", "es", "de", "it", "nl", "pt", "ru", "ja", "ar", "hi"] + assert len(extract_languages(many, [])) == 8 + + +class TestProfileSpecGrammar: + def test_round_trip(self): + for text in ("continuation", "continuation@code", "chat@fr", "task:translation@en-de"): + assert str(ProfileSpec.parse(text)) == text + + def test_rejects_unknown_kind(self): + with pytest.raises(ValueError): + ProfileSpec.parse("poetry@en") + + def test_rejects_translation_without_pair(self): + with pytest.raises(ValueError): + ProfileSpec.parse("task:translation@de") + + +class TestChatIdHeuristic: + """Instruct/chat/-it ids resolve to the chat profile — nothing else can + (the conversational tag covers base models; no arch distinguishes tuned + from base). Runtime downgrades template-less models back to continuation.""" + + def test_instruct_id_resolves_chat(self): + spec = resolve_profile("Qwen/Qwen2.5-0.5B-Instruct", "Qwen2ForCausalLM") + assert spec.kind == "chat" + + def test_it_suffix_resolves_chat(self): + spec = resolve_profile("google/gemma-2-2b-it", "Gemma2ForCausalLM") + assert spec.kind == "chat" + + def test_base_id_stays_continuation(self): + assert resolve_profile("Qwen/Qwen2.5-0.5B", "Qwen2ForCausalLM").kind == "continuation" + assert resolve_profile("google/gemma-2-2b", "Gemma2ForCausalLM").kind == "continuation" + + def test_override_still_beats_chat_heuristic(self, monkeypatch): + from transformer_lens.benchmarks import text_quality_profiles as tp + + monkeypatch.setitem(tp.MODEL_PROFILE_OVERRIDES, "someone/model-instruct", "continuation@fr") + spec = resolve_profile("someone/model-instruct", "LlamaForCausalLM") + assert spec == ProfileSpec("continuation", "fr") + + def test_arch_rule_beats_chat_heuristic(self): + """A Blenderbot-style arch keeps its rule even with a chatty id.""" + spec = resolve_profile("someone/blenderbot-chat", "BlenderbotForConditionalGeneration") + assert spec.kind == "chat" # via arch rule, not id — same outcome + spec2 = resolve_profile("someone/opus-mt-nl-en-chat", "MarianMTModel") + assert spec2.kind == "task:translation" + + +def test_chat_heuristic_keeps_stored_language(): + """The id heuristic fixes the kind; a stored chat@fr must survive + resolve->writeback (it was flattened to chat@en and clobbered).""" + from transformer_lens.benchmarks.text_quality_profiles import resolve_profile + + spec = resolve_profile("org/model-7b-instruct", "LlamaForCausalLM", registry_profile="chat@fr") + assert str(spec) == "chat@fr" + # A stored non-chat profile does not hijack the heuristic. + spec = resolve_profile( + "org/model-7b-instruct", "LlamaForCausalLM", registry_profile="continuation@fr" + ) + assert str(spec) == "chat" + + +def test_non_english_denoise_is_a_coverage_gap(): + """IndicBART's stale score was measured under a broken MBart profile; + until Indic denoise prompts exist, a non-en denoise profile must SKIP + (coverage gap), never score against English sentences.""" + from transformer_lens.benchmarks.text_quality_profiles import ( + ProfileSpec, + prompts_for, + resolve_profile, + ) + + assert str(resolve_profile("ai4bharat/IndicBART", "MBartForConditionalGeneration")) == ( + "task:denoise@hi" + ) + assert prompts_for(ProfileSpec("task:denoise", lang="hi")) is None + assert prompts_for(ProfileSpec("task:denoise", lang="en")) is not None diff --git a/tests/unit/tools/model_registry/test_update_model_registry.py b/tests/unit/tools/model_registry/test_update_model_registry.py index 7e9d7a3a11..663457c2fe 100644 --- a/tests/unit/tools/model_registry/test_update_model_registry.py +++ b/tests/unit/tools/model_registry/test_update_model_registry.py @@ -146,3 +146,186 @@ def test_failing_scores_write_failed_not_verified(self, registry_paths): assert entry["status"] == STATUS_FAILED assert "Below threshold" in entry["note"] assert data["total_verified"] == 0 + + +class TestPromptProfileWriteback: + """The Phase-4 profile actually used must land in the registry sparsely: + non-default profiles are recorded, the default writes no key at all (the + registry JSON is served to the docs site; 15k default keys are dead weight).""" + + def _p4(self, profile): + return _result( + 4, + True, + name="text_quality", + details={"score": 91.0, "prompt_profile": profile}, + ) + + def test_prompt_profile_written_from_p4_details(self, registry_paths): + supported_path, _ = registry_paths + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("task:translation@en-de")], + use_hf_reference=True, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["prompt_profile"] == "task:translation@en-de" + # Key order: sparse key sits right after note, before phase scores. + keys = list(entry) + assert keys.index("prompt_profile") == keys.index("note") + 1 + + def test_default_profile_not_written(self, registry_paths): + supported_path, _ = registry_paths + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("continuation")], + use_hf_reference=True, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert "prompt_profile" not in entry + + def test_existing_profile_survives_profileless_rerun(self, registry_paths): + """A later run without a P4 result must not clobber the stored profile.""" + supported_path, _ = registry_paths + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("chat@fr")], + use_hf_reference=True, + ) + update_model_registry("seeded/model", [_result(1, True)], use_hf_reference=True) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["prompt_profile"] == "chat@fr" + + def test_default_profile_clears_stale_nondefault(self, registry_paths): + """A model re-resolved to the default must lose its old sparse key — + otherwise a stale 'chat@fr' misdescribes how the score was produced.""" + supported_path, _ = registry_paths + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("chat@fr")], + use_hf_reference=True, + ) + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("continuation")], + use_hf_reference=True, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert "prompt_profile" not in entry + + def test_new_entry_append_carries_profile(self, registry_paths): + """The append (model-not-in-registry) branch must also write the sparse + key, positioned after note.""" + supported_path, _ = registry_paths + update_model_registry( + "unseeded/model", + [_result(1, True), self._p4("task:summarization")], + use_hf_reference=True, + ) + entry, _ = _entry(supported_path, "unseeded/model") + assert entry["prompt_profile"] == "task:summarization" + keys = list(entry) + assert keys.index("prompt_profile") == keys.index("note") + 1 + + +class TestP4ScoringVersionStamp: + """phase4_score is a mixed-scale column (old GPT-2 scale vs pinned-judge + ratio scale); every P4-bearing write must stamp the scale it measured on, + and writes without a P4 result must not touch an existing stamp.""" + + def test_p4_write_stamps_current_version(self, registry_paths): + from transformer_lens.benchmarks.text_quality_profiles import P4_SCORING_VERSION + from transformer_lens.tools.model_registry import registry_io + + supported_path, _ = registry_paths + registry_io.update_model_status( + "seeded/model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0, 4: 91.0}, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["p4_scoring_version"] == P4_SCORING_VERSION + + def test_no_p4_write_preserves_existing_stamp(self, registry_paths): + from transformer_lens.tools.model_registry import registry_io + + supported_path, _ = registry_paths + registry_io.update_model_status( + "seeded/model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0, 4: 91.0}, + ) + registry_io.update_model_status( + "seeded/model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0}, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["p4_scoring_version"] == 2 + assert entry["phase4_score"] == 91.0 + + def test_old_scale_entry_has_no_stamp(self, registry_paths): + supported_path, _ = registry_paths + entry, _ = _entry(supported_path, "seeded/model") + assert "p4_scoring_version" not in entry + + def test_new_entry_with_p4_is_stamped(self, registry_paths): + from transformer_lens.tools.model_registry import registry_io + + supported_path, _ = registry_paths + registry_io.update_model_status( + "brand/new-model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0, 4: 77.0}, + ) + entry, _ = _entry(supported_path, "brand/new-model") + assert entry["p4_scoring_version"] == 2 + + +class TestPreservedIssueSuffix: + """A phases-1-4 pass must not overwrite tracked residue from phases it + did not re-run (gemma-2-2b-it's P3=95.5 unembed_centering note was + clobbered by a bare 'Core verification completed').""" + + def test_sub100_score_from_unrun_phase_is_retained(self, registry_paths): + from transformer_lens.tools.model_registry import registry_io + from transformer_lens.tools.model_registry.verify_models import ( + _preserved_issue_suffix, + ) + + registry_io.update_model_status( + "seeded/model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0, 3: 95.5}, + ) + assert _preserved_issue_suffix("seeded/model", [1, 4]) == ( + " (prior issues retained: P3=95.5%)" + ) + # Re-running the phase drops it from the suffix (the fresh score speaks). + assert _preserved_issue_suffix("seeded/model", [1, 3, 4]) == "" + + def test_clean_entry_has_no_suffix(self, registry_paths): + from transformer_lens.tools.model_registry.verify_models import ( + _preserved_issue_suffix, + ) + + assert _preserved_issue_suffix("seeded/model", [1, 4]) == "" + + +def test_judge_overhead_not_charged_to_accelerator(): + """The judge is CPU-pinned; charging its 2.5 GB to a cuda budget caused + spurious VRAM skips.""" + from transformer_lens.tools.model_registry.verify_models import ( + estimate_benchmark_memory_gb, + ) + + # Small model so the phase-4 peak (model + judge) is the max across phases. + cpu = estimate_benchmark_memory_gb(int(1e6), phases=[1, 4], device="cpu") + cuda = estimate_benchmark_memory_gb(int(1e6), phases=[1, 4], device="cuda") + assert cpu > 2.5 + assert cuda < 0.1 diff --git a/transformer_lens/benchmarks/AGENTS.md b/transformer_lens/benchmarks/AGENTS.md index be6ad26d0b..a039bfd1e4 100644 --- a/transformer_lens/benchmarks/AGENTS.md +++ b/transformer_lens/benchmarks/AGENTS.md @@ -10,7 +10,7 @@ If an agent is here because the user asked to "update the registry" or "verify a ## What this directory IS for -- The phase-by-phase benchmark implementations (`forward_pass.py`, `generation.py`, `hook_registration.py`, `weight_processing.py`, `multimodal.py`, `audio.py`, `vision.py`, `encoder_common.py`, `text_quality.py`, `granular_weight_processing.py`, `component_outputs.py`, `backward_gradients.py`, `activation_cache.py`, `component_benchmark.py`, `hook_structure.py`). +- The phase-by-phase benchmark implementations (`forward_pass.py`, `generation.py`, `hook_registration.py`, `weight_processing.py`, `multimodal.py`, `audio.py`, `vision.py`, `encoder_common.py`, `text_quality.py`, `text_quality_profiles.py` (Phase-4 prompt-profile data + resolver), `granular_weight_processing.py`, `component_outputs.py`, `backward_gradients.py`, `activation_cache.py`, `component_benchmark.py`, `hook_structure.py`). - `main_benchmark.py` — exploratory benchmark runner for ad-hoc comparison. Useful for debugging a single model's phase scores without touching the registry. - `utils.py` — shared helpers including `BenchmarkSeverity`. diff --git a/transformer_lens/benchmarks/main_benchmark.py b/transformer_lens/benchmarks/main_benchmark.py index ceb434afc8..cc364e3ecd 100644 --- a/transformer_lens/benchmarks/main_benchmark.py +++ b/transformer_lens/benchmarks/main_benchmark.py @@ -5,7 +5,7 @@ Phase 1: HF + Bridge (unprocessed) - Compare against raw HuggingFace model Phase 2: Bridge (unprocessed) + HT (unprocessed) - Compare unprocessed models Phase 3: Bridge (processed) + HT (processed) - Full compatibility mode testing -Phase 4: Text Quality - Perplexity-based legibility scoring via GPT-2 Medium +Phase 4: Text Quality - profile prompts scored by a pinned judge's perplexity ratio Phase 5: Granular Weight Processing Tests (optional, individual flags) Phase 6: Granular Weight Processing Tests (optional, combined flags) Phase 7: Multimodal Tests (only for multimodal models with pixel_values support) @@ -592,8 +592,9 @@ def run_benchmark_suite( test_weight_processing_individually: bool = False, phases: list[int] | None = None, trust_remote_code: bool = False, - scoring_model: PreTrainedModel | None = None, - scoring_tokenizer: PreTrainedTokenizerBase | None = None, + judge_model: PreTrainedModel | None = None, + judge_tokenizer: PreTrainedTokenizerBase | None = None, + prompt_profile: str | None = None, ) -> List[BenchmarkResult]: """Run comprehensive benchmark suite for TransformerBridge. @@ -601,7 +602,7 @@ def run_benchmark_suite( Phase 1: HF + Bridge (unprocessed) - Compare against raw HuggingFace model Phase 2: Bridge (unprocessed) + HT (unprocessed) - Compare unprocessed models Phase 3: Bridge (processed) + HT (processed) - Full compatibility mode testing - Phase 4: Text Quality - Perplexity-based legibility scoring via GPT-2 + Phase 4: Text Quality - profile prompts scored by a pinned judge's perplexity ratio Phase 5: Individual Weight Processing Flags (optional) Phase 6: Combined Weight Processing Flags (optional) @@ -624,9 +625,12 @@ def run_benchmark_suite( tests that check each processing flag individually (default: False) phases: Optional list of phase numbers to run (e.g., [1, 2, 3]). If None, runs all phases. trust_remote_code: Whether to trust remote code for custom architectures. - scoring_model: Optional pre-loaded GPT-2 scoring model for Phase 4. When - provided with scoring_tokenizer, avoids reloading for each model in batch. - scoring_tokenizer: Optional pre-loaded tokenizer for Phase 4 scoring model. + judge_model: Optional pre-loaded Phase-4 judge. When provided with + judge_tokenizer, avoids reloading for each model in batch. + judge_tokenizer: Optional pre-loaded tokenizer for the Phase-4 judge. + prompt_profile: Optional Phase-4 prompt profile (e.g. "chat", + "task:translation@en-de"). Resolved from curation + the registry + when None. Returns: List of BenchmarkResult objects @@ -1441,7 +1445,7 @@ def cleanup_model(model, model_name_str: str): # (e.g., OpenELM). # ======================================================================== - # PHASE 4: Text Quality (GPT-2 perplexity scoring) + # PHASE 4: Text Quality (profile prompts, judge perplexity-ratio scoring) # Runs before Phase 3 so it can reuse bridge_unprocessed (Phase 3 # destructively processes the weights, consuming the bridge). # ======================================================================== @@ -1460,21 +1464,34 @@ def cleanup_model(model, model_name_str: str): and not is_masked_lm_model(model_name, trust_remote_code=trust_remote_code) and not is_audio_model(model_name, trust_remote_code=trust_remote_code) ): + if prompt_profile is None: + from transformer_lens.benchmarks.text_quality_profiles import ( + resolve_profile, + ) + from transformer_lens.tools.model_registry.registry_io import ( + registry_prompt_profile, + ) + + config = getattr(bridge_unprocessed, "original_model", None) + archs = getattr(getattr(config, "config", None), "architectures", None) or [] + prompt_profile = str( + resolve_profile( + model_name, archs[0] if archs else None, registry_prompt_profile(model_name) + ) + ) + if verbose: print(f"\n{'='*80}") - print("PHASE 2.5: Text Quality (GPT-2 perplexity scoring)") + print(f"PHASE 2.5: Text Quality (profile {prompt_profile}, judge ratio scoring)") print(f"{'='*80}\n") try: text_quality_result = benchmark_text_quality( bridge_unprocessed, - test_text, - max_new_tokens=50, - scoring_model_name="gpt2", - pass_threshold=85.0, - device=device, - scoring_model=scoring_model, - scoring_tokenizer=scoring_tokenizer, + prompt_profile, + judge_model=judge_model, + judge_tokenizer=judge_tokenizer, + model_name=model_name, ) text_quality_result.phase = 4 add_result(text_quality_result) @@ -2022,6 +2039,7 @@ def update_model_registry( _build_verified_note, _check_phase_scores, _extract_phase_scores, + _extract_prompt_profile, _pass_status, _sanitize_note, ) @@ -2057,6 +2075,7 @@ def update_model_registry( phase_scores=phase_scores, note=note, sanitize_fn=_sanitize_note, + prompt_profile=_extract_prompt_profile(results), ) # No history record for provisional runs — VerificationHistory.is_verified() diff --git a/transformer_lens/benchmarks/text_quality.py b/transformer_lens/benchmarks/text_quality.py index aa4c9b4bd9..b281c7dfb6 100644 --- a/transformer_lens/benchmarks/text_quality.py +++ b/transformer_lens/benchmarks/text_quality.py @@ -1,17 +1,22 @@ """Text quality benchmark for TransformerBridge. -Generates text with the bridge model from multiple diverse prompts and scores -each continuation's legibility using GPT-2 as a perplexity-based judge. -Only the generated continuation tokens are scored (prompt tokens are masked), -and a repetition penalty is applied to catch degenerate looping output. - -Generation is seeded for reproducibility, and the scoring model is loaded once -and reused across all prompts. +Generates text the way a real user of the model would (its prompt profile: +chat template, translation source, code, own-language continuation — see +``text_quality_profiles``) and scores each output against a known-good +reference completion with one pinned multilingual judge. The score derives +from the perplexity ratio PPL_judge(generated)/PPL_judge(reference), which +cancels the judge's per-language handicap; a repetition penalty catches +degenerate loops (which the ratio alone rewards) and a length penalty +catches truncated output. + +Generation is seeded per prompt for reproducibility, and the judge is loaded +once (CPU/fp32 always, so scores do not depend on the verifying machine) and +reused across all prompts. """ import gc import math -from typing import List, Optional, Tuple +from typing import Any, List, Optional, Tuple, Union import torch from transformers import ( @@ -21,6 +26,22 @@ PreTrainedTokenizerBase, ) +from transformer_lens.benchmarks.text_quality_profiles import ( + CAPTION_REFERENCES, + JUDGE_CONTEXT_KINDS, + JUDGE_R_FAIL, + LANG_ISO3, + LANG_NAMES, + MAX_NEW_TOKENS_BY_KIND, + NLLB_CODES, + PREPEND_BOS_BY_KIND, + T5_PREFIX_ARCHITECTURES, + TEMPERATURE_BY_KIND, + ProfilePrompt, + ProfileSpec, + p4_pass_threshold, + prompts_for, +) from transformer_lens.benchmarks.utils import ( BenchmarkResult, BenchmarkSeverity, @@ -28,96 +49,74 @@ ) from transformer_lens.model_bridge import TransformerBridge -# Diverse prompts used alongside the caller-provided test_text to get a robust -# quality signal across different domains and styles. -_DEFAULT_PROMPTS = [ - "The theory of relativity explains that", - "In the dense forests of the Amazon,", - "Modern computing relies heavily on", -] - - -def _load_scoring_model( - scoring_model_name: str, - device: str, -) -> Tuple[PreTrainedModel, PreTrainedTokenizerBase]: - """Load the scoring model and tokenizer. - - Separated from perplexity computation so the caller can load once and - reuse across multiple prompts. - """ - tokenizer = AutoTokenizer.from_pretrained(scoring_model_name) - model = AutoModelForCausalLM.from_pretrained(scoring_model_name) - torch.nn.Module.to(model, device) +# The one judge every model is scored with, pinned by revision so a Hub update +# can never silently move every score. Selection + measurements live in +# scripts/text_quality_judge_bakeoff.py; separation is weakest in de/ru, so +# scores there carry wider error bars. +JUDGE_MODEL_ID = "Qwen/Qwen2.5-0.5B" +JUDGE_REVISION = "060db6499f32faf8b98477b0a26969ef7d8b9987" + + +def load_judge() -> Tuple[PreTrainedModel, PreTrainedTokenizerBase]: + """Load the pinned judge on CPU in fp32 (machine-independent scores).""" + tokenizer = AutoTokenizer.from_pretrained(JUDGE_MODEL_ID, revision=JUDGE_REVISION) + model = AutoModelForCausalLM.from_pretrained( + JUDGE_MODEL_ID, revision=JUDGE_REVISION, dtype=torch.float32 + ) + torch.nn.Module.to(model, "cpu") model.eval() return model, tokenizer -def _compute_continuation_perplexity( - prompt: str, - full_text: str, - tokenizer: PreTrainedTokenizerBase, - scoring_model: PreTrainedModel, - device: str, +def _judge_perplexity( + text: str, + context: str, + tokenizer: Any, + judge: Any, ) -> Tuple[float, Optional[str]]: - """Compute perplexity of only the continuation tokens (excluding prompt). - - Prompt tokens are masked with -100 in labels so CrossEntropyLoss ignores - them. This prevents well-formed prompt text from artificially lowering - the perplexity of generated content. - - Args: - prompt: The original input prompt. - full_text: The complete text (prompt + generated continuation). - tokenizer: Pre-loaded tokenizer. - scoring_model: Pre-loaded scoring model. - device: Device string. - - Returns: - Tuple of (perplexity, error_message). error_message is None on success. - """ + """Judge perplexity of ``text``; ``context`` tokens are label-masked so only + ``text`` is scored. Returns (ppl, error).""" try: - encodings = tokenizer(full_text, return_tensors="pt") - input_ids = encodings["input_ids"].to(device) + # Tokenize the pieces separately: tokenizing the concatenated string + # lets BPE merge across the boundary and shifts the label mask into + # the scored text. + text_ids = tokenizer(text, return_tensors="pt")["input_ids"] + context_len = 0 + input_ids = text_ids + if context: + context_ids = tokenizer(context, return_tensors="pt")["input_ids"] + context_len = context_ids.shape[1] + input_ids = torch.cat([context_ids, text_ids], dim=1) + + if text_ids.shape[1] < 2: + return float("inf"), "Scored text too short (< 2 judge tokens)" - # Tokenize just the prompt to find where continuation starts - prompt_encodings = tokenizer(prompt, return_tensors="pt") - prompt_len = prompt_encodings["input_ids"].shape[1] - - # Build labels: -100 for prompt positions, actual ids for continuation labels = input_ids.clone() - labels[0, :prompt_len] = -100 - - continuation_len = input_ids.shape[1] - prompt_len - if continuation_len < 2: - return float("inf"), "Generated continuation too short (< 2 tokens)" + if context_len: + labels[0, :context_len] = -100 with torch.no_grad(): - outputs = scoring_model(input_ids, labels=labels) - loss = outputs.loss.item() - - perplexity = math.exp(loss) - return perplexity, None - + loss = judge(input_ids, labels=labels).loss.item() + return math.exp(loss), None except Exception as e: return float("inf"), f"Perplexity computation failed: {str(e)}" def _compute_repetition_penalty(text: str, ns: Tuple[int, ...] = (2, 3, 4)) -> float: - """Compute a repetition penalty based on n-gram uniqueness ratio. - - Returns a multiplier in [0.0, 1.0] where 1.0 means no repetition and - lower values penalize repetitive text. The penalty is the minimum - unique-n-gram ratio across all checked n-gram sizes. + """Minimum unique-n-gram ratio in [0, 1]; low values mean looping output. - Args: - text: The generated continuation text (prompt excluded). - ns: Tuple of n-gram sizes to check. - - Returns: - Penalty multiplier in [0.0, 1.0]. + Load-bearing under ratio scoring: a degenerate loop has LOW judge + perplexity, so without this multiplier it would score 100. """ words = text.lower().split() + # Scriptio continua (zh/ja): word n-grams are inert exactly where the + # judge rewards loops most, and a single stray space would restore the + # word path — so char n-grams whenever the text is CJK-dominated. + compact = "".join(text.split()) + if compact: + cjk = sum(1 for c in compact if 0x3040 <= ord(c) <= 0x30FF or 0x4E00 <= ord(c) <= 0x9FFF) + if cjk / len(compact) >= 0.3 and len(compact) >= 8: + words = list(compact) if len(words) < 2: return 1.0 @@ -134,23 +133,84 @@ def _compute_repetition_penalty(text: str, ns: Tuple[int, ...] = (2, 3, 4)) -> f return min_ratio -def _perplexity_to_score(perplexity: float) -> float: - """Map continuation perplexity to a 0-100 legibility score. - - Uses: score = 135 - 10 * ln(perplexity), capped to [0, 100]. - Calibrated for continuation-only perplexity (higher than full-text). - A well-functioning model typically gets ppl 40-60 -> score 94-98. - Default pass threshold of 85 corresponds to approximately ppl 150. +def _ratio_to_score(ratio: float) -> float: + """Map generated/reference perplexity ratio to 0-100. - Args: - perplexity: The perplexity value from the scoring model. - - Returns: - Score from 0.0 to 100.0. + score = 100 - 100*ln(ratio)/ln(R_FAIL), clamped: ratio<=1 (as good as the + reference) scores 100, ratio=R_FAIL scores 0, and score 50 falls at + sqrt(R_FAIL) — the geometric midpoint between reference quality and + unambiguously broken output, which keeps the registry's phase-4 floor of + 50 principled. """ - if perplexity <= 0 or math.isinf(perplexity): + if ratio <= 0 or math.isinf(ratio) or math.isnan(ratio): return 0.0 - return max(0.0, min(100.0, 135.0 - 10.0 * math.log(perplexity))) + if ratio <= 1.0: + return 100.0 + return max(0.0, min(100.0, 100.0 - 100.0 * math.log(ratio) / math.log(JUDGE_R_FAIL))) + + +_SCRIPT_RANGES: dict[str, Tuple[Tuple[int, int], ...]] = { + "zh": ((0x4E00, 0x9FFF),), + "ja": ((0x3040, 0x30FF), (0x4E00, 0x9FFF)), + "ar": ((0x0600, 0x06FF),), + "ru": ((0x0400, 0x04FF),), + "hi": ((0x0900, 0x097F),), +} + +_LATIN_STOPWORDS: dict[str, frozenset] = { + "en": frozenset("the and of to is that with for was are it in on".split()), + "fr": frozenset( + "le la les des une est que je pas dans de et il elle un en du au pour sur ne ce se".split() + ), + "es": frozenset("el los una es que no por con para como de en la y se del las".split()), + "de": frozenset( + "der die das und ist nicht ich ein eine mit den von zu im auf f\u00fcr sich".split() + ), + "it": frozenset("il la di che non per una sono del gli e in un le si con".split()), + "nl": frozenset("de het een en is niet ik van dat met op voor aan zijn".split()), + "pt": frozenset("o os uma de e que n\u00e3o para com por em um as dos da".split()), +} + + +def _wrong_language(text: str, lang: str) -> bool: + """Conservatively true only when the text is clearly NOT in ``lang``. + + Ratio scoring alone measures fluency, not language: fluent English output + beats a short German reference and clamps to 100, so an untranslated echo + would otherwise score perfectly. Non-Latin targets check script presence; + Latin targets require zero expected-language stopwords while another + covered language has several. + """ + if lang in ("code", ""): + return False + ranges = _SCRIPT_RANGES.get(lang) + if ranges is not None: + letters = [c for c in text if c.isalpha()] + if not letters: + return False + in_script = sum(1 for c in letters if any(lo <= ord(c) <= hi for lo, hi in ranges)) + return in_script / len(letters) < 0.3 + expected = _LATIN_STOPWORDS.get(lang) + if expected is None: + return False + tokens = [w.strip(".,;:!?\"'()") for w in text.lower().split()] + hits = {code: sum(1 for w in tokens if w in stops) for code, stops in _LATIN_STOPWORDS.items()} + return hits[lang] == 0 and max(hits.values(), default=0) >= 3 + + +def _length_penalty(gen_tokens: int, ref_tokens: int) -> float: + """Penalize output far shorter OR far longer than its reference. + + Neutral band [0.5x, 3x] of reference length. The old 25% floor never + fired once in four validation sweeps — a contentless 13-token chat stub + against a 41-token reference scored 93.6; at a 0.5x floor it drops below + the pass line. The 3x cap is the second net for rambling output the + repetition penalty misses.""" + if ref_tokens <= 0: + return 1.0 + under = gen_tokens / (0.5 * ref_tokens) + over = (3.0 * ref_tokens) / max(gen_tokens, 1) + return max(0.0, min(1.0, under, over)) def _build_caption_test_images(n: int = 3) -> list: @@ -184,7 +244,7 @@ def _build_caption_test_images(n: int = 3) -> list: def _generate_image_conditioned_captions( bridge: TransformerBridge, max_new_tokens: int -) -> List[Tuple[str, str]]: +) -> List[Tuple[int, str]]: """Caption synthetic images for image-conditioned seq2seq (Florence-2 emits nothing text-only, so text-only P4 is uninformative); [] if no processor/PIL.""" processor = getattr(bridge, "processor", None) @@ -202,7 +262,7 @@ def _generate_image_conditioned_captions( is_task_captioner = hasattr(processor, "post_process_generation") task = "" if is_task_captioner else "Describe this image in detail." - samples: List[Tuple[str, str]] = [] + samples: List[Tuple[int, str]] = [] for i, image in enumerate(images): try: inputs = processor(text=task, images=image, return_tensors="pt") @@ -216,48 +276,135 @@ def _generate_image_conditioned_captions( input_ids, max_new_tokens=max_new_tokens, return_type="tokens", **extra ) if isinstance(out, torch.Tensor): - text = bridge.tokenizer.decode(out[0], skip_special_tokens=True).strip() + is_encoder_decoder = bool( + getattr(getattr(bridge, "original_model", None), "config", None) + and getattr(bridge.original_model.config, "is_encoder_decoder", False) + ) + # Decoder-only VLM output is prompt + continuation; scoring the + # fluent prompt as caption text would inflate every sample. + caption_ids = out[0] if is_encoder_decoder else out[0, input_ids.shape[-1] :] + text = bridge.tokenizer.decode(caption_ids, skip_special_tokens=True).strip() if text: - samples.append((f"image_{i}", text)) + samples.append((i, text)) except Exception: continue return samples +def _architecture_id(bridge: Any) -> str: + """First HF architecture name of the wrapped model, or ''.""" + config = getattr(getattr(bridge, "original_model", None), "config", None) + architectures = getattr(config, "architectures", None) or [] + return architectures[0] if architectures else "" + + +def _resolve_lang_code(tokenizer, lang: str) -> Optional[str]: + """The tokenizer's own code string for ``lang`` ("de" / "de_DE" / + "deu_Latn"), or None. transformers 5.x NllbTokenizer exposes neither + get_lang_id nor lang_code_to_id, so candidates are probed through the + vocab as well.""" + lang = lang.lower() + lang_code_to_id = getattr(tokenizer, "lang_code_to_id", None) + if isinstance(lang_code_to_id, dict): + if lang in lang_code_to_id: + return lang + iso3 = LANG_ISO3.get(lang, "") + for code in lang_code_to_id: + code_lower = code.lower() + if code_lower.startswith(lang + "_") or (iso3 and code_lower.startswith(iso3 + "_")): + return code + # Vocab probing is only safe for DISTINCTIVE code forms ("deu_Latn"): a + # bare ISO code collides with ordinary subwords (T5's "de", Marian's "en") + # and would be injected as a forced decoder token. + nllb = NLLB_CODES.get(lang) + unk_id = getattr(tokenizer, "unk_token_id", None) + convert = getattr(tokenizer, "convert_tokens_to_ids", None) + if nllb and callable(convert): + try: + token_id = convert(nllb) + except Exception: + return None + if isinstance(token_id, int) and token_id >= 0 and token_id != unk_id: + return nllb + return None + + +def _forced_bos_for_target(tokenizer, tgt_lang: str) -> Optional[int]: + """Target-language decoder token for multilingual translators, or None.""" + get_lang_id = getattr(tokenizer, "get_lang_id", None) + if callable(get_lang_id): + try: + return int(get_lang_id(tgt_lang)) + except Exception: + return None + code = _resolve_lang_code(tokenizer, tgt_lang) + if code is None: + return None + lang_code_to_id = getattr(tokenizer, "lang_code_to_id", None) + if isinstance(lang_code_to_id, dict) and code in lang_code_to_id: + return int(lang_code_to_id[code]) + try: + token_id = tokenizer.convert_tokens_to_ids(code) + except Exception: + return None + if ( + isinstance(token_id, int) + and token_id >= 0 + and token_id != getattr(tokenizer, "unk_token_id", None) + ): + return int(token_id) + return None + + +def _build_model_input( + bridge: Any, + spec: ProfileSpec, + prompt: ProfilePrompt, + architecture_id: str, +) -> str: + """Render one profile prompt into the text this model expects.""" + if spec.kind == "chat": + return bridge.tokenizer.apply_chat_template( + [{"role": "user", "content": prompt.prompt}], + add_generation_prompt=True, + tokenize=False, + ) + if spec.kind == "task:translation" and architecture_id in T5_PREFIX_ARCHITECTURES: + src_name = LANG_NAMES.get(spec.src or "en", "English") + tgt_name = LANG_NAMES.get(spec.lang, "German") + return f"translate {src_name} to {tgt_name}: {prompt.prompt}" + if spec.kind == "task:summarization" and architecture_id in T5_PREFIX_ARCHITECTURES: + return f"summarize: {prompt.prompt}" + return prompt.prompt + + def benchmark_text_quality( - bridge: TransformerBridge, - test_text: str, - max_new_tokens: int = 50, - scoring_model_name: str = "gpt2", - pass_threshold: float = 85.0, - device: str = "cpu", - scoring_model: Optional[PreTrainedModel] = None, - scoring_tokenizer: Optional[PreTrainedTokenizerBase] = None, + bridge: Any, + profile: Union[str, ProfileSpec] = "continuation", + *, + max_new_tokens: Optional[int] = None, + judge_model: Optional[Any] = None, + judge_tokenizer: Optional[Any] = None, + model_name: Optional[str] = None, ) -> BenchmarkResult: - """Benchmark text generation quality using continuation-only perplexity scoring. - - Generates text from multiple diverse prompts, scores each continuation using - GPT-2 perplexity (prompt tokens masked), applies a repetition penalty, - and returns the averaged score. - - Args: - bridge: TransformerBridge model to test. - test_text: Primary input prompt (additional diverse prompts are also used). - max_new_tokens: Number of tokens to generate per prompt. - scoring_model_name: HuggingFace model to use as scorer. - pass_threshold: Minimum average score to pass (default 95.0). - device: Device for the scoring model. - scoring_model: Optional pre-loaded scoring model. When provided alongside - scoring_tokenizer, skips loading and avoids cleanup (caller owns lifecycle). - scoring_tokenizer: Optional pre-loaded tokenizer for the scoring model. - - Returns: - BenchmarkResult with quality score details. + """Benchmark text generation quality with profile prompts and reference-ratio scoring. + + Generates from the model's prompt-profile prompts through the real user + path (``bridge.generate``), then scores each output against the prompt's + reference completion via the pinned judge's perplexity ratio, with + repetition and length penalties. """ + if model_name is not None and model_name.lower() == JUDGE_MODEL_ID.lower(): + # Ratio scoring against the judge's own perplexity is self-grading. + return BenchmarkResult( + name="text_quality", + severity=BenchmarkSeverity.SKIPPED, + message=f"P4 skipped: {model_name} is the pinned judge — cannot self-score", + ) _loaded_locally = False - tokenizer = scoring_tokenizer + tokenizer = judge_tokenizer try: - prompts = [test_text] + _DEFAULT_PROMPTS + spec = ProfileSpec.parse(profile) if isinstance(profile, str) else profile # Diffusion LMs produce text through their native sampler; scoring that # text is as meaningful as scoring autoregressive output. @@ -271,95 +418,211 @@ def benchmark_text_quality( message="Skipped: architecture supports no text generation", ) - # Encoder-decoder models (T5/Marian/BART) emit a standalone decoder - # output (translation, summary), not a continuation of the prompt, so - # there is no prompt prefix to mask out — the whole generated text is the - # content to score. (An en-in→en translation whose output ~= the prompt - # length otherwise trips the "continuation too short" guard for every - # prompt and scores 0.) is_encoder_decoder = bool( getattr(getattr(bridge, "original_model", None), "config", None) and getattr(bridge.original_model.config, "is_encoder_decoder", False) ) - # Image-conditioned seq2seq (e.g. Florence-2) emits a 1-token EOS for a - # text-only prompt — it needs pixel_values to produce anything. For those - # we drive real caption generation from test images and score that. is_multimodal = bool(getattr(getattr(bridge, "cfg", None), "is_multimodal", False)) - image_conditioned = is_encoder_decoder and is_multimodal - # Generate text to score (prompt, full_text) - generations: List[Tuple[str, str]] = [] + # Effective-profile adjustments. Image-conditioned seq2seq (Florence-2) + # emits a bare EOS for text-only prompts — caption real images instead. + # A chat profile without a chat template downgrades to continuation; + # never the other direction (base models may ship templates). + adjustment = "" + if is_encoder_decoder and is_multimodal: + spec = ProfileSpec("caption") + elif spec.kind == "chat": + if getattr(bridge.tokenizer, "chat_template", None) is None: + spec = ProfileSpec("continuation", spec.lang) + adjustment = "chat profile downgraded: tokenizer has no chat template" + else: + try: + bridge.tokenizer.apply_chat_template( + [{"role": "user", "content": "probe"}], + add_generation_prompt=True, + tokenize=False, + ) + except Exception as template_error: + spec = ProfileSpec("continuation", spec.lang) + adjustment = f"chat profile downgraded: template raised {template_error!r}" + + denoise_style = "mask" if getattr(bridge.tokenizer, "mask_token", None) else "t5" + profile_prompts = prompts_for(spec, denoise_style=denoise_style) + if profile_prompts is None: + return BenchmarkResult( + name="text_quality", + severity=BenchmarkSeverity.SKIPPED, + message=( + f"P4 skipped: no prompts for profile '{spec}' — file a " + "TransformerLens issue to add coverage in " + "benchmarks/text_quality_profiles.py" + ), + ) + + if max_new_tokens is None: + max_new_tokens = MAX_NEW_TOKENS_BY_KIND.get(spec.kind, 50) + + architecture_id = _architecture_id(bridge) + forced_bos: Optional[int] = None + if spec.kind == "task:translation": + src_lang_attr = getattr(bridge.tokenizer, "src_lang", None) + if src_lang_attr is not None and spec.src: + src_code = _resolve_lang_code(bridge.tokenizer, spec.src) + if src_code is not None: + try: + bridge.tokenizer.src_lang = src_code + except Exception: + pass + forced_bos = _forced_bos_for_target(bridge.tokenizer, spec.lang) + + # Generate: (profile_prompt, generated_text) pairs. Token-level slicing — + # generate() decodes with skip_special_tokens, so the prompt string is + # not reliably a prefix of the output string (chat templates). + generations: List[Tuple[ProfilePrompt, str]] = [] primary_generated = "" - if image_conditioned: + if spec.kind == "caption": with deterministic_rng(): captions = _generate_image_conditioned_captions(bridge, max_new_tokens) if not captions: - # Cannot reach this model's real (image-conditioned) generation — - # skip rather than score its degenerate text-only output. return BenchmarkResult( name="text_quality", severity=BenchmarkSeverity.SKIPPED, message="Skipped: image-conditioned model; image processor/PIL unavailable", ) - # No prompt prefix to mask — the whole caption is the content (handled - # by the is_encoder_decoder path in the scoring loop below). - generations = [("", text) for _, text in captions] + generations = [ + (ProfilePrompt(prompt="", reference=CAPTION_REFERENCES[i]), text) + for i, text in captions + if i < len(CAPTION_REFERENCES) + ] primary_generated = captions[0][1] else: - with deterministic_rng(): - for i, prompt in enumerate(prompts): - generated = generator( - prompt, - max_new_tokens=max_new_tokens, - temperature=0.7, + prepend_bos = PREPEND_BOS_BY_KIND.get(spec.kind) + # Native diffusion samplers take neither return_type nor forced_bos; + # bound-method identity can't detect them (new object per access). + is_autoregressive = getattr(bridge.adapter, "supports_generation", True) + for prompt in profile_prompts: + model_input = _build_model_input(bridge, spec, prompt, architecture_id) + if is_encoder_decoder: + # Encoder input follows the tokenizer's own recipe (lang + # token + trailing ); to_tokens' BOS policy corrupts it + # (m2m100 loops on a stray ). + prompt_ids = bridge.tokenizer(model_input, return_tensors="pt")["input_ids"].to( + bridge.cfg.device ) - if not isinstance(generated, str) or len(generated.strip()) == 0: - continue - generations.append((prompt, generated)) - if i == 0: - primary_generated = generated + else: + prompt_ids = bridge.to_tokens(model_input, prepend_bos=prepend_bos) + gen_kwargs: dict = { + "max_new_tokens": max_new_tokens, + "temperature": TEMPERATURE_BY_KIND.get(spec.kind, 0.7), + } + if is_autoregressive: + gen_kwargs["return_type"] = "tokens" + if forced_bos is not None: + gen_kwargs["forced_bos_token_id"] = forced_bos + # Seeded per prompt so each sample stream is independent of the + # previous prompt's length. + with deterministic_rng(): + out = generator(prompt_ids, **gen_kwargs) + if not isinstance(out, torch.Tensor): + continue + generated_ids = out[0, 1:] if is_encoder_decoder else out[0, prompt_ids.shape[-1] :] + generated = bridge.tokenizer.decode(generated_ids, skip_special_tokens=True) + if spec.kind == "task:denoise" and denoise_style == "t5": + # Splice the fill back so both ratio sides are full + # sentences (bare fragments judge in the thousands). An + # EMPTY fill must stay empty or a dead model inherits the + # near-reference sentence and a free 100. + if generated.strip(): + generated = prompt.prompt.replace("", generated.strip()) + # Empty output is a scored failure (0), not a dropped sample — + # dropping it would average only over the prompts that worked. + generations.append((prompt, generated)) + if not primary_generated: + primary_generated = generated if len(generations) == 0: return BenchmarkResult( name="text_quality", severity=BenchmarkSeverity.DANGER, - message="Generation produced empty output for all prompts", + message="Generation produced no scoreable output for any prompt", passed=False, ) - # Load scoring model if not pre-loaded by caller - if scoring_model is None or tokenizer is None: - scoring_model, tokenizer = _load_scoring_model(scoring_model_name, device) + if judge_model is None or tokenizer is None: + judge_model, tokenizer = load_judge() _loaded_locally = True - # Score each continuation + # Judge context per kind is JUDGE_CONTEXT_KINDS' call. Translation is + # scored jointly: per-sentence judge perplexity on the short pivots is + # unstable (measured spread 4.8-3497), so the samples concatenate into + # one gen/ref pair. + # Captured pre-merge so translation keeps its per-sentence texts. + all_generated_texts = [text for _, text in generations] + + if spec.kind == "task:translation" and len(generations) > 1: + joiner = "" if spec.lang in ("zh", "ja") else " " + joint = ProfilePrompt( + prompt="", + reference=joiner.join(g[0].reference for g in generations), + lang=spec.lang, + ) + generations = [(joint, joiner.join(g[1] for g in generations))] + + sample_lang = spec.lang if spec.kind != "caption" else "en" per_prompt_scores = [] - per_prompt_perplexities = [] + per_prompt_ratios = [] per_prompt_penalties = [] prompt_details_parts = [] - for prompt, full_text in generations: - # For encoder-decoder output there is no prompt-in-continuation to - # mask; score the entire generated sequence. - score_prompt = "" if is_encoder_decoder else prompt - perplexity, error = _compute_continuation_perplexity( - score_prompt, full_text, tokenizer, scoring_model, device - ) - if error is not None: + for prompt, generated in generations: + context = prompt.prompt if spec.kind in JUDGE_CONTEXT_KINDS else "" + + gen_token_count = len(tokenizer(generated)["input_ids"]) if generated.strip() else 0 + if gen_token_count < 2: + # Empty or one-token output is a scored failure, not a dropped + # sample (Florence-style bare EOS, dead generation). + per_prompt_scores.append(0.0) + per_prompt_ratios.append(float("inf")) + per_prompt_penalties.append(0.0) + prompt_details_parts.append("score=0.0 (output < 2 tokens)") + continue + check_lang = prompt.lang if spec.kind != "task:translation" else sample_lang + if _wrong_language(generated, check_lang): + # Fluency-only ratio scoring would rate untranslated or + # wrong-language output above the reference; hard zero. + per_prompt_scores.append(0.0) + per_prompt_ratios.append(float("inf")) + per_prompt_penalties.append(0.0) + prompt_details_parts.append(f"score=0.0 (output not in '{check_lang}')") continue - raw_score = _perplexity_to_score(perplexity) + gen_ppl, gen_err = _judge_perplexity(generated, context, tokenizer, judge_model) + ref_ppl, ref_err = _judge_perplexity(prompt.reference, context, tokenizer, judge_model) + if gen_err is not None: + # The model's own output was unjudgeable — scored failure. + per_prompt_scores.append(0.0) + per_prompt_ratios.append(float("inf")) + per_prompt_penalties.append(0.0) + prompt_details_parts.append(f"score=0.0 ({gen_err})") + continue + if ref_err is not None: + # Our reference failed to judge — a data problem, not the + # model's; exclude the sample and say so. + prompt_details_parts.append(f"excluded (reference: {ref_err})") + continue - # Repetition penalty on continuation only - continuation = full_text[len(score_prompt) :] - rep_penalty = _compute_repetition_penalty(continuation) - adjusted_score = raw_score * rep_penalty + ratio = gen_ppl / ref_ppl if ref_ppl > 0 else float("inf") + rep_penalty = _compute_repetition_penalty(generated) + ref_token_count = len(tokenizer(prompt.reference)["input_ids"]) + len_penalty = _length_penalty(gen_token_count, ref_token_count) + adjusted_score = _ratio_to_score(ratio) * rep_penalty * len_penalty per_prompt_scores.append(adjusted_score) - per_prompt_perplexities.append(perplexity) + per_prompt_ratios.append(ratio) per_prompt_penalties.append(rep_penalty) prompt_details_parts.append( - f"ppl={perplexity:.1f} score={adjusted_score:.1f} rep={rep_penalty:.2f}" + f"ratio={ratio:.2f} ppl={gen_ppl:.1f} ref_ppl={ref_ppl:.1f} " + f"rep={rep_penalty:.2f} len={len_penalty:.2f} score={adjusted_score:.1f}" ) if len(per_prompt_scores) == 0: @@ -372,19 +635,26 @@ def benchmark_text_quality( ) avg_score = sum(per_prompt_scores) / len(per_prompt_scores) - avg_perplexity = sum(per_prompt_perplexities) / len(per_prompt_perplexities) + finite_ratios = [r for r in per_prompt_ratios if math.isfinite(r)] + avg_ratio = sum(finite_ratios) / len(finite_ratios) if finite_ratios else float("inf") avg_rep_penalty = sum(per_prompt_penalties) / len(per_prompt_penalties) + pass_threshold = p4_pass_threshold() details = { "score": round(avg_score, 1), - "avg_perplexity": round(avg_perplexity, 2), + "prompt_profile": str(spec), + "judge_model": JUDGE_MODEL_ID, + "judge_revision": JUDGE_REVISION, + "avg_ratio": round(avg_ratio, 3) if math.isfinite(avg_ratio) else "inf", "avg_repetition_penalty": round(avg_rep_penalty, 2), "num_prompts": len(per_prompt_scores), "per_prompt": " | ".join(prompt_details_parts), - "scoring_model": scoring_model_name, "max_new_tokens": max_new_tokens, "generated_text": primary_generated, + "generated_texts": all_generated_texts, } + if adjustment: + details["profile_adjustment"] = adjustment if avg_score >= pass_threshold: return BenchmarkResult( @@ -392,18 +662,17 @@ def benchmark_text_quality( severity=BenchmarkSeverity.INFO, message=( f"Text quality score: {avg_score:.1f}/100 " - f"(avg perplexity: {avg_perplexity:.1f}, " - f"{len(per_prompt_scores)} prompts)" + f"(profile {spec}, {len(per_prompt_scores)} prompts)" ), details=details, ) - elif avg_score >= 80.0: + elif avg_score >= pass_threshold / 2: return BenchmarkResult( name="text_quality", severity=BenchmarkSeverity.WARNING, message=( f"Text quality score: {avg_score:.1f}/100 " - f"(below {pass_threshold}, avg perplexity: {avg_perplexity:.1f})" + f"(below {pass_threshold:.0f}, profile {spec})" ), details=details, passed=False, @@ -414,8 +683,7 @@ def benchmark_text_quality( severity=BenchmarkSeverity.DANGER, message=( f"Text quality score: {avg_score:.1f}/100 " - f"(avg perplexity: {avg_perplexity:.1f}) " - f"— generated text may be incoherent" + f"(profile {spec}) — generated text may be incoherent" ), details=details, passed=False, @@ -431,13 +699,8 @@ def benchmark_text_quality( finally: if _loaded_locally: - if scoring_model is not None: - del scoring_model + if judge_model is not None: + del judge_model if tokenizer is not None: del tokenizer gc.collect() - if device != "cpu" and torch.cuda.is_available(): - torch.cuda.empty_cache() - if device == "mps" and hasattr(torch, "mps") and hasattr(torch.mps, "empty_cache"): - torch.mps.synchronize() - torch.mps.empty_cache() diff --git a/transformer_lens/benchmarks/text_quality_profiles.py b/transformer_lens/benchmarks/text_quality_profiles.py new file mode 100644 index 0000000000..2b09083bcb --- /dev/null +++ b/transformer_lens/benchmarks/text_quality_profiles.py @@ -0,0 +1,1141 @@ +"""Prompt profiles and reference data for the Phase-4 text-quality benchmark. + +Each verified model is scored on prompts a real user would feed it (its +``prompt_profile``): chat models get their chat template, translation models get +source sentences, code models get code, multilingual models get their own +language. Every prompt carries a known-good reference completion; scoring is the +ratio of judge perplexities PPL(generated)/PPL(reference), which cancels the +judge's per-language handicap. + +Profile resolution is curation-first because Hub metadata is unreliable +(observed live 2026-08-20): ``bigscience/mt0-base`` is mis-tagged +``text-generation``; ``facebook/m2m100_418M`` and ``google/long-t5-tglobal-base`` +have no ``pipeline_tag`` at all; the ``conversational`` tag is added by HF for +*any* repo shipping a chat template, including base models like +``Qwen/Qwen2.5-0.5B``; Helsinki-NLP language tags are unordered, so Marian +direction must come from the model id. Precedence: per-model override > +architecture rule > fetched HF signals > stored registry value > default. + +Pivot sentences are from Tatoeba (https://tatoeba.org, CC BY 2.0 FR); source +sentence ids are noted inline. Everything else is hand-authored. + +This module stays stdlib-only: the registry scraper imports it at scan time. + +Language x kind coverage (prompts exist where marked; uncovered combinations +SKIP with a file-an-issue message, they never score against wrong-language +data): + + kind en fr es de zh ja ru ar hi it nl pt ro code + continuation x x x x x x x x - - - - - x + chat x x x x x x x x - - - - - - + task:instruction x x - - x - - - - - - - - - + task:summarization x x - - x - - - - - - - - - + task:denoise x - - - - - - - - - - - - - + PIVOT (translation) x x x x x x x x x x x x - - + +hi/it/nl/pt have pivot coverage only (translation targets); ro exists only in +NLLB_CODES. Filling continuation/chat for those plus it/nl/pt/hi bake-off +calibration is tracked as a follow-up. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Optional + +PROFILE_KINDS = ( + "continuation", + "chat", + "task:instruction", + "task:translation", + "task:summarization", + "task:denoise", + "caption", +) + + +@dataclass(frozen=True) +class ProfileSpec: + """A parsed prompt profile: what to feed the model and in which language.""" + + kind: str + lang: str = "en" + src: Optional[str] = None # translation source language + + @classmethod + def parse(cls, spec: str) -> "ProfileSpec": + """Parse ``kind[@lang]`` (translation: ``@src-tgt``); '@' because task kinds contain ':'.""" + kind, _, lang = spec.partition("@") + if kind not in PROFILE_KINDS: + raise ValueError(f"Unknown profile kind {kind!r} in {spec!r}") + if not lang: + return cls(kind=kind) + if kind == "task:translation": + src, sep, tgt = lang.partition("-") + if not sep or not src or not tgt: + raise ValueError(f"Translation profile needs '@src-tgt', got {spec!r}") + return cls(kind=kind, lang=tgt, src=src) + return cls(kind=kind, lang=lang) + + def __str__(self) -> str: + if self.kind == "task:translation" and self.src: + return f"{self.kind}@{self.src}-{self.lang}" + if self.lang != "en": + return f"{self.kind}@{self.lang}" + return self.kind + + +@dataclass(frozen=True) +class ProfilePrompt: + """One scored sample: model input and a known-good reference completion.""" + + prompt: str + reference: str + lang: str = "en" + + +DEFAULT_PROFILE = ProfileSpec("continuation", "en") + + +def is_default_profile(profile) -> bool: + """One sparse-encoding rule for every registry writer: the bare default + continuation@en profile is never stored (a lang-tagged continuation is).""" + if isinstance(profile, ProfileSpec): + return profile == DEFAULT_PROFILE + try: + return ProfileSpec.parse(str(profile)) == DEFAULT_PROFILE + except ValueError: + return False + + +# --------------------------------------------------------------------------- +# Pivot sentences (Tatoeba, CC BY 2.0 FR) — index-aligned across languages. +# English #1277 / #1284 / #1315; per-language ids in row comments. +# Feed translation pairs and the judge bake-off's fluent corpus. +# --------------------------------------------------------------------------- + +PIVOT_SENTENCES: dict[str, tuple[str, str, str]] = { + "en": ( # 1277, 1284, 1315 + "I have to go to sleep.", + "I will be back soon.", + "I can't live that kind of life.", + ), + "fr": ( # 373908, 3099, 3131 + "Je dois aller dormir.", + "Je serai bientôt de retour.", + "Je ne peux pas vivre comme ça.", + ), + "es": ( # 2482, 2489, 2521 + "Tengo que irme a dormir.", + "Volveré pronto.", + "No puedo vivir así.", + ), + "de": ( # 1195088, 85, 117 + "Ich muss schlafen.", + "Ich werde bald zurück sein.", + "Ich kann so ein Leben nicht leben.", + ), + "it": ( # 4369, 375118, 2733911 + "Devo andare a dormire.", + "Torno subito.", + "Non posso vivere quel tipo di vita.", + ), + "nl": ( # 5966, 5984, 378741 + "Ik moet gaan slapen.", + "Ik ben zo terug.", + "Ik kan zo niet leven.", + ), + "pt": ( # 182184, 331974, 405254 + "Preciso ir dormir.", + "Voltarei em breve.", + "Eu não posso viver esse tipo de vida.", + ), + "ru": ( # 5410, 374353, 5449 + "Мне пора идти спать.", + "Я скоро вернусь.", + "Я так жить не могу.", + ), + "zh": ( # 2, 9 (Hans transcription), 35 — one script; mixing traditional + # into a simplified-dominant judge destroys that row's zero point. + "我该去睡觉了。", + "我很快就会回来。", + "我不能这样活着。", + ), + "ja": ( # 4703, 4709, 4742 + "私は眠らなければなりません。", + "すぐに戻ります。", + "私はそんな風には生きられない。", + ), + "ar": ( # 372962, 400781, 549626 + "عليّ أن أنام.", + "سأعود قريباً.", + "لا أستطيع أن أعيش حياة كتلك.", + ), + "hi": ( # 3792910, 3793971, 11371181 + "मुझे सोना है।", + "मैं जल्द लौटूंगी।", + "मैं ऐसी जिंदगी नहीं जी सकता।", + ), +} + +# --------------------------------------------------------------------------- +# Continuation prompts. English is seeded from the pre-rework default prompts so +# control-model scores stay comparable. "code" is a language here: code models +# continue code the way prose models continue prose. +# --------------------------------------------------------------------------- + +CONTINUATION_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + "en": ( + ProfilePrompt( + "The theory of relativity explains that", + " time and space are not absolute but depend on the observer's " + "motion, so clocks moving at high speed tick more slowly than " + "clocks at rest.", + ), + ProfilePrompt( + "In the dense forests of the Amazon,", + " thousands of plant and animal species live in a delicate " + "balance, and scientists continue to discover new ones every year.", + ), + ProfilePrompt( + "Modern computing relies heavily on", + " fast processors and large amounts of memory, which allow " + "software to handle enormous quantities of data in real time.", + ), + ProfilePrompt( + "The city library opens early on weekdays, and", + # Judge PPL 8.7 (en median 8.9). References must stay within + # ~3.5x of the language median or this prompt's bar loosens + # proportionally; the integration test pins the band. + " many people stop by in the morning to read or borrow books before work.", + ), + ), + "fr": ( + ProfilePrompt( + "La tour Eiffel est l'un des monuments", + " les plus célèbres du monde, et des millions de visiteurs " + "montent chaque année à son sommet pour admirer Paris.", + lang="fr", + ), + ProfilePrompt( + "Chaque matin, le boulanger du village", + " prépare du pain frais et des croissants que les habitants " + "viennent acheter dès l'ouverture de la boutique.", + lang="fr", + ), + ProfilePrompt( + "La science moderne repose sur", + " l'observation, l'expérience et le raisonnement, qui permettent " + "de comprendre les lois de la nature.", + lang="fr", + ), + ProfilePrompt( + "Pendant l'hiver, les montagnes", + " se couvrent de neige et attirent de nombreux skieurs venus de " "toute l'Europe.", + lang="fr", + ), + ), + "es": ( + ProfilePrompt( + "El clima de la región mediterránea es", + " templado, con veranos secos y calurosos e inviernos suaves y " + "lluviosos, ideal para el cultivo de olivos.", + lang="es", + ), + ProfilePrompt( + "Cada domingo por la mañana, el mercado", + " se llena de gente que compra fruta fresca, verduras y flores a " + "los vendedores locales.", + lang="es", + ), + ProfilePrompt( + "La historia de América Latina está marcada por", + " una gran diversidad cultural, fruto del encuentro entre pueblos " + "indígenas, europeos y africanos.", + lang="es", + ), + ProfilePrompt( + "Los avances de la medicina moderna permiten", + " tratar enfermedades que hace pocas décadas se consideraban " + "incurables, y prolongar la vida de millones de personas.", + lang="es", + ), + ), + "de": ( + ProfilePrompt( + "Der Schwarzwald ist bekannt für", + " seine dichten Wälder, tiefen Täler und traditionellen " + "Bauernhäuser, die jedes Jahr viele Wanderer anziehen.", + lang="de", + ), + ProfilePrompt( + "Jeden Morgen fährt der Zug", + " pünktlich um sieben Uhr vom Hauptbahnhof ab und bringt die " + "Pendler in die umliegenden Städte zur Arbeit.", + lang="de", + ), + ProfilePrompt( + "Die deutsche Sprache hat", + " viele lange zusammengesetzte Wörter, die Lernende oft " + "überraschen, aber einer klaren Logik folgen.", + lang="de", + ), + ProfilePrompt( + "In der modernen Industrie spielen Roboter", + " eine immer größere Rolle, weil sie schwere und gefährliche " + "Arbeiten schneller und sicherer erledigen können.", + lang="de", + ), + ), + "zh": ( + ProfilePrompt( + "长城是中国古代", + "伟大的防御工程，绵延数千公里，每年吸引大量游客前来参观。", + lang="zh", + ), + ProfilePrompt( + "每天早晨，公园里", + "有许多老人打太极拳、散步和下棋，气氛十分热闹。", + lang="zh", + ), + ProfilePrompt( + "现代科技的发展使得", + "人们的生活越来越方便，购物、学习和工作都可以在网上完成。", + lang="zh", + ), + ProfilePrompt( + "春天到了，山上的", + "花都开了，许多家庭趁着周末去郊外踏青赏花。", + lang="zh", + ), + ), + "ja": ( + ProfilePrompt( + "日本の四季は", + "それぞれ美しく、春には桜、秋には紅葉を楽しむために多くの人が旅行に出かけます。", + lang="ja", + ), + ProfilePrompt( + "毎朝、駅の周りには", + "通勤や通学の人々が行き交い、店が次々と開き始めます。", + lang="ja", + ), + ProfilePrompt( + "現代の技術の進歩により、", + "私たちの生活はますます便利になり、買い物も勉強も家にいながらできるようになりました。", + lang="ja", + ), + ProfilePrompt( + "図書館は静かな場所で、", + "学生たちが本を読んだり、勉強したりするのに最適です。", + lang="ja", + ), + ), + "ru": ( + ProfilePrompt( + "Зимой в Сибири", + " очень холодно, температура часто опускается ниже сорока " + "градусов, но местные жители привыкли к таким морозам.", + lang="ru", + ), + ProfilePrompt( + "Каждое утро студенты", + " спешат на занятия в университет, а вечером собираются в " + "библиотеке, чтобы готовиться к экзаменам.", + lang="ru", + ), + ProfilePrompt( + "Современная наука позволяет", + " лечить болезни, которые раньше считались неизлечимыми, и " + "продлевать жизнь миллионам людей.", + lang="ru", + ), + ProfilePrompt( + "Русская литература известна", + " во всём мире благодаря произведениям Толстого, Достоевского и " + "Чехова, которые переведены на десятки языков.", + lang="ru", + ), + ), + "ar": ( + ProfilePrompt( + "تشتهر مدينة القاهرة", + " بتاريخها العريق ومساجدها القديمة وأسواقها الشعبية التي يزورها " + "السياح من جميع أنحاء العالم.", + lang="ar", + ), + ProfilePrompt( + "في كل صباح يذهب الطلاب", + " إلى المدرسة مبكرين، ويقضون اليوم في تعلم القراءة والكتابة " "والعلوم.", + lang="ar", + ), + ProfilePrompt( + "يساعد التقدم العلمي الحديث", + " الأطباء على علاج أمراض كانت تعتبر مستعصية قبل عقود قليلة.", + lang="ar", + ), + ProfilePrompt( + "تعتبر اللغة العربية", + " من أقدم اللغات الحية في العالم، ويتحدث بها ملايين الناس في " "الوطن العربي وخارجه.", + lang="ar", + ), + ), + "code": ( + ProfilePrompt( + 'def is_prime(n):\n """Return True if n is a prime number."""\n', + " if n < 2:\n return False\n" + " for i in range(2, int(n ** 0.5) + 1):\n" + " if n % i == 0:\n return False\n" + " return True\n", + lang="code", + ), + ProfilePrompt( + 'def count_words(text):\n """Count occurrences of each word in text."""\n', + " counts = {}\n for word in text.split():\n" + " counts[word] = counts.get(word, 0) + 1\n" + " return counts\n", + lang="code", + ), + ProfilePrompt( + "def fibonacci(n):\n" ' """Return the first n Fibonacci numbers as a list."""\n', + " result = []\n a, b = 0, 1\n" + " for _ in range(n):\n result.append(a)\n" + " a, b = b, a + b\n return result\n", + lang="code", + ), + ProfilePrompt( + "// Return the largest number in the array.\n" "function findMax(numbers) {\n", + " let max = numbers[0];\n" + " for (const n of numbers) {\n" + " if (n > max) max = n;\n }\n" + " return max;\n}\n", + lang="code", + ), + ), +} + +# --------------------------------------------------------------------------- +# Chat prompts: realistic user turns (rendered through the tokenizer's chat +# template at run time) with a good assistant reply as reference. +# --------------------------------------------------------------------------- + +CHAT_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + "en": ( + ProfilePrompt( + "How do I keep basil alive indoors?", + "Keep basil in a warm spot with at least six hours of sunlight a " + "day, water it when the top of the soil feels dry, and pinch off " + "flower buds so the plant keeps producing leaves.", + ), + ProfilePrompt( + "What's a good way to remember people's names?", + "Repeat the name right after you hear it, use it once or twice in " + "conversation, and link it to something memorable about the " + "person, like their job or where you met.", + ), + ProfilePrompt( + "Explain why the sky is blue in simple terms.", + "Sunlight is made of many colors, and the air scatters blue light " + "more than the other colors because blue travels in shorter " + "waves. When you look up, that scattered blue light is what you " + "see.", + ), + ), + "fr": ( + ProfilePrompt( + "Comment préparer un bon café à la maison ?", + "Utilisez du café fraîchement moulu, une eau à environ 90 degrés " + "et un dosage d'une cuillère à soupe par tasse. Laissez infuser " + "quelques minutes avant de servir.", + lang="fr", + ), + ProfilePrompt( + "Quels sont les avantages de la lecture quotidienne ?", + "Lire chaque jour enrichit le vocabulaire, améliore la " + "concentration et réduit le stress. C'est aussi un excellent " + "moyen de découvrir de nouvelles idées.", + lang="fr", + ), + ProfilePrompt( + "Explique-moi simplement pourquoi les feuilles tombent en automne.", + "En automne, les jours raccourcissent et les arbres reçoivent " + "moins de lumière. Ils cessent de nourrir leurs feuilles, qui " + "changent de couleur puis tombent pour économiser l'énergie " + "pendant l'hiver.", + lang="fr", + ), + ), + "es": ( + ProfilePrompt( + "¿Cómo puedo mejorar mi memoria para estudiar?", + "Estudia en sesiones cortas y regulares, repasa lo aprendido al " + "día siguiente y explica el tema en voz alta con tus propias " + "palabras. Dormir bien también ayuda mucho a fijar los " + "recuerdos.", + lang="es", + ), + ProfilePrompt( + "¿Qué debo tener en cuenta al adoptar un gato?", + "Prepara un espacio tranquilo con comida, agua y un arenero " + "limpio. Dale tiempo para adaptarse, llévalo al veterinario para " + "sus vacunas y juega con él todos los días.", + lang="es", + ), + ProfilePrompt( + "Explícame de forma sencilla cómo funciona un molino de viento.", + "El viento empuja las aspas del molino y las hace girar. Ese giro " + "mueve un eje conectado a una máquina o a un generador, que " + "convierte el movimiento en trabajo útil o en electricidad.", + lang="es", + ), + ), + "de": ( + ProfilePrompt( + "Wie kann ich beim Einkaufen Geld sparen?", + "Schreiben Sie vorher eine Einkaufsliste und halten Sie sich " + "daran, vergleichen Sie Preise und kaufen Sie saisonale " + "Produkte. Große Packungen lohnen sich nur, wenn Sie alles " + "verbrauchen.", + lang="de", + ), + ProfilePrompt( + "Was ist ein guter Weg, eine neue Sprache zu lernen?", + "Üben Sie jeden Tag ein wenig, hören Sie Podcasts oder Musik in " + "der Sprache und sprechen Sie so früh wie möglich mit " + "Muttersprachlern. Regelmäßigkeit ist wichtiger als lange " + "Lerneinheiten.", + lang="de", + ), + ProfilePrompt( + "Erkläre mir einfach, warum es Ebbe und Flut gibt.", + "Der Mond zieht mit seiner Schwerkraft am Wasser der Ozeane. Auf " + "der dem Mond zugewandten Seite der Erde hebt sich das Wasser, " + "und während sich die Erde dreht, wandert dieser Wasserberg — so " + "entstehen Ebbe und Flut.", + lang="de", + ), + ), + "zh": ( + ProfilePrompt( + "怎样才能养成早起的习惯？", + "每天固定同一时间睡觉和起床，睡前少看手机，把闹钟放在离床远一点的地方。坚持两三个星期，身体就会慢慢适应新的作息。", + lang="zh", + ), + ProfilePrompt( + "第一次做饭应该注意什么？", + "先从简单的菜开始，提前准备好所有材料，注意用火安全，切菜时小心手指。做完后记得关闭燃气，慢慢积累经验就会越来越熟练。", + lang="zh", + ), + ProfilePrompt( + "请用简单的话解释为什么会下雨。", + "太阳把地面上的水晒热，水变成水蒸气升到天上，遇冷凝结成小水滴，聚在一起形成云。当水滴越来越重，云托不住它们时，就落下来变成雨。", + lang="zh", + ), + ), + "ja": ( + ProfilePrompt( + "朝型の生活に変えるにはどうすればいいですか？", + "毎日同じ時間に寝起きし、寝る前はスマートフォンを見ないようにしましょう。朝に日光を浴びると体内時計が整い、二、三週間続ければ自然に朝型になります。", + lang="ja", + ), + ProfilePrompt( + "初めての一人暮らしで気をつけることは何ですか?", + "毎月の家賃や食費など生活費の計画を立て、無理のない範囲で貯金をしましょう。防犯のために戸締まりを忘れず、近所のスーパーや病院の場所も早めに確認しておくと安心です。", + lang="ja", + ), + ProfilePrompt( + "虹がどうしてできるのか、簡単に説明してください。", + "雨上がりの空気中には小さな水滴がたくさん残っています。太陽の光がその水滴の中で曲がって反射すると、光が七つの色に分かれて見えます。これが虹です。", + lang="ja", + ), + ), + "ru": ( + ProfilePrompt( + "Как научиться рано вставать?", + "Ложитесь и вставайте в одно и то же время каждый день, не " + "смотрите в телефон перед сном и ставьте будильник подальше от " + "кровати. Через пару недель организм привыкнет к новому режиму.", + lang="ru", + ), + ProfilePrompt( + "Что почитать, чтобы полюбить чтение?", + "Начните с коротких книг на темы, которые вам действительно " + "интересны, — детективы, приключения или научно-популярные " + "рассказы. Главное — читать понемногу каждый день и не " + "заставлять себя дочитывать скучное.", + lang="ru", + ), + ProfilePrompt( + "Объясни простыми словами, почему летом жарко, а зимой холодно.", + "Земля вращается вокруг Солнца с наклонённой осью. Летом наше " + "полушарие наклонено к Солнцу, лучи падают прямее и сильнее " + "нагревают землю. Зимой оно отклонено от Солнца, лучи идут под " + "углом и греют слабее.", + lang="ru", + ), + ), + "ar": ( + ProfilePrompt( + "كيف أنظم وقتي أثناء الدراسة؟", + "قسّم يومك إلى فترات قصيرة للدراسة مع فترات راحة منتظمة، وابدأ " + "بأصعب المواد عندما يكون ذهنك صافياً. اكتب قائمة بالمهام كل صباح " + "والتزم بها قدر الإمكان.", + lang="ar", + ), + ProfilePrompt( + "ما هي فوائد المشي اليومي؟", + "المشي كل يوم يقوي القلب والعضلات ويساعد على تخفيف التوتر " + "وتحسين المزاج. كما أنه يساعد على النوم بشكل أفضل ولا يحتاج إلى " + "أي معدات خاصة.", + lang="ar", + ), + ProfilePrompt( + "اشرح لي ببساطة كيف تصنع النحلة العسل.", + "تجمع النحلة رحيق الأزهار وتخزنه في معدة خاصة، ثم تعود إلى " + "الخلية وتسلمه لنحلات أخرى تضيف إليه مواد تحوله إلى عسل. بعد ذلك " + "يوضع العسل في الأقراص الشمعية ويجفف بتحريك الأجنحة حتى ينضج.", + lang="ar", + ), + ), +} + +# --------------------------------------------------------------------------- +# Task prompts. +# --------------------------------------------------------------------------- + +SUMMARIZATION_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + "en": ( + ProfilePrompt( + "The city council voted on Tuesday to approve funding for a new " + "public library in the downtown district. The project, which has " + "been debated for over two years, will cost an estimated twelve " + "million dollars and is expected to open in the spring of 2028. " + "Supporters argued that the current library, built in 1962, is " + "too small and lacks modern facilities. Opponents raised " + "concerns about the cost and the loss of a parking lot at the " + "proposed site. The mayor said the new building would include " + "community meeting rooms, a children's wing, and free computer " + "access for residents.", + "The city council approved a twelve million dollar downtown " + "library, expected to open in spring 2028, replacing the " + "outdated 1962 building despite concerns over cost and parking.", + ), + ProfilePrompt( + "Researchers at a European university have published a study " + "showing that regular walking can significantly improve sleep " + "quality in adults over sixty. The study followed four hundred " + "participants for one year, half of whom walked for thirty " + "minutes a day while the other half kept their usual habits. " + "Those in the walking group fell asleep faster, woke less often " + "during the night, and reported feeling more rested in the " + "morning. The researchers noted that the benefits appeared " + "within the first two months and lasted for the rest of the " + "study.", + "A year-long study of four hundred older adults found that " + "walking thirty minutes daily improved sleep quality within two " + "months, helping participants fall asleep faster and wake less " + "often.", + ), + ProfilePrompt( + "A severe storm swept through the coastal region on Friday " + "night, leaving thousands of homes without electricity and " + "forcing the closure of the main highway. Emergency crews worked " + "through the weekend to clear fallen trees and restore power " + "lines. Officials said no serious injuries were reported, though " + "several boats were damaged in the harbor. Schools in the area " + "remained closed on Monday while cleanup continued, and " + "residents were advised to avoid the beachfront until inspectors " + "declared it safe.", + "A Friday night storm cut power to thousands of coastal homes " + "and closed the main highway; crews restored services over the " + "weekend with no serious injuries reported.", + ), + ), +} + +INSTRUCTION_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + "en": ( + ProfilePrompt( + "List three things to pack for a day hike.", + "Water, snacks, and a map of the trail.", + ), + ProfilePrompt( + "Write one sentence describing what a lighthouse does.", + "A lighthouse shines a bright light to guide ships safely along " "the coast at night.", + ), + ProfilePrompt( + "Name the four seasons of the year.", + "Spring, summer, autumn, and winter.", + ), + ), +} + +# Pretrained-only seq2seq models (T5, BART) were trained to fill masked spans, +# not to follow instructions; feed them their native denoising format. +DENOISE_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + # ONE sentinel per "t5" prompt: the whole special-stripped output is the + # fill, spliced back by the runner so both ratio sides are full sentences + # (bare span fragments judge in the thousands). + "t5": ( + ProfilePrompt( + "The children in the park until the sun went down.", + "The children played happily in the park until the sun went down.", + ), + ProfilePrompt( + "Every morning she drinks a cup of and reads the newspaper.", + "Every morning she drinks a cup of coffee and reads the newspaper.", + ), + ProfilePrompt( + "The old bridge across the was built many years ago.", + "The old bridge across the river was built many years ago.", + ), + ), + "mask": ( + ProfilePrompt( + "The children played in the park until the sun went down.", + "The children played happily in the park until the sun went down.", + ), + ProfilePrompt( + "Every morning she drinks a cup of and reads the newspaper.", + "Every morning she drinks a cup of coffee and reads the newspaper.", + ), + ProfilePrompt( + "The old bridge across the was built many years ago.", + "The old bridge across the river was built many years ago.", + ), + ), +} + +# References for the synthetic caption images built by the text-quality +# benchmark (index-aligned with _build_caption_test_images). +CAPTION_REFERENCES: tuple[str, ...] = ( + "The image shows a blue rectangle and a green oval on a white background.", + "The image shows a large yellow circle on a black background.", + "The image shows a dark green rectangle and an orange oval on a light " "blue background.", +) + +# --------------------------------------------------------------------------- +# Per-kind generation and judging knobs. +# --------------------------------------------------------------------------- + +MAX_NEW_TOKENS_BY_KIND: dict[str, int] = { + "continuation": 50, + "chat": 64, + "task:instruction": 48, + "task:translation": 48, + "task:summarization": 48, + "task:denoise": 24, + "caption": 50, +} + +# Chat prompts arrive pre-templated (the template supplies its own BOS); +# everything else follows the adapter default. +PREPEND_BOS_BY_KIND: dict[str, Optional[bool]] = { + "chat": False, +} + +# Bake-off-measured scoring anchors; scripts/text_quality_judge_bakeoff.py +# regenerates them (it prints these names verbatim; last run 2026-08-20, full +# 13-domain corpus). R_FAIL = geo-mean of per-language MEDIAN corrupted/fluent +# ratios — a low percentile degenerates below 1 in weak-separation languages. +# R_GOOD = the paraphrase noise floor; score(R_GOOD) is the pass line. +JUDGE_R_FAIL = 18.1 +JUDGE_R_GOOD = 3.74 + + +# Known scale properties (measured during the 2026-08 output audit): +# - Saturation: any ratio <= 1 scores 100 — "at least reference-fluent" is the +# top of the scale, with no resolution above it. +# - Judge family-favoring: the pinned Qwen judge rates Qwen-family models a +# few points friendlier than others; watch Qwen entries in campaign reruns. + + +def p4_pass_threshold() -> float: + """The P4 pass line, derived from the bake-off noise floor. The registry + floor imports this so [floor, pass) can never silently diverge again.""" + return round(100.0 - 100.0 * math.log(JUDGE_R_GOOD) / math.log(JUDGE_R_FAIL), 1) + + +# Registry scale marker for phase4_score. Absent = v1 (unpinned GPT-2, +# 135-10*ln(ppl), pass 85); 2 = pinned-judge reference-ratio scale (pass 56). +# The column mixes populations until the backlog is re-run, so every P4 write +# stamps the scale it was measured on. +P4_SCORING_VERSION = 2 + +# Task output is generated greedily — that is how users run translators and +# summarizers, and it removes sampling variance from a single-sample score. +# Open-ended kinds keep sampling (greedy makes base models loop). +TEMPERATURE_BY_KIND: dict[str, float] = { + "continuation": 0.7, + "chat": 0.7, + "task:instruction": 0.0, + "task:translation": 0.0, + "task:summarization": 0.0, + "task:denoise": 0.0, + "caption": 0.0, +} + +# Kinds whose judge PPL is conditioned on the prompt — the relevance signal: +# unconditioned, fluent-but-off-topic or hallucinated output judges as well as +# a real answer. Translation stays unconditioned (cross-lingual conditioning +# is noisy; the language check covers it); caption's source is an image the +# judge cannot read. +JUDGE_CONTEXT_KINDS = frozenset( + {"continuation", "chat", "task:instruction", "task:summarization", "task:denoise"} +) + +# T5-family checkpoints expect a natural-language task prefix on the source. +T5_PREFIX_ARCHITECTURES = frozenset( + { + "T5ForConditionalGeneration", + "T5WithLMHeadModel", + "MT5ForConditionalGeneration", + "LongT5ForConditionalGeneration", + "SwitchTransformersForConditionalGeneration", + "UMT5ForConditionalGeneration", + } +) + +# Full NLLB (flores-200) codes for covered languages; transformers 5.x +# NllbTokenizer resolves them only via convert_tokens_to_ids. +NLLB_CODES: dict[str, str] = { + "en": "eng_Latn", + "fr": "fra_Latn", + "es": "spa_Latn", + "de": "deu_Latn", + "it": "ita_Latn", + "nl": "nld_Latn", + "pt": "por_Latn", + "ru": "rus_Cyrl", + "zh": "zho_Hans", + "ja": "jpn_Jpan", + "ar": "arb_Arab", + "hi": "hin_Deva", + "ro": "ron_Latn", +} + +# ISO 639-3 equivalents for NLLB-style language codes ("deu_Latn"). +LANG_ISO3: dict[str, str] = { + "en": "eng", + "fr": "fra", + "es": "spa", + "de": "deu", + "it": "ita", + "nl": "nld", + "pt": "por", + "ru": "rus", + "zh": "zho", + "ja": "jpn", + "ar": "ara", + "hi": "hin", + "ro": "ron", +} + +LANG_NAMES: dict[str, str] = { + "en": "English", + "fr": "French", + "es": "Spanish", + "de": "German", + "it": "Italian", + "nl": "Dutch", + "pt": "Portuguese", + "ru": "Russian", + "zh": "Chinese", + "ja": "Japanese", + "ar": "Arabic", + "hi": "Hindi", + "ro": "Romanian", +} + +# --------------------------------------------------------------------------- +# Curation: architecture rules and per-model overrides. +# --------------------------------------------------------------------------- + +# Architectures whose task is unambiguous. T5/BART/Switch and Falcon/MPT are +# deliberately absent: their task depends on the checkpoint, so they resolve +# through overrides or fetched Hub signals. +ARCHITECTURE_PROFILE_KINDS: dict[str, str] = { + "MarianMTModel": "task:translation", + "M2M100ForConditionalGeneration": "task:translation", + "PegasusForConditionalGeneration": "task:summarization", + "LEDForConditionalGeneration": "task:summarization", + "BlenderbotForConditionalGeneration": "chat", + "BlenderbotSmallForConditionalGeneration": "chat", + "GPTBigCodeForCausalLM": "continuation@code", + "CodeGenForCausalLM": "continuation@code", +} + +# An unlabelled seq2seq model cannot continue text; its pretraining task is the +# only prompt it understands. +SEQ2SEQ_FALLBACK_KIND = "task:denoise" + +MODEL_PROFILE_OVERRIDES: dict[str, str] = { + # T5 v1.0 checkpoints were multitask-trained with task prefixes; the WMT + # en-de pair is their canonical supervised task. + "google-t5/t5-small": "task:translation@en-de", + "google-t5/t5-base": "task:translation@en-de", + "google-t5/t5-large": "task:translation@en-de", + "t5-small": "task:translation@en-de", + "t5-base": "task:translation@en-de", + "t5-large": "task:translation@en-de", + # mt0 is instruction-tuned MT5 (Hub mis-tags it text-generation). + "bigscience/mt0-small": "task:instruction", + "bigscience/mt0-base": "task:instruction", + "bigscience/mt0-large": "task:instruction", + # Pretrained-only checkpoints: denoising is their only language. + "google/long-t5-tglobal-base": "task:denoise", + "google/long-t5-local-base": "task:denoise", + # Base model that ships a chat template (Hub tags it conversational). + "Qwen/Qwen2.5-0.5B": "continuation", + # Task depends on the checkpoint for BART (arch rule deliberately absent); + # without a scraped registry profile these canonical ones need curation. + "facebook/bart-large-cnn": "task:summarization", + "facebook/bart-large-xsum": "task:summarization", + # MBart has no arch rule (base checkpoints are denoising pretrains, task + # varies by fine-tune) — the canonical translators are curated instead. + "facebook/mbart-large-50-many-to-many-mmt": "task:translation@en-de", + "facebook/mbart-large-50-one-to-many-mmt": "task:translation@en-de", + "facebook/mbart-large-50-many-to-one-mmt": "task:translation@de-en", + # Indic-language denoiser; English denoise prompts measure the wrong + # thing, so this skips until Indic coverage exists. + "ai4bharat/IndicBART": "task:denoise@hi", + # Code checkpoints on general-purpose architectures. + "Salesforce/codegen-350M-mono": "continuation@code", + "bigcode/starcoderbase-1b": "continuation@code", + "replit/replit-code-v1-3b": "continuation@code", +} + +# --------------------------------------------------------------------------- +# Hub-signal distillation and profile resolution. +# --------------------------------------------------------------------------- + +# Full ISO 639-1 code set, used to pick language codes out of unstructured Hub +# tag lists. Complete on purpose: a dropped code silently reroutes a model to +# English prompts. +ISO_639_1 = frozenset( + "aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce " + "ch co cr cs cu cv cy da de dv dz ee el en eo es et eu fa ff fi fj fo fr " + "fy ga gd gl gn gu gv ha he hi ho hr ht hu hy hz ia id ie ig ii ik io is " + "it iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln " + "lo lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv " + "ny oc oj om or os pa pi pl ps pt qu rm rn ro ru rw sa sc sd se sg si sk " + "sl sm sn so sq sr ss st su sv sw ta te tg th ti tk tl tn to tr ts tt tw " + "ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu".split() +) + +_PIPELINE_TAG_KINDS: dict[str, str] = { + "translation": "task:translation", + "summarization": "task:summarization", + "text2text-generation": "task:denoise", + "text-generation": "continuation", + "image-text-to-text": "caption", + "image-to-text": "caption", +} + +# Hub tags that mark code models (`conversational` is deliberately NOT mapped +# to chat: HF adds it for any repo shipping a chat template, base models +# included). +_CODE_TAGS = frozenset({"code", "code-generation", "coding"}) + + +@dataclass(frozen=True) +class HFSignals: + """Distilled Hub metadata for one model, as fetched by the scraper.""" + + pipeline_tag: Optional[str] = None + languages: tuple[str, ...] = () + tags: tuple[str, ...] = () + + +def extract_languages(card_data_language: object, tags: object) -> tuple[str, ...]: + """Normalize cardData.language (str or list) plus tag-list ISO codes, noise dropped.""" + langs: list[str] = [] + if isinstance(card_data_language, str): + langs.append(card_data_language.lower()) + elif isinstance(card_data_language, (list, tuple)): + langs.extend(str(item).lower() for item in card_data_language) + if isinstance(tags, (list, tuple)): + langs.extend(str(t).lower() for t in tags) + seen: list[str] = [] + for lang in langs: + # The ISO gate alone filters framework/task tag noise ("pytorch", + # "marian", "multilingual" are not ISO 639-1 codes). + if lang in ISO_639_1 and lang not in seen: + seen.append(lang) + if len(seen) >= 8: + break + return tuple(seen) + + +def _marian_pair_from_model_id(model_id: str) -> Optional[tuple[str, str]]: + """Parse opus-mt-{src}-{tgt} from the id; Helsinki-NLP language tags are unordered.""" + name = model_id.rsplit("/", 1)[-1].lower() + if not name.startswith("opus-mt-"): + return None + parts = name[len("opus-mt-") :].split("-") + if len(parts) == 2 and all(len(p) in (2, 3) for p in parts): + return parts[0], parts[1] + return None + + +_CHAT_ID_MARKERS = ("instruct", "-chat", "_chat") + + +def _id_says_chat(model_id: str) -> bool: + """Instruction-tuned checkpoints are used through their chat template; the + id is the only reliable signal (HF's `conversational` tag also covers base + models, and no architecture distinguishes tuned from base).""" + name = model_id.rsplit("/", 1)[-1].lower() + if name.endswith("-it") or "-it-" in name: + return True + return any(marker in name for marker in _CHAT_ID_MARKERS) + + +def _first_covered_language(languages: tuple[str, ...], table: dict) -> Optional[str]: + for lang in languages: + if lang in table: + return lang + return None + + +def profile_from_hf_signals( + model_id: str, + architecture_id: str, + signals: HFSignals, +) -> Optional[ProfileSpec]: + """Distill fetched Hub metadata into a profile, or None when it says nothing.""" + tags_lower = {t.lower() for t in signals.tags} + if tags_lower & _CODE_TAGS: + return ProfileSpec("continuation", "code") + kind = _PIPELINE_TAG_KINDS.get((signals.pipeline_tag or "").lower()) + if kind is None: + return None + if kind == "task:translation": + pair = _marian_pair_from_model_id(model_id) + if pair is not None: + return ProfileSpec(kind, lang=pair[1], src=pair[0]) + non_en = [lang for lang in signals.languages if lang != "en"] + if "en" in signals.languages and non_en: + return ProfileSpec(kind, lang=non_en[0], src="en") + # Direction unknowable from tags alone (tag lists are unordered); + # fall through rather than guess a reversed or identity pair. + return None + lang = _first_covered_language(signals.languages, CONTINUATION_PROMPTS) or "en" + if kind == "continuation": + return ProfileSpec(kind, lang) + return ProfileSpec(kind) + + +def resolve_profile( + model_id: str, + architecture_id: Optional[str], + registry_profile: Optional[str] = None, + signals: Optional[HFSignals] = None, +) -> ProfileSpec: + """Resolve a model's profile: override > architecture rule > live signals > + stored registry value > default (seq2seq falls back to denoising).""" + override = MODEL_PROFILE_OVERRIDES.get(model_id) + if override is not None: + return ProfileSpec.parse(override) + + # Instruction-tuned ids get the chat profile (the runtime downgrades to + # continuation when no chat template actually exists). Checked before the + # signals layer: the `conversational` tag is deliberately not mapped. + if _id_says_chat(model_id) and ARCHITECTURE_PROFILE_KINDS.get(architecture_id or "") is None: + # The heuristic fixes only the KIND; a stored chat profile keeps its + # language or writeback would flatten curation to @en. + if registry_profile: + try: + stored = ProfileSpec.parse(registry_profile) + if stored.kind == "chat": + return stored + except ValueError: + pass + lang = "en" + if signals is not None: + lang = _first_covered_language(signals.languages, CHAT_PROMPTS) or "en" + return ProfileSpec("chat", lang) + + arch_kind = ARCHITECTURE_PROFILE_KINDS.get(architecture_id or "") + if arch_kind is not None: + arch_spec = ProfileSpec.parse(arch_kind) + # The arch rule fixes only the KIND; a stored same-kind profile keeps + # its language so curation survives the writeback round-trip. + if registry_profile: + try: + stored = ProfileSpec.parse(registry_profile) + if stored.kind == arch_spec.kind: + if arch_spec.kind != "task:translation": + return stored + except ValueError: + pass + if arch_spec.kind == "task:translation": + pair = _marian_pair_from_model_id(model_id) + if pair is not None: + return ProfileSpec(arch_spec.kind, lang=pair[1], src=pair[0]) + if signals is not None: + from_signals = profile_from_hf_signals(model_id, architecture_id or "", signals) + if from_signals is not None and from_signals.kind == "task:translation": + return from_signals + if registry_profile: + try: + stored = ProfileSpec.parse(registry_profile) + if stored.kind == "task:translation": + return stored + except ValueError: + pass + return ProfileSpec(arch_spec.kind, lang="de", src="en") + return arch_spec + + if signals is not None: + from_signals = profile_from_hf_signals(model_id, architecture_id or "", signals) + if from_signals is not None: + return from_signals + + if registry_profile: + try: + return ProfileSpec.parse(registry_profile) + except ValueError: + pass + + try: + from transformer_lens.utilities.architectures import classify_architecture + + if architecture_id and classify_architecture(architecture_id) == "seq2seq": + return ProfileSpec.parse(SEQ2SEQ_FALLBACK_KIND) + except ImportError: # pragma: no cover - torch-free scraper environments + pass + return DEFAULT_PROFILE + + +def prompts_for( + spec: ProfileSpec, denoise_style: str = "t5" +) -> Optional[tuple[ProfilePrompt, ...]]: + """Prompt set for a profile, or None when coverage is missing (caller skips + with a file-an-issue message naming the gap).""" + if spec.kind == "continuation": + return CONTINUATION_PROMPTS.get(spec.lang) + if spec.kind == "chat": + return CHAT_PROMPTS.get(spec.lang) + if spec.kind == "task:instruction": + return INSTRUCTION_PROMPTS.get(spec.lang) + if spec.kind == "task:summarization": + return SUMMARIZATION_PROMPTS.get(spec.lang) + if spec.kind == "task:denoise": + # Denoise prompts are English-only; a non-en denoise profile + # (IndicBART) is a coverage gap, not a zero. + if spec.lang not in ("en", ""): + return None + return DENOISE_PROMPTS.get(denoise_style) + if spec.kind == "task:translation": + src = spec.src or "en" + if src not in PIVOT_SENTENCES or spec.lang not in PIVOT_SENTENCES: + return None + return tuple( + ProfilePrompt(prompt=s, reference=t, lang=spec.lang) + for s, t in zip(PIVOT_SENTENCES[src], PIVOT_SENTENCES[spec.lang]) + ) + if spec.kind == "caption": + return tuple(ProfilePrompt(prompt="", reference=ref) for ref in CAPTION_REFERENCES) + return None diff --git a/transformer_lens/model_bridge/bridge.py b/transformer_lens/model_bridge/bridge.py index a54f1206c2..670e7ef139 100644 --- a/transformer_lens/model_bridge/bridge.py +++ b/transformer_lens/model_bridge/bridge.py @@ -3268,6 +3268,21 @@ def _resolve_stopping_criteria( return criteria if len(criteria) > 0 else None + def _encdec_ngram_processor(self) -> Optional[Any]: + """generation_config.no_repeat_ngram_size as transformers' own + processor, or None. HF applies it by default; parity for models whose + greedy decode needs it to escape token attractors.""" + size = getattr( + getattr(self.original_model, "generation_config", None), + "no_repeat_ngram_size", + None, + ) + if not size: + return None + from transformers.generation.logits_process import NoRepeatNGramLogitsProcessor + + return NoRepeatNGramLogitsProcessor(size) + def _generate_tokens( self, current_tokens: torch.Tensor, @@ -3300,6 +3315,9 @@ def _generate_tokens( verbose: bool, stopping_criteria_list: Optional[Any] = None, initial_attention_mask: Optional[torch.Tensor] = None, + min_decoder_length: Optional[int] = None, + ngram_processor: Optional[Any] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, ) -> Generator[Tuple[torch.Tensor, torch.Tensor, bool], None, None]: """Core generation loop. Yields (sampled_tokens, final_logits, all_finished) per step. @@ -3344,10 +3362,17 @@ def _generate_tokens( for gen_step_idx in tqdm.tqdm(range(max_new_tokens), disable=not verbose): with torch.no_grad(): if is_encoder_decoder: + assert encoder_input is not None + encdec_kwargs: Dict[str, Any] = {} + if encoder_attention_mask is not None: + encdec_kwargs["attention_mask"] = encoder_attention_mask.to( + encoder_input.device + ) logits = self( encoder_input, return_type="logits", decoder_input=decoder_tokens, + **encdec_kwargs, ) else: forward_kwargs: Dict[str, Any] = {} @@ -3496,6 +3521,23 @@ def _generate_tokens( if _generate_from_embeds and generated_token_ids else None ) + # transformers' own NoRepeatNGramLogitsProcessor, honoring + # generation_config (bart-large-cnn pins 3; without it greedy + # decoding falls into a BOS attractor and emits nothing). + if ngram_processor is not None and decoder_tokens is not None: + final_logits = ngram_processor(decoder_tokens, final_logits) + # HF's generate() suppresses EOS below generation_config.min_length + # (bart-large-cnn pins 56); without this the loop can EOS on step + # one and emit an empty summary. + if ( + min_decoder_length is not None + and is_encoder_decoder + and decoder_tokens is not None + and decoder_tokens.shape[1] < min_decoder_length + and stop_tokens + ): + final_logits = final_logits.clone() + final_logits[:, stop_tokens] = float("-inf") if do_sample: sampled_tokens = utils.sample_logits( final_logits, @@ -3631,6 +3673,7 @@ def generate( stop_strings: Optional[Union[str, List[str]]] = None, stopping_criteria: Optional[Any] = None, attention_mask: Optional[torch.Tensor] = None, + forced_bos_token_id: Optional[int] = None, **multimodal_kwargs, ) -> ( str @@ -3721,6 +3764,10 @@ def generate( paths the mask is forwarded to the model as-is rather than grown per step, which is what processors emitting one alongside ``pixel_values`` expect. + forced_bos_token_id: Optional token id seeded as the first decoder token + after ``decoder_start`` on encoder-decoder models. Multilingual + translators (M2M100/MBart/NLLB) select their target language this way. + Raises ValueError on decoder-only models. Returns: Generated sequence as string, list of strings, or tensor depending on input type and return_type. @@ -3751,22 +3798,44 @@ def generate( use_past_kv_cache = self._resolve_generation_caching(use_past_kv_cache, _is_batched_list) _generate_from_embeds = False + _encdec_early = hasattr(self.original_model, "config") and getattr( + self.original_model.config, "is_encoder_decoder", False + ) if isinstance(input, str): - input_tokens = self.to_tokens( - input, prepend_bos=prepend_bos, move_to_device=True, truncate=False - ) + if _encdec_early: + # Deliberate divergence: prepend_bos is IGNORED for enc-dec + # string/list input. Encoder input follows the tokenizer's own + # recipe (lang token + trailing ); to_tokens' decoder-style + # BOS policy corrupts it — m2m100 degenerates to loops. + input_tokens = self.tokenizer(input, return_tensors="pt")["input_ids"].to( + self.cfg.device + ) + else: + input_tokens = self.to_tokens( + input, prepend_bos=prepend_bos, move_to_device=True, truncate=False + ) input_type = "str" elif isinstance(input, list): - # Force left-padding for batched generation so real tokens are - # flush-right and logits[:, -1, :] is always the last real token. - if _is_batched_list: - _orig_padding_side = self.tokenizer.padding_side - self.tokenizer.padding_side = "left" - input_tokens = self.to_tokens( - input, prepend_bos=prepend_bos, move_to_device=True, truncate=False - ) - if _is_batched_list: - self.tokenizer.padding_side = _orig_padding_side + if _encdec_early: + # Same native-recipe rule as the str branch: to_tokens' BOS + # policy corrupts encoder inputs (stray , dropped ). + # Keep the tokenizer's mask too — unequal rows otherwise + # attend over pads in the encoder. + _enc_batch = self.tokenizer(input, return_tensors="pt", padding=True) + input_tokens = _enc_batch["input_ids"].to(self.cfg.device) + if attention_mask is None and "attention_mask" in _enc_batch: + attention_mask = _enc_batch["attention_mask"].to(self.cfg.device) + else: + # Force left-padding for batched generation so real tokens are + # flush-right and logits[:, -1, :] is always the last real token. + if _is_batched_list: + _orig_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = "left" + input_tokens = self.to_tokens( + input, prepend_bos=prepend_bos, move_to_device=True, truncate=False + ) + if _is_batched_list: + self.tokenizer.padding_side = _orig_padding_side input_type = "list" elif isinstance(input, torch.Tensor) and input.is_floating_point(): # inputs_embeds: pre-computed embeddings (e.g., from multimodal models) @@ -3885,6 +3954,18 @@ def generate( is_encoder_decoder = hasattr(self.original_model, "config") and getattr( self.original_model.config, "is_encoder_decoder", False ) + if forced_bos_token_id is None and is_encoder_decoder: + # HF's generate() applies generation_config defaults; bart-large-cnn + # pins forced_bos_token_id=0 there and degrades without it. + forced_bos_token_id = getattr( + getattr(self.original_model, "generation_config", None), + "forced_bos_token_id", + None, + ) + if forced_bos_token_id is not None and not is_encoder_decoder: + # Raise before any state mutation (_capture_hf_cache) and before + # the stateful hf_generate early-return would drop the kwarg. + raise ValueError("forced_bos_token_id is only meaningful for encoder-decoder models") # return_cache recomputes run_with_cache on the generated output (see issue #697). # That is well-defined only for single-sequence, decoder-only text generation, so @@ -4053,6 +4134,16 @@ def generate( dtype=input_tokens.dtype, device=self.cfg.device, ) + if forced_bos_token_id is not None: + # Multilingual seq2seq (M2M100/MBart/NLLB) selects the target + # language via the first decoder token after decoder_start. + forced = torch.full( + (batch_size, 1), + forced_bos_token_id, + dtype=input_tokens.dtype, + device=self.cfg.device, + ) + decoder_tokens = torch.cat([decoder_tokens, forced], dim=1) try: for sampled_tokens, final_logits, all_finished in self._generate_tokens( @@ -4085,6 +4176,17 @@ def generate( verbose=verbose, stopping_criteria_list=stopping_criteria_list, initial_attention_mask=initial_attention_mask, + min_decoder_length=( + getattr( + getattr(self.original_model, "generation_config", None), + "min_length", + None, + ) + if is_encoder_decoder + else None + ), + ngram_processor=(self._encdec_ngram_processor() if is_encoder_decoder else None), + encoder_attention_mask=(attention_mask if is_encoder_decoder else None), ): sampled_tokens_list.append(sampled_tokens.unsqueeze(1)) if logits_seq_list is not None: @@ -4100,7 +4202,8 @@ def generate( sampled_tokens = torch.cat(sampled_tokens_list, dim=1) if is_encoder_decoder: # Reconstruct full decoder sequence: start token + generated tokens - output_tokens = torch.cat([decoder_tokens[:, :1], sampled_tokens], dim=1) + decoder_seed_len = 2 if forced_bos_token_id is not None else 1 + output_tokens = torch.cat([decoder_tokens[:, :decoder_seed_len], sampled_tokens], dim=1) elif _generate_from_embeds: # For inputs_embeds, we only have the generated token IDs (no input token IDs) output_tokens = sampled_tokens @@ -4297,22 +4400,38 @@ def generate_stream( _is_batched_list = isinstance(input, list) and len(input) > 1 use_past_kv_cache = self._resolve_generation_caching(use_past_kv_cache, _is_batched_list) + _encdec_early = hasattr(self.original_model, "config") and getattr( + self.original_model.config, "is_encoder_decoder", False + ) if isinstance(input, str): - input_tokens = self.to_tokens( - input, prepend_bos=prepend_bos, move_to_device=True, truncate=False - ) + if _encdec_early: + # Native recipe: to_tokens' BOS policy corrupts encoder inputs. + input_tokens = self.tokenizer(input, return_tensors="pt")["input_ids"].to( + self.cfg.device + ) + else: + input_tokens = self.to_tokens( + input, prepend_bos=prepend_bos, move_to_device=True, truncate=False + ) input_type = "str" elif isinstance(input, list): - if _is_batched_list: + if _encdec_early: + input_tokens = self.tokenizer(input, return_tensors="pt", padding=True)[ + "input_ids" + ].to(self.cfg.device) + elif _is_batched_list: _orig_ps = self.tokenizer.padding_side self.tokenizer.padding_side = "left" - try: + try: + input_tokens = self.to_tokens( + input, prepend_bos=prepend_bos, move_to_device=True, truncate=False + ) + finally: + self.tokenizer.padding_side = _orig_ps + else: input_tokens = self.to_tokens( input, prepend_bos=prepend_bos, move_to_device=True, truncate=False ) - finally: - if _is_batched_list: - self.tokenizer.padding_side = _orig_ps input_type = "list" else: input_tokens = input.to(self.cfg.device) diff --git a/transformer_lens/tools/model_registry/AGENTS.md b/transformer_lens/tools/model_registry/AGENTS.md index 8198be7013..e3e2c1e4c1 100644 --- a/transformer_lens/tools/model_registry/AGENTS.md +++ b/transformer_lens/tools/model_registry/AGENTS.md @@ -114,7 +114,7 @@ Never edit manually. | 1 | Core forward correctness vs HuggingFace logits | | 2 | Hook firing + gradient flow | | 3 | Weight processing (compatibility mode, fold/centre) | -| 4 | Text-generation quality | +| 4 | Text-generation quality (per-model prompt profile, scored by a pinned multilingual judge) | | 7 | Multimodal (vision/text alignment) — only Llava / Gemma3-multimodal | | 8 | Audio — Hubert (waveform) and AST (spectrogram) | | 9 | Vision — ViT/DeiT pixel forward, hook/cache firing, representation stability, classification decode | @@ -132,24 +132,26 @@ SSM / recurrent families and the hybrids (Mamba-1/2, gated-delta-net, NemotronH, | 1 | **100%** | — | `STATUS_FAILED` | | 2 | 75% | `logits_equivalence`, `loss_equivalence` | `STATUS_FAILED` | | 3 | 75% | `logits_equivalence`, `loss_equivalence` | `STATUS_FAILED` | -| 4 | 50% | — | **Non-gating.** Below 50% adds `"low text quality"` to the registry `note`; never causes `STATUS_FAILED`. | +| 4 | 54.5% — the measured pass line `p4_pass_threshold()` (score of the bake-off noise floor `JUDGE_R_GOOD`) | — | **Non-gating.** Below the line adds `"text quality poor (P4=…)"` to the registry `note`; never causes `STATUS_FAILED`. | | 7 | 75% | `multimodal_forward` | `STATUS_FAILED`. NULL score (processor unavailable) also fails. | | 8 | 75% | `audio_forward` | `STATUS_FAILED`. NULL score also fails. | | 9 | 75% | `vision_forward`, `vision_cache` | `STATUS_FAILED`. NULL score also fails. | -Phase 4 is intentionally lenient — source ([`verify_models.py:554`](verify_models.py)) calls it *"a quality metric, not a correctness check."* The 50% bar asks "is the text coherent at all?" not "is this adapter clean?" +P4 prompts each model with its resolved **prompt profile** — chat template, translation, code, own-language continuation, or another task kind — via `resolve_profile()` in [`benchmarks/text_quality_profiles.py`](../../benchmarks/text_quality_profiles.py) (precedence: per-model override > architecture rule > live HF Hub signals > stored registry value > default). The resolved profile is cached sparsely on the entry as `prompt_profile` (key omitted when it's just the default). Each generation is scored against a known-good reference by one pinned multilingual judge via the perplexity ratio `PPL(generated)/PPL(reference)`, which cancels the judge's per-language handicap; the pass/fail constants are measured, not hand-picked, in [`benchmarks/text_quality.py`](../../benchmarks/text_quality.py). + +Phase 4 is a quality metric, not a correctness check. Its floor is not hand-picked: it equals the benchmark pass line, derived from the judge bake-off's fluent-vs-fluent noise floor (`p4_pass_threshold()` in [`benchmarks/text_quality_profiles.py`](../../benchmarks/text_quality_profiles.py)), so the registry note and the benchmark verdict can never disagree. **For adapter authors:** a `STATUS_VERIFIED` entry with P4 well below 100% on a small parity-test model can still indicate a real bug the system doesn't gate on (e.g. missing `preprocess_weights` fold). Investigate manually even when VERIFIED. **Reading the result:** - `status==1` + `note="Full verification completed"` → all gates passed, no quality flag. Good. -- `status==1` + `note` mentions `"low text quality"` → P4 < 50%; investigate. +- `status==1` + `note` mentions `"text quality poor"` → P4 below the pass line; investigate (`scripts/phase4_review.py` orders the candidates and separates old-scale scores). - `status==1` + P4 < 100% on a small model, no quality flag → potential weight-fold/tokenizer bug; investigate. - `status==3` (FAILED) → `note` carries the failure reason; debug from there. - `status==4` (PROVISIONAL) → structural-only pass via `--no-hf-reference`; Phase 1 was never numerically compared to HF, so it does **not** count as verified (`note` is prefixed `Structural only (no HF reference)`). Re-run without the flag for a real verification. -P1/P3 failures: [supported_architectures/AGENTS.md §When to override preprocess_weights](../../model_bridge/supported_architectures/AGENTS.md#when-to-override-preprocess_weights), [debugging_numerical_divergence.md](../../../docs/source/content/debugging_numerical_divergence.md). P4 drift: [§Tokenizer policy](../../model_bridge/supported_architectures/AGENTS.md#tokenizer-policy) (logit-scale / embedding-scale folds typically degrade P4 without crossing the 50% gate). +P1/P3 failures: [supported_architectures/AGENTS.md §When to override preprocess_weights](../../model_bridge/supported_architectures/AGENTS.md#when-to-override-preprocess_weights), [debugging_numerical_divergence.md](../../../docs/source/content/debugging_numerical_divergence.md). P4 drift: [§Tokenizer policy](../../model_bridge/supported_architectures/AGENTS.md#tokenizer-policy) (logit-scale / embedding-scale folds typically degrade P4 without crossing the pass line). --- diff --git a/transformer_lens/tools/model_registry/data/supported_models.json b/transformer_lens/tools/model_registry/data/supported_models.json index f27bacfa62..fabaf12e6d 100644 --- a/transformer_lens/tools/model_registry/data/supported_models.json +++ b/transformer_lens/tools/model_registry/data/supported_models.json @@ -101,7 +101,7 @@ "status": 3, "verified_date": "2026-06-27", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.013494, mean_rel=0.006767", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.013494, mean_rel=0.006767", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -115,7 +115,7 @@ "status": 3, "verified_date": "2026-06-27", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.020484, mean_rel=0.006617", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.020484, mean_rel=0.006617", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -481,7 +481,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -936,7 +936,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -1240,7 +1240,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=59.411900, mean_rel=0.569168", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=59.411900, mean_rel=0.569168", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -1636,7 +1636,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -1830,7 +1830,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=59.402462, mean_rel=0.441563", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=59.402462, mean_rel=0.441563", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -2933,7 +2933,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -3756,7 +3756,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=16.391922, mean_rel=3.526243", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=16.391922, mean_rel=3.526243", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 95.0, @@ -3770,7 +3770,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=18.059958, mean_rel=2.710044", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=18.059958, mean_rel=2.710044", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 95.0, @@ -4221,7 +4221,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002074, mean_rel=0.000409", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002074, mean_rel=0.000409", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -5839,7 +5839,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -6034,7 +6034,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=33.204865, mean_rel=0.370595", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=33.204865, mean_rel=0.370595", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -6300,7 +6300,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene \u2014 Forward pass failed: index out of range in self", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene — Forward pass failed: index out of range in self", "phase1_score": 50.0, "phase2_score": 7.7, "phase3_score": 44.4, @@ -7659,7 +7659,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7673,7 +7673,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=6.747103, mean_rel=0.054769", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=6.747103, mean_rel=0.054769", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -7687,7 +7687,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7701,7 +7701,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7715,7 +7715,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.479654, mean_rel=0.052641", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.479654, mean_rel=0.052641", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -7729,7 +7729,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7743,7 +7743,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=6.650925, mean_rel=0.050073", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=6.650925, mean_rel=0.050073", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -7757,7 +7757,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7771,7 +7771,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.641898, mean_rel=0.054789", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.641898, mean_rel=0.054789", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -7785,7 +7785,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.808517, mean_rel=0.051374", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.808517, mean_rel=0.051374", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -11825,7 +11825,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=32.255035, mean_rel=0.318908", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=32.255035, mean_rel=0.318908", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -12813,7 +12813,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 76.9, "phase3_score": 90.0, @@ -14366,7 +14366,7 @@ "status": 3, "verified_date": "2026-07-01", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 24/51 components failed (24 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 24/51 components failed (24 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -14380,7 +14380,7 @@ "status": 3, "verified_date": "2026-07-01", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 50/99 components failed (50 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 50/99 components failed (50 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -14394,7 +14394,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 32/68 components failed (32 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 32/68 components failed (32 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -17234,7 +17234,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=3.625000, mean_rel=0.024780", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=3.625000, mean_rel=0.024780", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -18186,7 +18186,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Can't load the model for 'EleutherAI/pythia-410m-seed1'. If you were trying to load it from 'https://hug", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Can't load the model for 'EleutherAI/pythia-410m-seed1'. If you were trying to load it from 'https://hug", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -18604,13 +18604,14 @@ "architecture_id": "GPTNeoXForCausalLM", "model_id": "EleutherAI/pythia-70m", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 89.9, + "phase4_score": 56.6, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -18663,7 +18664,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 77.0/100 (avg perplexity: 327.9) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 77.0/100 (avg perplexity: 327.9) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -18677,7 +18678,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 68.8/100 (avg perplexity: 743.0) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 68.8/100 (avg perplexity: 743.0) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -19538,7 +19539,7 @@ "status": 3, "verified_date": "2026-03-11", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Ex0bit/Elbaz-Olmo-3-7B-Instruct-abliterated does not appear to have files named ('model-00001-of-00006.s", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Ex0bit/Elbaz-Olmo-3-7B-Instruct-abliterated does not appear to have files named ('model-00001-of-00006.s", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -20288,7 +20289,7 @@ "status": 3, "verified_date": "2026-03-11", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -20338,7 +20339,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=31.355835, mean_rel=0.607207", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=31.355835, mean_rel=0.607207", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -22638,7 +22639,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.011660, mean_rel=0.002703", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.011660, mean_rel=0.002703", "phase1_score": 50.0, "phase2_score": 91.7, "phase3_score": 100.0, @@ -22819,7 +22820,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 144/196 components failed (144 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 144/196 components failed (144 critical)", "phase1_score": 0.0, "phase2_score": 15.4, "phase3_score": 21.1, @@ -26799,7 +26800,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 73.9/100 (avg perplexity: 10.2) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 73.9/100 (avg perplexity: 10.2) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -29217,7 +29218,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -31877,7 +31878,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: MachadoDeCastro/krull-micro does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: MachadoDeCastro/krull-micro does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -33397,7 +33398,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 12/87 components failed (12 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 12/87 components failed (12 critical)", "phase1_score": 0.0, "phase2_score": 15.4, "phase3_score": 50.0, @@ -33783,7 +33784,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -35093,7 +35094,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=39.516827, mean_rel=0.391392", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=39.516827, mean_rel=0.391392", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -36233,7 +36234,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=63.272919, mean_rel=0.497796", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=63.272919, mean_rel=0.497796", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -37153,7 +37154,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=34.928375, mean_rel=0.262157", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=34.928375, mean_rel=0.262157", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -37989,7 +37990,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=29.677444, mean_rel=0.238732", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=29.677444, mean_rel=0.238732", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -38504,7 +38505,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 45.4/100 (avg perplexity: 196.5) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 45.4/100 (avg perplexity: 196.5) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -39926,7 +39927,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=50.417328, mean_rel=0.326480", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=50.417328, mean_rel=0.326480", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -40407,7 +40408,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=34.771893, mean_rel=0.288610", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=34.771893, mean_rel=0.288610", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -41136,7 +41137,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=61.738430, mean_rel=0.447178", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=61.738430, mean_rel=0.447178", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -41730,7 +41731,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: extra_special_tokens must be a list/tuple of str or AddedToken, or a dict mapping names to tokens", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: extra_special_tokens must be a list/tuple of str or AddedToken, or a dict mapping names to tokens", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -43628,15 +43629,18 @@ "architecture_id": "Qwen2ForCausalLM", "model_id": "Qwen/Qwen2.5-0.5B-Instruct", "status": 1, - "verified_date": "2026-03-10", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "chat", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 96.6, + "phase4_score": 97.9, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Qwen2ForCausalLM", @@ -47449,7 +47453,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 1152 (input tensor's size at dimension 0), but go", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 1152 (input tensor's size at dimension 0), but go", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -54305,7 +54309,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=nan, mean_rel=nan", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=nan, mean_rel=nan", "phase1_score": 50.0, "phase2_score": 75.0, "phase3_score": 94.1, @@ -55738,7 +55742,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: argument 'vocab': 'dict' object cannot be converted to 'Sequence'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: argument 'vocab': 'dict' object cannot be converted to 'Sequence'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -58401,7 +58405,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 130/132 components failed (125 high, 5 medium)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 130/132 components failed (125 high, 5 medium)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -59956,7 +59960,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Xenova/distilgpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Xenova/distilgpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -59998,7 +60002,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Xenova/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Xenova/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -64445,7 +64449,7 @@ "status": 3, "verified_date": "2026-04-02", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.004379, mean_rel=0.022909", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.004379, mean_rel=0.022909", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -68283,7 +68287,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 12/87 components failed (12 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 12/87 components failed (12 critical)", "phase1_score": 0.0, "phase2_score": 15.4, "phase3_score": 50.0, @@ -71406,7 +71410,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.005348, mean_rel=0.000007", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.005348, mean_rel=0.000007", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 95.0, @@ -72882,7 +72886,7 @@ "downloads": 89081, "total_params": null }, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -73007,7 +73011,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74364,7 +74368,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74421,7 +74425,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74450,7 +74454,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74478,7 +74482,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74492,7 +74496,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P2=69.2% < 75.0% (f \u2014 59/64 components failed (59 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P2=69.2% < 75.0% (f — 59/64 components failed (59 critical)", "phase1_score": 0.0, "phase2_score": 69.2, "phase3_score": 75.0, @@ -74674,7 +74678,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74688,7 +74692,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74700,13 +74704,15 @@ "architecture_id": "MT5ForConditionalGeneration", "model_id": "bigscience/mt0-base", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed with issues, low text quality", + "note": "Core verification passed, but text quality poor (P4=26.8). Needs review", + "prompt_profile": "task:instruction", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 37.4, + "phase4_score": 26.8, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -74815,7 +74821,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=65.931717, mean_rel=2.066483", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=65.931717, mean_rel=2.066483", "phase1_score": 50.0, "phase2_score": 92.3, "phase3_score": 95.0, @@ -74829,7 +74835,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=22.915417, mean_rel=11.391559", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=22.915417, mean_rel=11.391559", "phase1_score": 50.0, "phase2_score": 92.3, "phase3_score": 95.0, @@ -76281,7 +76287,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -79435,7 +79441,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.003122, mean_rel=0.000469", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.003122, mean_rel=0.000469", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -80359,7 +80365,7 @@ "status": 3, "verified_date": "2026-05-08", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 36/282 components failed (36 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 36/282 components failed (36 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -85238,7 +85244,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -87406,7 +87412,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -90414,7 +90420,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 68.7/100 (avg perplexity: 776.1) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 68.7/100 (avg perplexity: 776.1) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -92466,7 +92472,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=9.275972, mean_rel=13.166794", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=9.275972, mean_rel=13.166794", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -92564,7 +92570,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", + "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", "phase1_score": 100.0, "phase2_score": 66.7, "phase3_score": 100.0, @@ -92578,7 +92584,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", + "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", "phase1_score": 100.0, "phase2_score": 66.7, "phase3_score": 100.0, @@ -92606,7 +92612,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", + "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", "phase1_score": 100.0, "phase2_score": 66.7, "phase3_score": 100.0, @@ -92632,13 +92638,15 @@ "architecture_id": "T5ForConditionalGeneration", "model_id": "google-t5/t5-base", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed with issues, low text quality", + "note": "Core verification completed", + "prompt_profile": "task:translation@en-de", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 49.3, + "phase4_score": 100.0, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -92661,13 +92669,15 @@ "architecture_id": "T5ForConditionalGeneration", "model_id": "google-t5/t5-small", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:translation@en-de", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 93.1, + "phase4_score": 90.4, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -93012,13 +93022,15 @@ "architecture_id": "Gemma2ForCausalLM", "model_id": "google/gemma-2-2b-it", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed with issues: P3=95.5% (failed: unembed_centering)", + "note": "Core verification completed (prior issues retained: P3=95.5%)", + "prompt_profile": "chat", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 95.5, - "phase4_score": 100.0, + "phase4_score": 97.9, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -98730,7 +98742,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.003898, mean_rel=0.027017", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.003898, mean_rel=0.027017", "phase1_score": 50.0, "phase2_score": 92.3, "phase3_score": 95.0, @@ -100830,7 +100842,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 4/24 components failed (4 high)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 4/24 components failed (4 high)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 85.0, @@ -101775,7 +101787,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=4.489960, mean_rel=1.238444", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=4.489960, mean_rel=1.238444", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -101901,7 +101913,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=20.307718, mean_rel=6.347236", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=20.307718, mean_rel=6.347236", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -102000,7 +102012,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=4.789991, mean_rel=1.159405", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=4.789991, mean_rel=1.159405", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -102210,7 +102222,7 @@ "status": 3, "verified_date": "2026-04-14", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'in_proj'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'in_proj'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -102224,7 +102236,7 @@ "status": 3, "verified_date": "2026-04-15", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102238,7 +102250,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102252,7 +102264,7 @@ "status": 3, "verified_date": "2026-04-15", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102266,7 +102278,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102280,7 +102292,7 @@ "status": 3, "verified_date": "2026-04-15", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102294,7 +102306,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102336,7 +102348,7 @@ "status": 3, "verified_date": "2026-04-15", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102350,7 +102362,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102392,7 +102404,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -103699,7 +103711,7 @@ "downloads": 5738, "total_params": null }, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'type'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'type'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -108842,7 +108854,7 @@ "status": 3, "verified_date": "2026-04-08", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load katuni4ka/tiny-random-deepseek-v3:\n(Request ID", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load katuni4ka/tiny-random-deepseek-v3:\n(Request ID", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -108856,7 +108868,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 256 (input tensor's size at dimension 0), but got", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 256 (input tensor's size at dimension 0), but got", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -117871,7 +117883,7 @@ "status": 1, "verified_date": "2026-02-25", "metadata": null, - "note": "Below threshold: P3=81.8% but required tests failed: logits_equivalence \u2014 Scalars differ: 0.000000 vs -0.015625", + "note": "Below threshold: P3=81.8% but required tests failed: logits_equivalence — Scalars differ: 0.000000 vs -0.015625", "phase1_score": 100.0, "phase2_score": 78.6, "phase3_score": 81.8, @@ -124483,7 +124495,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -125459,7 +125471,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=75.0% but required tests failed \u2014 Tensors differ: max_diff=378.613281, mean_rel=0.057195", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=75.0% but required tests failed — Tensors differ: max_diff=378.613281, mean_rel=0.057195", "phase1_score": 50.0, "phase2_score": 76.9, "phase3_score": 75.0, @@ -126611,7 +126623,7 @@ "status": 3, "verified_date": "2026-03-11", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/gemma-3-270m-it-ONNX does not appear to have a file named pytorch_model.bin or model.safe", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/gemma-3-270m-it-ONNX does not appear to have a file named pytorch_model.bin or model.safe", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -126639,7 +126651,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-1b-ONNX-web does not appear to have a file named pytorch_model.bin or model.s", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-1b-ONNX-web does not appear to have a file named pytorch_model.bin or model.s", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -126653,7 +126665,7 @@ "status": 3, "verified_date": "2026-04-14", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-350m-ONNX-web does not appear to have a file named pytorch_model.bin or model", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-350m-ONNX-web does not appear to have a file named pytorch_model.bin or model", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -126681,7 +126693,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/tiny-random-olmo-hf does not appear to have a file named pytorch_model.bin or model.safet", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/tiny-random-olmo-hf does not appear to have a file named pytorch_model.bin or model.safet", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -127241,13 +127253,14 @@ "architecture_id": "GPT2LMHeadModel", "model_id": "openai-community/gpt2", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 88.5, + "phase4_score": 68.2, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -128009,7 +128022,7 @@ "status": 3, "verified_date": "2026-04-08", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load optimum-intel-internal-testing/tiny-random-dee", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load optimum-intel-internal-testing/tiny-random-dee", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -128127,7 +128140,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: optimum/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: optimum/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -129659,7 +129672,7 @@ "status": 3, "verified_date": "2026-05-08", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/190 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/190 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -134450,7 +134463,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P4=4.1% < 50.0% (failed: text_quality) \u2014 Text quality score: 4.1/100 (avg perplexity: 3.4) \u2014 generated text may be incoherent", + "note": "Below threshold: P4=4.1% < 50.0% (failed: text_quality) — Text quality score: 4.1/100 (avg perplexity: 3.4) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, @@ -136120,7 +136133,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "note": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 69.2, "phase3_score": 95.0, @@ -137724,7 +137737,7 @@ "status": 3, "verified_date": "2026-03-11", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 104/242 components failed (104 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 104/242 components failed (104 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -138319,7 +138332,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence; P4=6.8% < 50.0% (failed: te \u2014 Text quality score: 6.8/100 (avg perplexity: 372419.9) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence; P4=6.8% < 50.0% (failed: te — Text quality score: 6.8/100 (avg perplexity: 372419.9) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -138333,7 +138346,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'FalconDecoderLayer' object has no attribute 'ln_attn'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'FalconDecoderLayer' object has no attribute 'ln_attn'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -141818,7 +141831,7 @@ "status": 3, "verified_date": "2026-04-09", "metadata": null, - "note": "Below threshold: P3=50.0% < 75.0% (failed: process_bridge_weights, layer_norm_folding, weight_modifi \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "note": "Below threshold: P3=50.0% < 75.0% (failed: process_bridge_weights, layer_norm_folding, weight_modifi — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "phase1_score": 100.0, "phase2_score": 83.3, "phase3_score": 50.0, @@ -143278,7 +143291,7 @@ "status": 3, "verified_date": "2026-04-08", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/12 components failed (2 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/12 components failed (2 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 94.7, @@ -143292,7 +143305,7 @@ "status": 3, "verified_date": "2026-04-08", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 94.7, @@ -143320,7 +143333,7 @@ "status": 3, "verified_date": "2026-02-22", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/12 components failed (1 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/12 components failed (1 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 83.3, @@ -143561,7 +143574,7 @@ "downloads": 207171, "total_params": 2574656 }, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 72.2/100 (avg perplexity: 558.8) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 72.2/100 (avg perplexity: 558.8) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -148073,7 +148086,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -148087,7 +148100,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -148101,7 +148114,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g \u2014 144/307 components failed (144 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g — 144/307 components failed (144 critical)", "phase1_score": 0.0, "phase2_score": 7.7, "phase3_score": 22.2, @@ -149896,7 +149909,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.004045, mean_rel=0.000066", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.004045, mean_rel=0.000066", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -154674,7 +154687,7 @@ "status": 3, "verified_date": "2026-04-09", "metadata": null, - "note": "Below threshold: P3=55.6% < 75.0% (failed: process_bridge_weights, weight_modification, hook_functio \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "note": "Below threshold: P3=55.6% < 75.0% (failed: process_bridge_weights, weight_modification, hook_functio — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "phase1_score": 100.0, "phase2_score": 83.3, "phase3_score": 55.6, @@ -160112,7 +160125,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=0.009886, mean_rel=0.980186", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=0.009886, mean_rel=0.980186", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -162282,7 +162295,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "note": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 69.2, "phase3_score": 95.0, @@ -168358,7 +168371,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: [Errno 2] No such file or directory: 'baichuan-inc/Baichuan2-7B-Chat-4bits/pytorch_model.bin'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: [Errno 2] No such file or directory: 'baichuan-inc/Baichuan2-7B-Chat-4bits/pytorch_model.bin'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -168400,7 +168413,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 70/609 components failed (70 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 70/609 components failed (70 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -172783,7 +172796,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 70/609 components failed (70 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 70/609 components failed (70 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -172797,7 +172810,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 60/534 components failed (60 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 60/534 components failed (60 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -174031,7 +174044,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -175109,7 +175122,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -175935,7 +175948,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -176607,7 +176620,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -177321,7 +177334,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -177713,7 +177726,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=38.5% < 75.0% (failed: \u2014 1/15 components failed (1 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=38.5% < 75.0% (failed: — 1/15 components failed (1 critical)", "phase1_score": 0.0, "phase2_score": 38.5, "phase3_score": 38.9, @@ -178525,7 +178538,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: LiquidAI/LFM2-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: LiquidAI/LFM2-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -178553,7 +178566,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: LiquidAI/LFM2.5-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: LiquidAI/LFM2.5-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -178747,13 +178760,15 @@ "architecture_id": "BartForConditionalGeneration", "model_id": "facebook/bart-large-cnn", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:summarization", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 77.2, + "phase4_score": 99.2, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -178988,7 +179003,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179002,7 +179017,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179086,7 +179101,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 9/167 components failed (9 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 9/167 components failed (9 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": null, @@ -179100,7 +179115,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179114,7 +179129,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179128,7 +179143,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179142,7 +179157,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179156,7 +179171,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179170,7 +179185,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179184,7 +179199,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179198,7 +179213,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179212,7 +179227,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179226,7 +179241,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179240,7 +179255,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179254,7 +179269,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179268,7 +179283,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179282,7 +179297,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179296,7 +179311,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179310,7 +179325,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179324,7 +179339,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -181326,7 +181341,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=11.357496, mean_rel=4.069944", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=11.357496, mean_rel=4.069944", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -182710,15 +182725,18 @@ "architecture_id": "MarianMTModel", "model_id": "Helsinki-NLP/opus-mt-nl-en", "status": 1, - "verified_date": "2026-07-24", + "verified_date": "2026-08-21", "metadata": null, "note": "Core verification completed", + "prompt_profile": "task:translation@nl-en", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 80.7, + "phase4_score": 94.2, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MarianMTModel", @@ -183116,13 +183134,15 @@ "architecture_id": "M2M100ForConditionalGeneration", "model_id": "facebook/m2m100_418M", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed with issues, low text quality", + "note": "Core verification completed", + "prompt_profile": "task:translation@en-de", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 41.2, + "phase4_score": 100.0, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -183285,43 +183305,52 @@ "architecture_id": "MBartForConditionalGeneration", "model_id": "facebook/mbart-large-50-many-to-many-mmt", "status": 1, - "verified_date": "2026-07-24", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:translation@en-de", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 89.4, + "phase4_score": 100.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MBartForConditionalGeneration", "model_id": "facebook/mbart-large-50", "status": 1, - "verified_date": "2026-07-07", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:denoise", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 100.0, + "phase4_score": 87.1, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MBartForConditionalGeneration", "model_id": "facebook/mbart-large-cc25", "status": 1, - "verified_date": "2026-07-07", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification passed, but text quality poor (P4=25.0). Needs review", + "prompt_profile": "task:denoise", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 92.4, + "phase4_score": 25.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MBartForConditionalGeneration", @@ -183329,11 +183358,11 @@ "status": 1, "verified_date": "2026-07-07", "metadata": null, - "note": "Full verification completed with issues, low text quality", + "note": "P4 nulled: prior score measured under a broken MBart profile (mis-profiled + unconstrained target language); awaiting Indic denoise prompt coverage", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 32.5, + "phase4_score": null, "phase7_score": null, "phase8_score": null }, @@ -183355,13 +183384,15 @@ "architecture_id": "PegasusForConditionalGeneration", "model_id": "google/pegasus-xsum", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:summarization", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 98.3, + "phase4_score": 90.0, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -183512,7 +183543,7 @@ "status": 3, "verified_date": "2026-07-07", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=27.217707, mean_rel=1.057937", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=27.217707, mean_rel=1.057937", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 94.7, @@ -183889,13 +183920,15 @@ "architecture_id": "LongT5ForConditionalGeneration", "model_id": "google/long-t5-tglobal-base", "status": 1, - "verified_date": "2026-08-20", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed with issues, low text quality", + "note": "Core verification passed, but text quality poor (P4=48.4). Needs review", + "prompt_profile": "task:denoise", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 33.8, + "phase4_score": 48.4, "phase7_score": null, "phase8_score": null, "phase9_score": null @@ -183906,7 +183939,7 @@ "status": 3, "verified_date": "2026-07-07", "metadata": null, - "note": "Checkpoint ships no lm_head.weight with tie_word_embeddings=false, so HF randomly re-initializes the LM head on every load (Google released it as a pretraining artifact requiring fine-tuning); the nondeterministic unembed is the only P1 failure \u2014 all 233 other components including the local-attention encoder pass. Not an adapter bug.", + "note": "Checkpoint ships no lm_head.weight with tie_word_embeddings=false, so HF randomly re-initializes the LM head on every load (Google released it as a pretraining artifact requiring fine-tuning); the nondeterministic unembed is the only P1 failure — all 233 other components including the local-attention encoder pass. Not an adapter bug.", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -184074,7 +184107,7 @@ "status": 1, "verified_date": "2026-07-07", "metadata": null, - "note": "Verified on a local snapshot of yujiepan/llama-4-tiny-random with text_config.attn_temperature_tuning coerced to bool \u2014 the upstream config declares it as int 4, which transformers 5.x strict config validation rejects. P7 skipped: the tiny ships no processor files. P3=95 (attention_output_centering worst_mean=0.099 on random weights). Official Scout/Maverick checkpoints registered for big-hardware verification.", + "note": "Verified on a local snapshot of yujiepan/llama-4-tiny-random with text_config.attn_temperature_tuning coerced to bool — the upstream config declares it as int 4, which transformers 5.x strict config validation rejects. P7 skipped: the tiny ships no processor files. P3=95 (attention_output_centering worst_mean=0.099 on random weights). Official Scout/Maverick checkpoints registered for big-hardware verification.", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 95.0, @@ -186594,7 +186627,7 @@ "status": 3, "verified_date": "2026-07-07", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -187392,7 +187425,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -187490,7 +187523,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -187546,7 +187579,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -216316,7 +216349,7 @@ "status": 1, "verified_date": "2026-07-23", "metadata": null, - "note": "Full verification completed (P1 in fp32, P2/P4 in bf16 for memory). bf16 P1 is precision-bound: max_diff=0.375, mean_rel=0.022 at bf16, but the same comparison in fp32 matches within tolerance \u2014 measured, not assumed. Required a reconstruction dtype fix: FlexOlmoRotaryEmbedding returns fp32 cos/sin without casting to the input dtype, promoting the attention output to fp32 against bf16 projection weights.", + "note": "Full verification completed (P1 in fp32, P2/P4 in bf16 for memory). bf16 P1 is precision-bound: max_diff=0.375, mean_rel=0.022 at bf16, but the same comparison in fp32 matches within tolerance — measured, not assumed. Required a reconstruction dtype fix: FlexOlmoRotaryEmbedding returns fp32 cos/sin without casting to the input dtype, promoting the attention output to fp32 against bf16 projection weights.", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, @@ -216554,7 +216587,7 @@ "status": 1, "verified_date": "2026-07-23", "metadata": null, - "note": "Forward parity PROVEN: bridge byte-identical to raw HF in fp32 on identical ids (max \u0394logit 0, max \u0394log_softmax 0, 100% argmax). P4=55.9 is genuine model behavior (one hard NL prompt, diffusion sampling, weak GPT-2 judge), not a bridge defect. HF end-to-end capture requires a 4D (b,1,s,s) block attention mask.", + "note": "Forward parity PROVEN: bridge byte-identical to raw HF in fp32 on identical ids (max Δlogit 0, max Δlog_softmax 0, 100% argmax). P4=55.9 is genuine model behavior (one hard NL prompt, diffusion sampling, weak GPT-2 judge), not a bridge defect. HF end-to-end capture requires a 4D (b,1,s,s) block attention mask.", "phase1_score": 100.0, "phase2_score": null, "phase3_score": null, diff --git a/transformer_lens/tools/model_registry/data/verification_history.json b/transformer_lens/tools/model_registry/data/verification_history.json index 49de0e7a21..24dd53444e 100644 --- a/transformer_lens/tools/model_registry/data/verification_history.json +++ b/transformer_lens/tools/model_registry/data/verification_history.json @@ -1,5 +1,5 @@ { - "last_updated": "2026-08-20T13:20:36.439867", + "last_updated": "2026-08-21T12:51:05.468344", "records": [ { "model_id": "Macropodus/macbert4mdcspell_v1", @@ -2287,7 +2287,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% \u2014 No tokenizer files on HuggingFace (ValueError: Couldn't instantiate the backend tokenizer)", + "notes": "Below threshold: P1=0.0% < 100.0% — No tokenizer files on HuggingFace (ValueError: Couldn't instantiate the backend tokenizer)", "invalidated": false, "invalidation_reason": null }, @@ -2297,7 +2297,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% \u2014 Requires bitsandbytes 8-bit quantization (ImportError: pip install -U bitsandbytes>=0.46.1)", + "notes": "Below threshold: P1=0.0% < 100.0% — Requires bitsandbytes 8-bit quantization (ImportError: pip install -U bitsandbytes>=0.46.1)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2397,7 +2397,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 47/292 components failed (14 critical, 33 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 47/292 components failed (14 critical, 33 high)", "invalidated": false, "invalidation_reason": null }, @@ -2407,7 +2407,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.125000, mean_rel=0.033691", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.125000, mean_rel=0.033691", "invalidated": false, "invalidation_reason": null }, @@ -2417,7 +2417,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.281250, mean_rel=0.051025", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.281250, mean_rel=0.051025", "invalidated": false, "invalidation_reason": null }, @@ -2447,7 +2447,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2457,7 +2457,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", "invalidated": false, "invalidation_reason": null }, @@ -2467,7 +2467,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2477,7 +2477,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2487,7 +2487,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 31/196 components failed (31 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 31/196 components failed (31 critical)", "invalidated": false, "invalidation_reason": null }, @@ -2497,7 +2497,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 48/76 components failed (48 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 48/76 components failed (48 critical)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2507,7 +2507,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.250000, mean_rel=0.045166", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.250000, mean_rel=0.045166", "invalidated": false, "invalidation_reason": null }, @@ -2677,7 +2677,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: '<' not supported between instances of 'NoneType' and 'int'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: '<' not supported between instances of 'NoneType' and 'int'", "invalidated": false, "invalidation_reason": null }, @@ -2697,7 +2697,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: '<' not supported between instances of 'NoneType' and 'int'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: '<' not supported between instances of 'NoneType' and 'int'", "invalidated": false, "invalidation_reason": null }, @@ -2777,7 +2777,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/12 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/12 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -2787,7 +2787,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, critical_forward_ho \u2014 Tensors differ: max_diff=0.083040, mean_rel=0.006218", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, critical_forward_ho — Tensors differ: max_diff=0.083040, mean_rel=0.006218", "invalidated": false, "invalidation_reason": null }, @@ -2837,7 +2837,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2847,7 +2847,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=0.097573, mean_rel=0.008319", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=0.097573, mean_rel=0.008319", "invalidated": false, "invalidation_reason": null }, @@ -2857,7 +2857,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=0.129729, mean_rel=0.023225", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=0.129729, mean_rel=0.023225", "invalidated": false, "invalidation_reason": null }, @@ -2867,7 +2867,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: logits_equivalence, hook_functionality, critical_forward_ \u2014 Tensors differ: max_diff=0.286175, mean_rel=0.028925", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: logits_equivalence, hook_functionality, critical_forward_ — Tensors differ: max_diff=0.286175, mean_rel=0.028925", "invalidated": false, "invalidation_reason": null }, @@ -2887,7 +2887,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/gemma-3-1b-it-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/gemma-3-1b-it-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2897,7 +2897,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2937,7 +2937,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, critical_forward_ho \u2014 Tensors differ: max_diff=0.705528, mean_rel=0.011718", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, critical_forward_ho — Tensors differ: max_diff=0.705528, mean_rel=0.011718", "invalidated": false, "invalidation_reason": null }, @@ -2967,7 +2967,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3037,7 +3037,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3107,7 +3107,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=8.3% < 75.0% (failed: g \u2014 2/148 components failed (2 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=8.3% < 75.0% (failed: g — 2/148 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -3147,7 +3147,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: ", "invalidated": false, "invalidation_reason": null }, @@ -3227,7 +3227,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3327,7 +3327,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=1.562500, mean_rel=0.753906", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=1.562500, mean_rel=0.753906", "invalidated": false, "invalidation_reason": null }, @@ -3357,7 +3357,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.250000, mean_rel=0.014526", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.250000, mean_rel=0.014526", "invalidated": false, "invalidation_reason": null }, @@ -3477,7 +3477,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3677,7 +3677,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3727,7 +3727,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3907,7 +3907,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/147 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/147 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -3947,7 +3947,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: ", "invalidated": false, "invalidation_reason": null }, @@ -4077,7 +4077,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Xenova/distilgpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Xenova/distilgpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -4087,7 +4087,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Xenova/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Xenova/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -4117,7 +4117,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: optimum/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: optimum/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -4157,7 +4157,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4167,7 +4167,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", "invalidated": false, "invalidation_reason": null }, @@ -4177,7 +4177,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4187,7 +4187,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4207,7 +4207,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 48/76 components failed (48 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 48/76 components failed (48 critical)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4247,7 +4247,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002074, mean_rel=0.000409", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002074, mean_rel=0.000409", "invalidated": false, "invalidation_reason": null }, @@ -4537,7 +4537,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": false, "invalidation_reason": null }, @@ -4637,7 +4637,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4717,7 +4717,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4807,7 +4807,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4847,7 +4847,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4867,7 +4867,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": false, "invalidation_reason": null }, @@ -5027,7 +5027,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5047,7 +5047,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/Llama-3.2-1B-Instruct-GGUF does not appear to have a file named pytorch_model.bin or model.safet", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/Llama-3.2-1B-Instruct-GGUF does not appear to have a file named pytorch_model.bin or model.safet", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5077,7 +5077,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5087,7 +5087,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5117,7 +5117,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5127,7 +5127,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5177,7 +5177,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5197,7 +5197,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5257,7 +5257,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5327,7 +5327,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5357,7 +5357,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5377,7 +5377,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/DeepSeek-R1-Distill-Qwen-1.5B-GGUF does not appear to have a file named pytorch_model.bin or mod", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/DeepSeek-R1-Distill-Qwen-1.5B-GGUF does not appear to have a file named pytorch_model.bin or mod", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5407,7 +5407,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5447,7 +5447,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 130/132 components failed (125 high, 5 medium)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 130/132 components failed (125 high, 5 medium)", "invalidated": false, "invalidation_reason": null }, @@ -5457,7 +5457,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002012, mean_rel=0.000401", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002012, mean_rel=0.000401", "invalidated": false, "invalidation_reason": null }, @@ -5597,7 +5597,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: extra_special_tokens must be a list/tuple of str or AddedToken, or a dict mapping names to tokens", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: extra_special_tokens must be a list/tuple of str or AddedToken, or a dict mapping names to tokens", "invalidated": false, "invalidation_reason": null }, @@ -5607,7 +5607,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.004045, mean_rel=0.000066", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.004045, mean_rel=0.000066", "invalidated": false, "invalidation_reason": null }, @@ -5617,7 +5617,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.003122, mean_rel=0.000469", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.003122, mean_rel=0.000469", "invalidated": false, "invalidation_reason": null }, @@ -5687,7 +5687,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.011660, mean_rel=0.002703", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.011660, mean_rel=0.002703", "invalidated": false, "invalidation_reason": null }, @@ -5797,7 +5797,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5827,7 +5827,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5857,7 +5857,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5867,7 +5867,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5877,7 +5877,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5917,7 +5917,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/Qwen3-0.6B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/Qwen3-0.6B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5937,7 +5937,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5947,7 +5947,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5957,7 +5957,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/Qwen3-4B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/Qwen3-4B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5967,7 +5967,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5977,7 +5977,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5987,7 +5987,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5997,7 +5997,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/Qwen3-1.7B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/Qwen3-1.7B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6007,7 +6007,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6017,7 +6017,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6027,7 +6027,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6037,7 +6037,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6047,7 +6047,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6057,7 +6057,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6067,7 +6067,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6077,7 +6077,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6087,7 +6087,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6097,7 +6097,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6117,7 +6117,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6127,7 +6127,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6137,7 +6137,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6147,7 +6147,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6157,7 +6157,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6177,7 +6177,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6197,7 +6197,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6207,7 +6207,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6217,7 +6217,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6227,7 +6227,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 1/130 components failed (1 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 1/130 components failed (1 high)", "invalidated": false, "invalidation_reason": null }, @@ -6237,7 +6237,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 1/130 components failed (1 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 1/130 components failed (1 high)", "invalidated": false, "invalidation_reason": null }, @@ -6247,7 +6247,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 1/130 components failed (1 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 1/130 components failed (1 high)", "invalidated": false, "invalidation_reason": null }, @@ -6317,7 +6317,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook \u2014 Bridge is missing 56 hooks from reference model", + "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook — Bridge is missing 56 hooks from reference model", "invalidated": false, "invalidation_reason": null }, @@ -6347,7 +6347,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook \u2014 Bridge is missing 56 hooks from reference model", + "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook — Bridge is missing 56 hooks from reference model", "invalidated": false, "invalidation_reason": null }, @@ -6707,7 +6707,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=85.0% but required tests failed: logits_equivalence \u2014 Found 1 significant mismatches in critical hooks", + "notes": "Below threshold: P3=85.0% but required tests failed: logits_equivalence — Found 1 significant mismatches in critical hooks", "invalidated": false, "invalidation_reason": null }, @@ -6767,7 +6767,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=nan, mean_rel=nan", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=nan, mean_rel=nan", "invalidated": false, "invalidation_reason": null }, @@ -6807,7 +6807,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810717, mean_rel=73.159515", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810717, mean_rel=73.159515", "invalidated": false, "invalidation_reason": null }, @@ -7077,7 +7077,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810717, mean_rel=73.159515", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810717, mean_rel=73.159515", "invalidated": false, "invalidation_reason": null }, @@ -7217,7 +7217,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=81.8% but required tests failed: logits_equivalence \u2014 Scalars differ: 0.000000 vs -0.015625", + "notes": "Below threshold: P3=81.8% but required tests failed: logits_equivalence — Scalars differ: 0.000000 vs -0.015625", "invalidated": false, "invalidation_reason": null }, @@ -7237,7 +7237,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho \u2014 Scalars differ: 5.875000 vs 5.812500", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho — Scalars differ: 5.875000 vs 5.812500", "invalidated": false, "invalidation_reason": null }, @@ -7247,7 +7247,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho \u2014 Scalars differ: 5.875000 vs 5.812500", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho — Scalars differ: 5.875000 vs 5.812500", "invalidated": false, "invalidation_reason": null }, @@ -7267,7 +7267,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho \u2014 Scalars differ: 5.968750 vs 5.875000", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho — Scalars differ: 5.968750 vs 5.875000", "invalidated": false, "invalidation_reason": null }, @@ -7287,7 +7287,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P4=0.0% < 50.0% (failed: text_quality) \u2014 Text quality score: 77.3/100 (avg perplexity: 30.4) \u2014 generated text may be incoherent", + "notes": "Below threshold: P4=0.0% < 50.0% (failed: text_quality) — Text quality score: 77.3/100 (avg perplexity: 30.4) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7297,7 +7297,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P4=0.0% < 50.0% (failed: text_quality) \u2014 Text quality score: 77.3/100 (avg perplexity: 30.4) \u2014 generated text may be incoherent", + "notes": "Below threshold: P4=0.0% < 50.0% (failed: text_quality) — Text quality score: 77.3/100 (avg perplexity: 30.4) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7317,7 +7317,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P4=4.1% < 50.0% (failed: text_quality) \u2014 Text quality score: 4.1/100 (avg perplexity: 3.4) \u2014 generated text may be incoherent", + "notes": "Below threshold: P4=4.1% < 50.0% (failed: text_quality) — Text quality score: 4.1/100 (avg perplexity: 3.4) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7337,7 +7337,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 68.7/100 (avg perplexity: 776.1) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 68.7/100 (avg perplexity: 776.1) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7347,7 +7347,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 68.7/100 (avg perplexity: 776.1) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 68.7/100 (avg perplexity: 776.1) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7357,7 +7357,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 68.7/100 (avg perplexity: 776.1) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 68.7/100 (avg perplexity: 776.1) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7367,7 +7367,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence; P4=6.8% < 50.0% (failed: te \u2014 Text quality score: 6.8/100 (avg perplexity: 372419.9) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence; P4=6.8% < 50.0% (failed: te — Text quality score: 6.8/100 (avg perplexity: 372419.9) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7377,7 +7377,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 77.0/100 (avg perplexity: 327.9) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 77.0/100 (avg perplexity: 327.9) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7387,7 +7387,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 68.8/100 (avg perplexity: 743.0) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 68.8/100 (avg perplexity: 743.0) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7397,7 +7397,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=3.625000, mean_rel=0.024780", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=3.625000, mean_rel=0.024780", "invalidated": false, "invalidation_reason": null }, @@ -7407,7 +7407,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Can't load the model for 'EleutherAI/pythia-410m-seed1'. If you were trying to load it from 'https://hug", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Can't load the model for 'EleutherAI/pythia-410m-seed1'. If you were trying to load it from 'https://hug", "invalidated": false, "invalidation_reason": null }, @@ -7437,7 +7437,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002956, mean_rel=0.000962", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002956, mean_rel=0.000962", "invalidated": false, "invalidation_reason": null }, @@ -7457,7 +7457,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002956, mean_rel=0.000962", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002956, mean_rel=0.000962", "invalidated": false, "invalidation_reason": null }, @@ -7577,7 +7577,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=13.293901, mean_rel=32.253456", + "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=13.293901, mean_rel=32.253456", "invalidated": false, "invalidation_reason": null }, @@ -7627,7 +7627,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence \u2014 Found 1 significant mismatches in critical hooks", + "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence — Found 1 significant mismatches in critical hooks", "invalidated": false, "invalidation_reason": null }, @@ -7637,7 +7637,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence \u2014 Text quality score: 77.5/100 (avg perplexity: 372.2) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence — Text quality score: 77.5/100 (avg perplexity: 372.2) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7647,7 +7647,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log \u2014 2/10 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log — 2/10 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -7697,7 +7697,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=12.128962, mean_rel=0.271985", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=12.128962, mean_rel=0.271985", "invalidated": false, "invalidation_reason": null }, @@ -7737,7 +7737,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810719, mean_rel=60.400272", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810719, mean_rel=60.400272", "invalidated": false, "invalidation_reason": null }, @@ -7767,7 +7767,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=19.425457, mean_rel=11.940315", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=19.425457, mean_rel=11.940315", "invalidated": false, "invalidation_reason": null }, @@ -7787,7 +7787,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "invalidated": false, "invalidation_reason": null }, @@ -7797,7 +7797,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=25.509125, mean_rel=0.521523", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=25.509125, mean_rel=0.521523", "invalidated": false, "invalidation_reason": null }, @@ -7807,7 +7807,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=21.740696, mean_rel=13.788611", + "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=21.740696, mean_rel=13.788611", "invalidated": false, "invalidation_reason": null }, @@ -7847,7 +7847,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002039, mean_rel=0.000401", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002039, mean_rel=0.000401", "invalidated": false, "invalidation_reason": null }, @@ -7947,7 +7947,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=12.128962, mean_rel=0.271985", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=12.128962, mean_rel=0.271985", "invalidated": false, "invalidation_reason": null }, @@ -8007,7 +8007,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence \u2014 Text quality score: 77.5/100 (avg perplexity: 372.2) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence — Text quality score: 77.5/100 (avg perplexity: 372.2) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -8027,7 +8027,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002039, mean_rel=0.000401", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002039, mean_rel=0.000401", "invalidated": false, "invalidation_reason": null }, @@ -8047,7 +8047,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "invalidated": false, "invalidation_reason": null }, @@ -8067,7 +8067,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002956, mean_rel=0.000962", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002956, mean_rel=0.000962", "invalidated": false, "invalidation_reason": null }, @@ -8077,7 +8077,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002956, mean_rel=0.000962", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002956, mean_rel=0.000962", "invalidated": false, "invalidation_reason": null }, @@ -8287,7 +8287,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=13.293901, mean_rel=32.253456", + "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=13.293901, mean_rel=32.253456", "invalidated": false, "invalidation_reason": null }, @@ -8337,7 +8337,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence \u2014 Found 1 significant mismatches in critical hooks", + "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence — Found 1 significant mismatches in critical hooks", "invalidated": false, "invalidation_reason": null }, @@ -8357,7 +8357,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log \u2014 2/10 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log — 2/10 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -8407,7 +8407,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks \u2014 Backward hooks check failed: 'tuple' object has no attribute 'clone'", + "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks — Backward hooks check failed: 'tuple' object has no attribute 'clone'", "invalidated": false, "invalidation_reason": null }, @@ -8417,7 +8417,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks \u2014 Backward hooks check failed: 'tuple' object has no attribute 'clone'", + "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks — Backward hooks check failed: 'tuple' object has no attribute 'clone'", "invalidated": false, "invalidation_reason": null }, @@ -8447,7 +8447,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810719, mean_rel=60.400272", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810719, mean_rel=60.400272", "invalidated": false, "invalidation_reason": null }, @@ -8477,7 +8477,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=19.425457, mean_rel=11.940315", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=19.425457, mean_rel=11.940315", "invalidated": false, "invalidation_reason": null }, @@ -8507,7 +8507,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks \u2014 Backward hooks check failed: 'tuple' object has no attribute 'clone'", + "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks — Backward hooks check failed: 'tuple' object has no attribute 'clone'", "invalidated": false, "invalidation_reason": null }, @@ -8687,7 +8687,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log \u2014 2/10 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log — 2/10 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -8707,7 +8707,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 1/149 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 1/149 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -8727,7 +8727,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=18.423609, mean_rel=0.259477", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=18.423609, mean_rel=0.259477", "invalidated": false, "invalidation_reason": null }, @@ -8927,7 +8927,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=13.293901, mean_rel=32.253456", + "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=13.293901, mean_rel=32.253456", "invalidated": false, "invalidation_reason": null }, @@ -8977,7 +8977,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence \u2014 Found 1 significant mismatches in critical hooks", + "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence — Found 1 significant mismatches in critical hooks", "invalidated": false, "invalidation_reason": null }, @@ -9087,7 +9087,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810719, mean_rel=60.400272", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810719, mean_rel=60.400272", "invalidated": false, "invalidation_reason": null }, @@ -9117,7 +9117,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=19.425457, mean_rel=11.940315", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=19.425457, mean_rel=11.940315", "invalidated": false, "invalidation_reason": null }, @@ -9137,7 +9137,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "invalidated": false, "invalidation_reason": null }, @@ -9627,7 +9627,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "invalidated": false, "invalidation_reason": null }, @@ -9637,7 +9637,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/gemma-3-270m-it-ONNX does not appear to have a file named pytorch_model.bin or model.safe", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/gemma-3-270m-it-ONNX does not appear to have a file named pytorch_model.bin or model.safe", "invalidated": false, "invalidation_reason": null }, @@ -9787,7 +9787,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 104/242 components failed (104 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 104/242 components failed (104 critical)", "invalidated": false, "invalidation_reason": null }, @@ -9997,7 +9997,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": false, "invalidation_reason": null }, @@ -10137,7 +10137,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Ex0bit/Elbaz-Olmo-3-7B-Instruct-abliterated does not appear to have files named ('model-00001-of-00006.s", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Ex0bit/Elbaz-Olmo-3-7B-Instruct-abliterated does not appear to have files named ('model-00001-of-00006.s", "invalidated": false, "invalidation_reason": null }, @@ -10367,7 +10367,7 @@ "verified_date": "2026-03-19", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 3/197 components failed (3 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 3/197 components failed (3 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10567,7 +10567,7 @@ "verified_date": "2026-03-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.1% < 75.0% (failed: generation, gene \u2014 Forward pass failed: 'PhiAttention' object has no attribute 'o_proj'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.1% < 75.0% (failed: generation, gene — Forward pass failed: 'PhiAttention' object has no attribute 'o_proj'", "invalidated": false, "invalidation_reason": null }, @@ -10577,7 +10577,7 @@ "verified_date": "2026-03-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.1% < 75.0% (failed: generation, gene \u2014 Forward pass failed: 'PhiAttention' object has no attribute 'o_proj'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.1% < 75.0% (failed: generation, gene — Forward pass failed: 'PhiAttention' object has no attribute 'o_proj'", "invalidated": false, "invalidation_reason": null }, @@ -10607,7 +10607,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10617,7 +10617,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10627,7 +10627,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10637,7 +10637,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10647,7 +10647,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gener \u2014 Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gener — Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", "invalidated": false, "invalidation_reason": null }, @@ -10697,7 +10697,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", "invalidated": false, "invalidation_reason": null }, @@ -10717,7 +10717,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10737,7 +10737,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", "invalidated": false, "invalidation_reason": null }, @@ -10747,7 +10747,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: shape '[1, 28, 24, 128]' is invalid for input of size 28672", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: shape '[1, 28, 24, 128]' is invalid for input of size 28672", "invalidated": false, "invalidation_reason": null }, @@ -10767,7 +10767,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: unsupported operand type(s) for *: 'NoneType' and 'int'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: unsupported operand type(s) for *: 'NoneType' and 'int'", "invalidated": false, "invalidation_reason": null }, @@ -11067,7 +11067,7 @@ "verified_date": "2026-04-02", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.004379, mean_rel=0.022909", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.004379, mean_rel=0.022909", "invalidated": false, "invalidation_reason": null }, @@ -11147,7 +11147,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 6/32 components failed (6 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 6/32 components failed (6 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11157,7 +11157,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 6/32 components failed (6 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 6/32 components failed (6 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11167,7 +11167,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 5/32 components failed (4 critical, 1 medium)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 5/32 components failed (4 critical, 1 medium)", "invalidated": false, "invalidation_reason": null }, @@ -11177,7 +11177,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 4/32 components failed (4 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 4/32 components failed (4 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11217,7 +11217,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11227,7 +11227,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load trl-internal-testing/tiny-DeepseekV3ForCausalL", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load trl-internal-testing/tiny-DeepseekV3ForCausalL", "invalidated": false, "invalidation_reason": null }, @@ -11237,7 +11237,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load katuni4ka/tiny-random-deepseek-v3:\n(Request ID", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load katuni4ka/tiny-random-deepseek-v3:\n(Request ID", "invalidated": false, "invalidation_reason": null }, @@ -11247,7 +11247,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load optimum-intel-internal-testing/tiny-random-dee", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load optimum-intel-internal-testing/tiny-random-dee", "invalidated": false, "invalidation_reason": null }, @@ -11257,7 +11257,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11267,7 +11267,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11277,7 +11277,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11287,7 +11287,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11297,7 +11297,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11317,7 +11317,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 10/28 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 10/28 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11337,7 +11337,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 4/22 components failed (4 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 4/22 components failed (4 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11347,7 +11347,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 4/22 components failed (4 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 4/22 components failed (4 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11357,7 +11357,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 10/24 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 10/24 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11367,7 +11367,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 10/24 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 10/24 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11377,7 +11377,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 10/24 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 10/24 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11397,7 +11397,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/16 components failed (1 medium)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/16 components failed (1 medium)", "invalidated": false, "invalidation_reason": null }, @@ -11437,7 +11437,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/16 components failed (1 medium)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/16 components failed (1 medium)", "invalidated": false, "invalidation_reason": null }, @@ -11457,7 +11457,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/12 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/12 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11467,7 +11467,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/12 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/12 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11737,7 +11737,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", "invalidated": false, "invalidation_reason": null }, @@ -11747,7 +11747,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", "invalidated": false, "invalidation_reason": null }, @@ -11757,7 +11757,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=74.608353, mean_rel=1.619285", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=74.608353, mean_rel=1.619285", "invalidated": false, "invalidation_reason": null }, @@ -11767,7 +11767,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=78.619270, mean_rel=1.866265", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=78.619270, mean_rel=1.866265", "invalidated": false, "invalidation_reason": null }, @@ -11777,7 +11777,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=nan, mean_rel=nan", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=nan, mean_rel=nan", "invalidated": false, "invalidation_reason": null }, @@ -11787,7 +11787,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=33.073044, mean_rel=0.316714", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=33.073044, mean_rel=0.316714", "invalidated": false, "invalidation_reason": null }, @@ -11797,7 +11797,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=33.073044, mean_rel=0.316714", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=33.073044, mean_rel=0.316714", "invalidated": false, "invalidation_reason": null }, @@ -11807,7 +11807,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generation failed: 'NoneType' object is not subscriptable", + "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generation failed: 'NoneType' object is not subscriptable", "invalidated": false, "invalidation_reason": null }, @@ -11827,7 +11827,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene \u2014 Forward pass failed: Cannot copy out of meta tensor; no data!", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene — Forward pass failed: Cannot copy out of meta tensor; no data!", "invalidated": false, "invalidation_reason": null }, @@ -11857,7 +11857,7 @@ "verified_date": "2026-05-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/190 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/190 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11867,7 +11867,7 @@ "verified_date": "2026-05-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/190 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/190 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11877,7 +11877,7 @@ "verified_date": "2026-05-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/558 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/558 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12277,7 +12277,7 @@ "verified_date": "2026-06-04", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12307,7 +12307,7 @@ "verified_date": "2026-06-04", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12337,7 +12337,7 @@ "verified_date": "2026-06-05", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Weight magnitude issues: 1 too large", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Weight magnitude issues: 1 too large", "invalidated": false, "invalidation_reason": null }, @@ -12347,7 +12347,7 @@ "verified_date": "2026-06-05", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Weight magnitude issues: 1 too large", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Weight magnitude issues: 1 too large", "invalidated": false, "invalidation_reason": null }, @@ -12447,7 +12447,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/211 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/211 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12467,7 +12467,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 9/167 components failed (9 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 9/167 components failed (9 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12487,7 +12487,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12497,7 +12497,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12507,7 +12507,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12517,7 +12517,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12527,7 +12527,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12537,7 +12537,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12547,7 +12547,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12557,7 +12557,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12567,7 +12567,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12577,7 +12577,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12587,7 +12587,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12597,7 +12597,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12657,7 +12657,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12667,7 +12667,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12677,7 +12677,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12687,7 +12687,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12697,7 +12697,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12707,7 +12707,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12717,7 +12717,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12727,7 +12727,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12737,7 +12737,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12747,7 +12747,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12757,7 +12757,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12767,7 +12767,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12777,7 +12777,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12787,7 +12787,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12797,7 +12797,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12807,7 +12807,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12817,7 +12817,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12827,7 +12827,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12837,7 +12837,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12877,7 +12877,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: MachadoDeCastro/krull-micro does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: MachadoDeCastro/krull-micro does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -12907,7 +12907,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", + "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", "invalidated": false, "invalidation_reason": null }, @@ -12917,7 +12917,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=9.275972, mean_rel=13.166794", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=9.275972, mean_rel=13.166794", "invalidated": false, "invalidation_reason": null }, @@ -13057,7 +13057,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13197,7 +13197,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: [Errno 2] No such file or directory: 'baichuan-inc/Baichuan2-7B-Chat-4bits/pytorch_model.bin'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: [Errno 2] No such file or directory: 'baichuan-inc/Baichuan2-7B-Chat-4bits/pytorch_model.bin'", "invalidated": false, "invalidation_reason": null }, @@ -13217,7 +13217,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", + "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", "invalidated": false, "invalidation_reason": null }, @@ -13227,7 +13227,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", + "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", "invalidated": false, "invalidation_reason": null }, @@ -13247,7 +13247,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", + "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", "invalidated": false, "invalidation_reason": null }, @@ -13277,7 +13277,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13287,7 +13287,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13297,7 +13297,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13307,7 +13307,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13317,7 +13317,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P2=69.2% < 75.0% (f \u2014 59/64 components failed (59 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P2=69.2% < 75.0% (f — 59/64 components failed (59 critical)", "invalidated": false, "invalidation_reason": null }, @@ -13407,7 +13407,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=0.009886, mean_rel=0.980186", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=0.009886, mean_rel=0.980186", "invalidated": false, "invalidation_reason": null }, @@ -13417,7 +13417,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13427,7 +13427,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13447,7 +13447,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=65.931717, mean_rel=2.066483", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=65.931717, mean_rel=2.066483", "invalidated": false, "invalidation_reason": null }, @@ -13457,7 +13457,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=22.915417, mean_rel=11.391559", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=22.915417, mean_rel=11.391559", "invalidated": false, "invalidation_reason": null }, @@ -13477,7 +13477,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.003898, mean_rel=0.027017", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.003898, mean_rel=0.027017", "invalidated": false, "invalidation_reason": null }, @@ -13577,7 +13577,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 4/24 components failed (4 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 4/24 components failed (4 high)", "invalidated": false, "invalidation_reason": null }, @@ -13667,7 +13667,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "invalidated": false, "invalidation_reason": null }, @@ -13817,7 +13817,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -13837,7 +13837,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -13847,7 +13847,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=38.5% < 75.0% (failed: \u2014 1/15 components failed (1 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=38.5% < 75.0% (failed: — 1/15 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -13867,7 +13867,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 1152 (input tensor's size at dimension 0), but go", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 1152 (input tensor's size at dimension 0), but go", "invalidated": false, "invalidation_reason": null }, @@ -14077,7 +14077,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 12/87 components failed (12 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 12/87 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14107,7 +14107,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 12/87 components failed (12 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 12/87 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14157,7 +14157,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 144/196 components failed (144 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 144/196 components failed (144 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14287,7 +14287,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=59.411900, mean_rel=0.569168", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=59.411900, mean_rel=0.569168", "invalidated": false, "invalidation_reason": null }, @@ -14297,7 +14297,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=59.402462, mean_rel=0.441563", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=59.402462, mean_rel=0.441563", "invalidated": false, "invalidation_reason": null }, @@ -14307,7 +14307,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=33.204865, mean_rel=0.370595", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=33.204865, mean_rel=0.370595", "invalidated": false, "invalidation_reason": null }, @@ -14317,7 +14317,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=32.255035, mean_rel=0.318908", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=32.255035, mean_rel=0.318908", "invalidated": false, "invalidation_reason": null }, @@ -14327,7 +14327,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -14337,7 +14337,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=31.355835, mean_rel=0.607207", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=31.355835, mean_rel=0.607207", "invalidated": false, "invalidation_reason": null }, @@ -14347,7 +14347,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 73.9/100 (avg perplexity: 10.2) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 73.9/100 (avg perplexity: 10.2) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -14357,7 +14357,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", "invalidated": false, "invalidation_reason": null }, @@ -14367,7 +14367,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 70/609 components failed (70 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 70/609 components failed (70 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14377,7 +14377,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 70/609 components failed (70 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 70/609 components failed (70 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14387,7 +14387,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 60/534 components failed (60 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 60/534 components failed (60 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14467,7 +14467,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "invalidated": false, "invalidation_reason": null }, @@ -14477,7 +14477,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=16.391922, mean_rel=3.526243", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=16.391922, mean_rel=3.526243", "invalidated": false, "invalidation_reason": null }, @@ -14487,7 +14487,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=18.059958, mean_rel=2.710044", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=18.059958, mean_rel=2.710044", "invalidated": false, "invalidation_reason": null }, @@ -14517,7 +14517,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=4.489960, mean_rel=1.238444", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=4.489960, mean_rel=1.238444", "invalidated": false, "invalidation_reason": null }, @@ -14527,7 +14527,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=20.307718, mean_rel=6.347236", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=20.307718, mean_rel=6.347236", "invalidated": false, "invalidation_reason": null }, @@ -14567,7 +14567,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14597,7 +14597,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14607,7 +14607,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14617,7 +14617,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14627,7 +14627,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14637,7 +14637,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14647,7 +14647,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'type'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'type'", "invalidated": false, "invalidation_reason": null }, @@ -14697,7 +14697,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -14787,7 +14787,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: argument 'vocab': 'dict' object cannot be converted to 'Sequence'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: argument 'vocab': 'dict' object cannot be converted to 'Sequence'", "invalidated": false, "invalidation_reason": null }, @@ -14857,7 +14857,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -14877,7 +14877,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 256 (input tensor's size at dimension 0), but got", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 256 (input tensor's size at dimension 0), but got", "invalidated": false, "invalidation_reason": null }, @@ -14997,7 +14997,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.005348, mean_rel=0.000007", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.005348, mean_rel=0.000007", "invalidated": false, "invalidation_reason": null }, @@ -15067,7 +15067,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=75.0% but required tests failed \u2014 Tensors differ: max_diff=378.613281, mean_rel=0.057195", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=75.0% but required tests failed — Tensors differ: max_diff=378.613281, mean_rel=0.057195", "invalidated": false, "invalidation_reason": null }, @@ -15227,7 +15227,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", "invalidated": false, "invalidation_reason": null }, @@ -15237,7 +15237,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=39.516827, mean_rel=0.391392", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=39.516827, mean_rel=0.391392", "invalidated": false, "invalidation_reason": null }, @@ -15247,7 +15247,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=63.272919, mean_rel=0.497796", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=63.272919, mean_rel=0.497796", "invalidated": false, "invalidation_reason": null }, @@ -15257,7 +15257,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=34.928375, mean_rel=0.262157", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=34.928375, mean_rel=0.262157", "invalidated": false, "invalidation_reason": null }, @@ -15267,7 +15267,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=29.677444, mean_rel=0.238732", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=29.677444, mean_rel=0.238732", "invalidated": false, "invalidation_reason": null }, @@ -15277,7 +15277,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 45.4/100 (avg perplexity: 196.5) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 45.4/100 (avg perplexity: 196.5) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -15287,7 +15287,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=50.417328, mean_rel=0.326480", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=50.417328, mean_rel=0.326480", "invalidated": false, "invalidation_reason": null }, @@ -15297,7 +15297,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=34.771893, mean_rel=0.288610", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=34.771893, mean_rel=0.288610", "invalidated": false, "invalidation_reason": null }, @@ -15307,7 +15307,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=61.738430, mean_rel=0.447178", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=61.738430, mean_rel=0.447178", "invalidated": false, "invalidation_reason": null }, @@ -15407,7 +15407,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "invalidated": false, "invalidation_reason": null }, @@ -15427,7 +15427,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=4.789991, mean_rel=1.159405", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=4.789991, mean_rel=1.159405", "invalidated": false, "invalidation_reason": null }, @@ -15487,7 +15487,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-1b-ONNX-web does not appear to have a file named pytorch_model.bin or model.s", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-1b-ONNX-web does not appear to have a file named pytorch_model.bin or model.s", "invalidated": false, "invalidation_reason": null }, @@ -15517,7 +15517,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "invalidated": false, "invalidation_reason": null }, @@ -15527,7 +15527,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -15537,7 +15537,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g \u2014 144/307 components failed (144 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g — 144/307 components failed (144 critical)", "invalidated": false, "invalidation_reason": null }, @@ -15567,7 +15567,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -15577,7 +15577,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "invalidated": false, "invalidation_reason": null }, @@ -15587,7 +15587,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: LiquidAI/LFM2-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: LiquidAI/LFM2-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -15597,7 +15597,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: LiquidAI/LFM2.5-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: LiquidAI/LFM2.5-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -15617,7 +15617,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": false, "invalidation_reason": null }, @@ -15957,7 +15957,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", "invalidated": false, "invalidation_reason": null }, @@ -15967,7 +15967,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", "invalidated": false, "invalidation_reason": null }, @@ -15977,7 +15977,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -15987,7 +15987,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=6.747103, mean_rel=0.054769", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=6.747103, mean_rel=0.054769", "invalidated": false, "invalidation_reason": null }, @@ -15997,7 +15997,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -16007,7 +16007,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -16017,7 +16017,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.479654, mean_rel=0.052641", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.479654, mean_rel=0.052641", "invalidated": false, "invalidation_reason": null }, @@ -16027,7 +16027,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -16037,7 +16037,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=6.650925, mean_rel=0.050073", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=6.650925, mean_rel=0.050073", "invalidated": false, "invalidation_reason": null }, @@ -16047,7 +16047,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -16057,7 +16057,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.641898, mean_rel=0.054789", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.641898, mean_rel=0.054789", "invalidated": false, "invalidation_reason": null }, @@ -16067,7 +16067,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.808517, mean_rel=0.051374", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.808517, mean_rel=0.051374", "invalidated": false, "invalidation_reason": null }, @@ -16307,7 +16307,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/tiny-random-olmo-hf does not appear to have a file named pytorch_model.bin or model.safet", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/tiny-random-olmo-hf does not appear to have a file named pytorch_model.bin or model.safet", "invalidated": false, "invalidation_reason": null }, @@ -16327,7 +16327,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 32/68 components failed (32 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 32/68 components failed (32 critical)", "invalidated": false, "invalidation_reason": null }, @@ -16337,7 +16337,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "invalidated": false, "invalidation_reason": null }, @@ -16347,7 +16347,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "invalidated": false, "invalidation_reason": null }, @@ -16357,7 +16357,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.013494, mean_rel=0.006767", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.013494, mean_rel=0.006767", "invalidated": false, "invalidation_reason": null }, @@ -16367,7 +16367,7 @@ "verified_date": "2026-06-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed \u2014 Tensors differ: max_diff=0.437500, mean_rel=0.223633", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed — Tensors differ: max_diff=0.437500, mean_rel=0.223633", "invalidated": false, "invalidation_reason": null }, @@ -16377,7 +16377,7 @@ "verified_date": "2026-06-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.013494, mean_rel=0.006767", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.013494, mean_rel=0.006767", "invalidated": false, "invalidation_reason": null }, @@ -16387,7 +16387,7 @@ "verified_date": "2026-06-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.020484, mean_rel=0.006617", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.020484, mean_rel=0.006617", "invalidated": false, "invalidation_reason": null }, @@ -16397,7 +16397,7 @@ "verified_date": "2026-07-01", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 96/171 components failed (96 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 96/171 components failed (96 critical)", "invalidated": true, "invalidation_reason": "Superseded by the clean re-run (P1=100) after the component-benchmark fix that skips SSM mixer-internal submodules; the 96/171 component failures were the isolated harness feeding d_model-shaped inputs to SSM-internal projections, not a real divergence." }, @@ -16427,7 +16427,7 @@ "verified_date": "2026-07-01", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=0.375000, mean_rel=0.002045", + "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence — Tensors differ: max_diff=0.375000, mean_rel=0.002045", "invalidated": true, "invalidation_reason": "bf16 precision of compatibility-mode center_unembed (root-caused by toggle: off=0.000 both dtypes, fp32=4.2e-5); superseded by the clean fp32 run (P3=100). Not an algorithmic bug." }, @@ -16547,7 +16547,7 @@ "verified_date": "2026-07-01", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 24/51 components failed (24 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 24/51 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -16577,7 +16577,7 @@ "verified_date": "2026-07-01", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 50/99 components failed (50 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 50/99 components failed (50 critical)", "invalidated": false, "invalidation_reason": null }, @@ -16897,7 +16897,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/16 components failed (2 high)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/16 components failed (2 high)", "invalidated": false, "invalidation_reason": null }, @@ -16907,7 +16907,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/16 components failed (2 high)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/16 components failed (2 high)", "invalidated": false, "invalidation_reason": null }, @@ -16977,7 +16977,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass) \u2014 Forward pass failed: Could not infer dtype of NoneType", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass) — Forward pass failed: Could not infer dtype of NoneType", "invalidated": false, "invalidation_reason": null }, @@ -16987,7 +16987,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass) \u2014 Forward pass failed: Could not infer dtype of NoneType", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass) — Forward pass failed: Could not infer dtype of NoneType", "invalidated": false, "invalidation_reason": null }, @@ -17007,7 +17007,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/75 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/75 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17017,7 +17017,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/75 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/75 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17027,7 +17027,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/75 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/75 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17037,7 +17037,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/75 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/75 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17067,7 +17067,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Cannot build Piece from string \":0\"", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Cannot build Piece from string \":0\"", "invalidated": false, "invalidation_reason": null }, @@ -17077,7 +17077,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed \u2014 Tensors differ: max_diff=0.084499, mean_rel=1.582682", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed — Tensors differ: max_diff=0.084499, mean_rel=1.582682", "invalidated": false, "invalidation_reason": null }, @@ -17087,7 +17087,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed \u2014 Tensors differ: max_diff=0.096940, mean_rel=2.925628", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed — Tensors differ: max_diff=0.096940, mean_rel=2.925628", "invalidated": false, "invalidation_reason": null }, @@ -17097,7 +17097,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=0.079056, mean_rel=0.949212", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=0.079056, mean_rel=0.949212", "invalidated": false, "invalidation_reason": null }, @@ -17117,7 +17117,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=30.8% < 75.0% (failed: \u2014 5/12 components failed (5 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=30.8% < 75.0% (failed: — 5/12 components failed (5 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17127,7 +17127,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=30.8% < 75.0% (failed: \u2014 5/12 components failed (5 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=30.8% < 75.0% (failed: — 5/12 components failed (5 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17147,7 +17147,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/16 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/16 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17157,7 +17157,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/16 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/16 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17177,7 +17177,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=13.567083, mean_rel=3.490963", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=13.567083, mean_rel=3.490963", "invalidated": false, "invalidation_reason": null }, @@ -17187,7 +17187,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=13.567083, mean_rel=3.490963", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=13.567083, mean_rel=3.490963", "invalidated": false, "invalidation_reason": null }, @@ -17227,7 +17227,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=78.9% but requir \u2014 30/184 components failed (30 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=78.9% but requir — 30/184 components failed (30 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17237,7 +17237,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=78.9% but requir \u2014 30/184 components failed (30 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=78.9% but requir — 30/184 components failed (30 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17247,7 +17247,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=78.9% but required tests failed \u2014 Tensors differ: max_diff=27.805214, mean_rel=17.542879", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=78.9% but required tests failed — Tensors differ: max_diff=27.805214, mean_rel=17.542879", "invalidated": false, "invalidation_reason": null }, @@ -17257,7 +17257,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Attention output weights not well-centered (worst_mean=0.061788)", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Attention output weights not well-centered (worst_mean=0.061788)", "invalidated": false, "invalidation_reason": null }, @@ -17267,7 +17267,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Attention output weights not well-centered (worst_mean=0.061788)", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Attention output weights not well-centered (worst_mean=0.061788)", "invalidated": false, "invalidation_reason": null }, @@ -17277,7 +17277,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Attention output weights not well-centered (worst_mean=0.061788)", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Attention output weights not well-centered (worst_mean=0.061788)", "invalidated": false, "invalidation_reason": null }, @@ -17307,7 +17307,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 Error running comprehensive component benchmark: index 2 is out of range", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — Error running comprehensive component benchmark: index 2 is out of range", "invalidated": false, "invalidation_reason": null }, @@ -17367,7 +17367,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Checkpoint ships no lm_head.weight with tie_word_embeddings=false, so HF randomly re-initializes the LM head on every load (Google released it as a pretraining artifact requiring fine-tuning); the nondeterministic unembed is the only P1 failure \u2014 all 233 other components including the local-attention encoder pass. Not an adapter bug.", + "notes": "Checkpoint ships no lm_head.weight with tie_word_embeddings=false, so HF randomly re-initializes the LM head on every load (Google released it as a pretraining artifact requiring fine-tuning); the nondeterministic unembed is the only P1 failure — all 233 other components including the local-attention encoder pass. Not an adapter bug.", "invalidated": false, "invalidation_reason": null }, @@ -17437,7 +17437,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Verified on a local snapshot of yujiepan/llama-4-tiny-random with text_config.attn_temperature_tuning coerced to bool \u2014 the upstream config declares it as int 4, which transformers 5.x strict config validation rejects. P7 skipped: the tiny ships no processor files. P3=95 (attention_output_centering worst_mean=0.099 on random weights). Official Scout/Maverick checkpoints registered for big-hardware verification.", + "notes": "Verified on a local snapshot of yujiepan/llama-4-tiny-random with text_config.attn_temperature_tuning coerced to bool — the upstream config declares it as int 4, which transformers 5.x strict config validation rejects. P7 skipped: the tiny ships no processor files. P3=95 (attention_output_centering worst_mean=0.099 on random weights). Official Scout/Maverick checkpoints registered for big-hardware verification.", "invalidated": false, "invalidation_reason": null }, @@ -17517,7 +17517,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=30.078743, mean_rel=0.607381", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=30.078743, mean_rel=0.607381", "invalidated": false, "invalidation_reason": null }, @@ -17537,7 +17537,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=30.078743, mean_rel=0.607381", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=30.078743, mean_rel=0.607381", "invalidated": false, "invalidation_reason": null }, @@ -17567,7 +17567,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=27.217707, mean_rel=1.057937", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=27.217707, mean_rel=1.057937", "invalidated": false, "invalidation_reason": null }, @@ -17597,7 +17597,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g \u2014 90/154 components failed (90 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g — 90/154 components failed (90 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17777,7 +17777,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -17797,7 +17797,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/462 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/462 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17907,7 +17907,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -17927,7 +17927,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=30.8% < 75.0% (failed: hook_functional \u2014 Forward pass failed: index out of range in self", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=30.8% < 75.0% (failed: hook_functional — Forward pass failed: index out of range in self", "invalidated": false, "invalidation_reason": null }, @@ -18027,7 +18027,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Input must be a List[Union[str, AddedToken]]", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Input must be a List[Union[str, AddedToken]]", "invalidated": false, "invalidation_reason": null }, @@ -18097,7 +18097,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: DreamGenerationConfig.validate() got an unexpected keyword argument 'user_set_attributes'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: DreamGenerationConfig.validate() got an unexpected keyword argument 'user_set_attributes'", "invalidated": false, "invalidation_reason": null }, @@ -18187,7 +18187,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 76.4/100 (avg perplexity: 16.6) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 76.4/100 (avg perplexity: 16.6) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -18197,7 +18197,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=0.207234, mean_rel=0.000825", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=0.207234, mean_rel=0.000825", "invalidated": false, "invalidation_reason": null }, @@ -18237,7 +18237,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=19.557707, mean_rel=0.281903", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=19.557707, mean_rel=0.281903", "invalidated": false, "invalidation_reason": null }, @@ -18257,7 +18257,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=19.557709, mean_rel=0.281903", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=19.557709, mean_rel=0.281903", "invalidated": false, "invalidation_reason": null }, @@ -18267,7 +18267,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=19.557707, mean_rel=0.281903", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=19.557707, mean_rel=0.281903", "invalidated": false, "invalidation_reason": null }, @@ -18297,7 +18297,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'GiddForDiffusionLM' object has no attribute 'all_tied_weights_keys'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'GiddForDiffusionLM' object has no attribute 'all_tied_weights_keys'", "invalidated": false, "invalidation_reason": null }, @@ -18307,7 +18307,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'GiddModel' object has no attribute 'weight'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'GiddModel' object has no attribute 'weight'", "invalidated": false, "invalidation_reason": null }, @@ -18317,7 +18317,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=26.121746, mean_rel=4.879314", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=26.121746, mean_rel=4.879314", "invalidated": false, "invalidation_reason": null }, @@ -18327,7 +18327,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/136 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/136 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18337,7 +18337,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/268 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/268 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18347,7 +18347,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/136 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/136 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18357,7 +18357,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/268 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/268 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18387,7 +18387,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", "invalidated": false, "invalidation_reason": null }, @@ -18397,7 +18397,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/100 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/100 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18417,7 +18417,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=42.9% < 75.0% (failed: hook_functional \u2014 Forward pass failed: 'tuple' object has no attribute 'dtype'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=42.9% < 75.0% (failed: hook_functional — Forward pass failed: 'tuple' object has no attribute 'dtype'", "invalidated": false, "invalidation_reason": null }, @@ -18447,7 +18447,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 36/475 components failed (36 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 36/475 components failed (36 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18467,7 +18467,7 @@ "verified_date": "2026-07-12", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log \u2014 8/98 components failed (8 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log — 8/98 components failed (8 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18477,7 +18477,7 @@ "verified_date": "2026-07-12", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18487,7 +18487,7 @@ "verified_date": "2026-07-12", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18497,7 +18497,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18507,7 +18507,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18517,7 +18517,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18527,7 +18527,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 20/114 components failed (20 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 20/114 components failed (20 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18537,7 +18537,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=23.349285, mean_rel=0.575189", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=23.349285, mean_rel=0.575189", "invalidated": false, "invalidation_reason": null }, @@ -18627,7 +18627,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -18637,7 +18637,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -18647,7 +18647,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -18787,7 +18787,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=11.357496, mean_rel=4.069944", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=11.357496, mean_rel=4.069944", "invalidated": false, "invalidation_reason": null }, @@ -18837,7 +18837,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene \u2014 Forward pass failed: create_causal_mask() got an unexpected keyword argument 'input_embeds'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene — Forward pass failed: create_causal_mask() got an unexpected keyword argument 'input_embeds'", "invalidated": false, "invalidation_reason": null }, @@ -18847,7 +18847,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'FalconDecoderLayer' object has no attribute 'ln_attn'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'FalconDecoderLayer' object has no attribute 'ln_attn'", "invalidated": false, "invalidation_reason": null }, @@ -18917,7 +18917,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene \u2014 Forward pass failed: index out of range in self", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene — Forward pass failed: index out of range in self", "invalidated": false, "invalidation_reason": null }, @@ -18947,7 +18947,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -19257,7 +19257,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 Error running comprehensive component benchmark: Component attn not found in blocks.0 components", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — Error running comprehensive component benchmark: Component attn not found in blocks.0 components", "invalidated": false, "invalidation_reason": null }, @@ -19297,7 +19297,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=32.690491, mean_rel=0.409154", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=32.690491, mean_rel=0.409154", "invalidated": false, "invalidation_reason": null }, @@ -19327,7 +19327,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=31.2% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook \u2014 Logits computation failed: Invalid positional_embedding_type passed in relative_positional_bias", + "notes": "Below threshold: P2=31.2% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook — Logits computation failed: Invalid positional_embedding_type passed in relative_positional_bias", "invalidated": false, "invalidation_reason": null }, @@ -19347,7 +19347,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/126 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/126 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -19457,7 +19457,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", "invalidated": false, "invalidation_reason": null }, @@ -19467,7 +19467,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=2.599856, mean_rel=0.102672", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=2.599856, mean_rel=0.102672", "invalidated": false, "invalidation_reason": null }, @@ -19517,7 +19517,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=83.3% but required tests failed: logits_equivalence, loss_equivalence \u2014 Unembed matrix not well-centered (mean=0.075038)", + "notes": "Below threshold: P3=83.3% but required tests failed: logits_equivalence, loss_equivalence — Unembed matrix not well-centered (mean=0.075038)", "invalidated": false, "invalidation_reason": null }, @@ -19927,7 +19927,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=17.494274, mean_rel=8.712387", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=17.494274, mean_rel=8.712387", "invalidated": false, "invalidation_reason": null }, @@ -19937,7 +19937,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=18.492882, mean_rel=11.615888", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=18.492882, mean_rel=11.615888", "invalidated": false, "invalidation_reason": null }, @@ -19947,7 +19947,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=17.494274, mean_rel=8.712387", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=17.494274, mean_rel=8.712387", "invalidated": false, "invalidation_reason": null }, @@ -19957,7 +19957,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=18.492882, mean_rel=11.615888", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=18.492882, mean_rel=11.615888", "invalidated": false, "invalidation_reason": null }, @@ -20187,7 +20187,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=30.8% < 75.0% (failed: hook_functional \u2014 Forward pass failed: index out of range in self", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=30.8% < 75.0% (failed: hook_functional — Forward pass failed: index out of range in self", "invalidated": false, "invalidation_reason": null }, @@ -20387,7 +20387,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/23 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/23 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -20397,7 +20397,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=90.0% but requir \u2014 1/15 components failed (1 low)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=90.0% but requir — 1/15 components failed (1 low)", "invalidated": false, "invalidation_reason": null }, @@ -20417,7 +20417,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/23 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/23 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -20427,7 +20427,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=90.0% but requir \u2014 1/15 components failed (1 low)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=90.0% but requir — 1/15 components failed (1 low)", "invalidated": false, "invalidation_reason": null }, @@ -20607,7 +20607,7 @@ "verified_date": "2026-07-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/107 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/107 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -20777,7 +20777,7 @@ "verified_date": "2026-07-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P8=0.0% < 75.0% (failed: audio_text_forward) \u2014 Audio-conditioned forward failed: 'NoneType' object has no attribute '_attn_implementation'", + "notes": "Below threshold: P8=0.0% < 75.0% (failed: audio_text_forward) — Audio-conditioned forward failed: 'NoneType' object has no attribute '_attn_implementation'", "invalidated": false, "invalidation_reason": null }, @@ -20907,7 +20907,7 @@ "verified_date": "2026-07-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P8=0.0% < 75.0% (failed: audio_forward, audio_cache, audio_representation_stability \u2014 Audio forward pass failed: Dimension out of range (expected to be in range of [-3, 2], but got 3)", + "notes": "Below threshold: P8=0.0% < 75.0% (failed: audio_forward, audio_cache, audio_representation_stability — Audio forward pass failed: Dimension out of range (expected to be in range of [-3, 2], but got 3)", "invalidated": false, "invalidation_reason": null }, @@ -20957,7 +20957,7 @@ "verified_date": "2026-07-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", "invalidated": false, "invalidation_reason": null }, @@ -21687,7 +21687,7 @@ "verified_date": "2026-08-19", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=23.143579, mean_rel=9.020543", + "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=23.143579, mean_rel=9.020543", "invalidated": false, "invalidation_reason": null }, @@ -21697,7 +21697,7 @@ "verified_date": "2026-08-19", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=72.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=26.260244, mean_rel=3.783418", + "notes": "Below threshold: P3=72.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=26.260244, mean_rel=3.783418", "invalidated": false, "invalidation_reason": null }, @@ -22801,13 +22801,193 @@ "invalidated": false, "invalidation_reason": null }, + { + "model_id": "openai-community/gpt2", + "architecture_id": "GPT2LMHeadModel", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "continuation" + }, + { + "model_id": "EleutherAI/pythia-70m", + "architecture_id": "GPTNeoXForCausalLM", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "continuation" + }, + { + "model_id": "Helsinki-NLP/opus-mt-nl-en", + "architecture_id": "MarianMTModel", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@nl-en" + }, + { + "model_id": "google-t5/t5-small", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@en-de" + }, + { + "model_id": "google-t5/t5-base", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@en-de" + }, + { + "model_id": "facebook/bart-large-cnn", + "architecture_id": "BartForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:summarization" + }, + { + "model_id": "bigscience/mt0-base", + "architecture_id": "MT5ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification passed, but text quality poor (P4=26.8). Needs review", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:instruction" + }, + { + "model_id": "google/long-t5-tglobal-base", + "architecture_id": "LongT5ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification passed, but text quality poor (P4=48.4). Needs review", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:denoise" + }, + { + "model_id": "google/pegasus-xsum", + "architecture_id": "PegasusForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:summarization" + }, + { + "model_id": "facebook/m2m100_418M", + "architecture_id": "M2M100ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@en-de" + }, + { + "model_id": "Qwen/Qwen2.5-0.5B-Instruct", + "architecture_id": "Qwen2ForCausalLM", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "chat" + }, + { + "model_id": "google/gemma-2-2b-it", + "architecture_id": "Gemma2ForCausalLM", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "chat" + }, + { + "model_id": "facebook/mbart-large-50-many-to-many-mmt", + "architecture_id": "MBartForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@en-de" + }, + { + "model_id": "facebook/mbart-large-50", + "architecture_id": "MBartForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:denoise" + }, + { + "model_id": "facebook/mbart-large-cc25", + "architecture_id": "MBartForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification passed, but text quality poor (P4=25.0). Needs review", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:denoise" + }, { "model_id": "google-bert/bert-base-cased", "architecture_id": "BertForMaskedLM", "verified_date": "2026-08-20", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva \u2014 1/79 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva — 1/79 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -22817,7 +22997,7 @@ "verified_date": "2026-08-20", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva \u2014 1/79 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva — 1/79 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -22827,7 +23007,7 @@ "verified_date": "2026-08-20", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva \u2014 2/79 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva — 2/79 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -22837,7 +23017,7 @@ "verified_date": "2026-08-20", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva \u2014 1/79 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva — 1/79 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -22847,7 +23027,7 @@ "verified_date": "2026-08-20", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=68.8% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=29.631586, mean_rel=1.923156", + "notes": "Below threshold: P2=68.8% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=29.631586, mean_rel=1.923156", "invalidated": false, "invalidation_reason": null }, diff --git a/transformer_lens/tools/model_registry/hf_scraper.py b/transformer_lens/tools/model_registry/hf_scraper.py index b34f470346..b48042af72 100644 --- a/transformer_lens/tools/model_registry/hf_scraper.py +++ b/transformer_lens/tools/model_registry/hf_scraper.py @@ -36,6 +36,16 @@ from pathlib import Path from typing import Optional +from transformer_lens.benchmarks.text_quality_profiles import ( + ARCHITECTURE_PROFILE_KINDS, + MODEL_PROFILE_OVERRIDES, + HFSignals, + extract_languages, + is_default_profile, + profile_from_hf_signals, + resolve_profile, +) + from . import HF_SUPPORTED_ARCHITECTURES from .registry_io import is_quantized_model @@ -158,9 +168,33 @@ def _load_existing_gaps(output_dir: Path) -> dict[str, dict]: return by_arch -def _build_model_entry(model_id: str, architecture_id: str) -> dict: - """Build a model entry dict matching the ModelEntry schema.""" - return { +def _extract_profile_signals(model_info) -> HFSignals: # type: ignore[no-untyped-def] + """Distill pipeline_tag/tags/cardData off a listing payload (no extra request). + + Args: + model_info: ModelInfo object from list_models(expand=[..., 'pipeline_tag', + 'tags', 'cardData']) + """ + pipeline_tag = getattr(model_info, "pipeline_tag", None) + tags = tuple(getattr(model_info, "tags", None) or []) + card_data = getattr(model_info, "card_data", None) + card_language = getattr(card_data, "language", None) if card_data is not None else None + if card_language is None and card_data is not None and hasattr(card_data, "get"): + card_language = card_data.get("language") + languages = extract_languages(card_language, tags) + return HFSignals(pipeline_tag=pipeline_tag, languages=languages, tags=tags) + + +def _build_model_entry( + model_id: str, architecture_id: str, signals: Optional[HFSignals] = None +) -> dict: + """Build a model entry dict matching the ModelEntry schema. + + ``signals``, when given, resolves and stores a sparse ``prompt_profile`` key + (omitted when it's just the default) and warns on tag/curation disagreement + — the warning is how curation gaps (missing override/architecture rule) surface. + """ + entry = { "architecture_id": architecture_id, "model_id": model_id, "status": 0, @@ -175,6 +209,25 @@ def _build_model_entry(model_id: str, architecture_id: str) -> dict: "phase8_score": None, "phase9_score": None, } + if signals is not None: + hinted = profile_from_hf_signals(model_id, architecture_id, signals) + resolved = resolve_profile(model_id, architecture_id, signals=signals) + deliberately_curated = ( + model_id in MODEL_PROFILE_OVERRIDES or architecture_id in ARCHITECTURE_PROFILE_KINDS + ) + if hinted is not None and hinted.kind != resolved.kind and not deliberately_curated: + # A disagreement nothing deliberate explains is a curation gap. + logger.warning( + f"Profile mismatch for {model_id} ({architecture_id}): Hub tags say " + f"{hinted.kind!r}, curation resolves {resolved.kind!r}" + ) + if not is_default_profile(resolved): + # Keep key position consistent with ModelEntry.to_dict (after note). + items = list(entry.items()) + items.insert([k for k, _ in items].index("note") + 1, ("prompt_profile", str(resolved))) + entry.clear() + entry.update(items) + return entry def _canonical_author_sweep( @@ -182,6 +235,7 @@ def _canonical_author_sweep( supported_models: list[dict], seen_models: set[str], architecture: Optional[str] = None, + refresh_profiles: bool = False, ) -> int: """Admit canonical-org supported-arch models regardless of downloads. Returns count added. @@ -201,15 +255,31 @@ def _canonical_author_sweep( if architecture is not None and architecture not in expected_archs: continue try: - models_iter = api.list_models(author=author, expand=["config", "safetensors"]) + models_iter = api.list_models( + author=author, + expand=["config", "safetensors", "pipeline_tag", "tags", "cardData"], + ) except Exception as exc: # pragma: no cover — network/transient logger.warning(f"Canonical sweep: list_models(author={author!r}) failed: {exc}") continue # Iterate paginated results; a single timeout shouldn't lose every prior author. + existing_by_id = {m["model_id"]: m for m in supported_models} if refresh_profiles else {} try: for model in models_iter: if model.id in seen_models: + # Below-threshold canonical models are reachable only here; + # the main scan's backfill never sees them. + if refresh_profiles: + existing_entry = existing_by_id.get(model.id) + if existing_entry is not None and "prompt_profile" not in existing_entry: + resolved = resolve_profile( + model.id, + existing_entry.get("architecture_id"), + signals=_extract_profile_signals(model), + ) + if not is_default_profile(resolved): + existing_entry["prompt_profile"] = str(resolved) continue if is_quantized_model(model.id): continue @@ -221,7 +291,8 @@ def _canonical_author_sweep( # Reject e.g. mistralai's non-Mistral checkpoints. if model_arch not in expected_archs: continue - supported_models.append(_build_model_entry(model.id, model_arch)) + signals = _extract_profile_signals(model) + supported_models.append(_build_model_entry(model.id, model_arch, signals)) seen_models.add(model.id) added += 1 logger.info(f"Canonical sweep added: {model.id} ({model_arch})") @@ -242,6 +313,7 @@ def scrape_all_models( min_downloads: int = 500, canonical_sweep: bool = True, architecture: Optional[str] = None, + refresh_profiles: bool = False, ) -> tuple[dict, dict]: """Scrape ALL models from HuggingFace and categorize by architecture. @@ -269,6 +341,9 @@ def scrape_all_models( this class (e.g. ``"LlamaForCausalLM"``). Applies to both the main scan and the canonical-author sweep. Useful for populating the registry after adding a single new adapter without rescanning every architecture. + refresh_profiles: If True, backfill a missing ``prompt_profile`` key onto + already-seen registry entries using the listing payload already in hand — no + extra requests (default: False). Returns: Tuple of (supported_models_dict, architecture_gaps_dict) @@ -292,6 +367,9 @@ def scrape_all_models( # Track all models by architecture (start with existing models) supported_models: list[dict] = list(existing_models) # Preserve existing + # Same dict objects as supported_models — mutating via this index (--refresh-profiles) + # is reflected in the final write. + existing_by_id: dict[str, dict] = {m["model_id"]: m for m in supported_models} unsupported_arch_counts: dict[str, int] = {} # arch -> count unsupported_arch_samples: dict[str, list[str]] = {} # arch -> top model IDs unsupported_arch_downloads: dict[str, int] = {} # arch -> total downloads @@ -361,19 +439,22 @@ def scrape_all_models( logger.info("Will scan ALL new models (this may take a while)") try: - # Use expand=['config', 'safetensors'] to get architecture and parameter - # count data inline with the listing, avoiding per-model API calls. - # With ~1000 models per page, a full scan of 200K+ models needs only - # ~200 paginated requests (well within the 1000 req / 5 min limit). - # Use ``filter`` rather than ``pipeline_tag`` so encoder-decoder models - # are discoverable: HF assigns T5/mT5 a primary pipeline_tag of - # "translation" (or None for mT5) and only lists "text2text-generation" - # in the broader tag list. ``filter`` matches against tags, ``pipeline_tag`` - # only against the canonical primary tag. + # Use expand=['config', 'safetensors', 'pipeline_tag', 'tags', 'cardData'] to get + # architecture, parameter count, and prompt-profile signals inline with the + # listing, avoiding per-model API calls. With ~1000 models per page, a full + # scan of 200K+ models needs only ~200 paginated requests (well within the + # 1000 req / 5 min limit). + # Use ``filter`` rather than ``pipeline_tag`` (the query param) so + # encoder-decoder models are discoverable: HF assigns T5/mT5 a primary + # pipeline_tag of "translation" (or None for mT5) and only lists + # "text2text-generation" in the broader tag list. ``filter`` matches against + # tags, ``pipeline_tag`` only against the canonical primary tag. The + # expanded ``pipeline_tag`` *field* below is a different thing — it's per-model + # metadata fed to profile_from_hf_signals, not a query filter. list_kwargs: dict = { "filter": task, "sort": "downloads", - "expand": ["config", "safetensors"], + "expand": ["config", "safetensors", "pipeline_tag", "tags", "cardData"], } if max_models is not None: list_kwargs["limit"] = max_models + len(seen_models) @@ -393,6 +474,19 @@ def scrape_all_models( # Skip if already in our JSON or processed in this run if model.id in seen_models: skipped += 1 + if refresh_profiles: + existing_entry = existing_by_id.get(model.id) + if ( + existing_entry is not None + and "prompt_profile" not in existing_entry + ): + resolved = resolve_profile( + model.id, + existing_entry.get("architecture_id"), + signals=_extract_profile_signals(model), + ) + if not is_default_profile(resolved): + existing_entry["prompt_profile"] = str(resolved) continue # Filter by minimum download count. Since results are sorted @@ -430,7 +524,8 @@ def scrape_all_models( if arch is None: errors += 1 elif arch in HF_SUPPORTED_ARCHITECTURES: - supported_models.append(_build_model_entry(model.id, arch)) + signals = _extract_profile_signals(model) + supported_models.append(_build_model_entry(model.id, arch, signals)) new_supported += 1 else: unsupported_arch_counts[arch] = unsupported_arch_counts.get(arch, 0) + 1 @@ -537,7 +632,11 @@ def scrape_all_models( # Don't lose the main-scan registry on a sweep-time failure. try: canonical_added = _canonical_author_sweep( - api, supported_models, seen_models, architecture=architecture + api, + supported_models, + seen_models, + architecture=architecture, + refresh_profiles=refresh_profiles, ) new_supported += canonical_added logger.info(f"Canonical sweep added {canonical_added} models.") @@ -830,6 +929,12 @@ def main(): "(e.g. 'LlamaForCausalLM'). Use after adding a new adapter to populate the " "registry with that architecture's models without rescanning everything.", ) + parser.add_argument( + "--refresh-profiles", + action="store_true", + help="Backfill a missing prompt_profile key onto already-seen registry entries " + "from the listing payload already in hand (no extra requests).", + ) args = parser.parse_args() @@ -843,6 +948,7 @@ def main(): min_downloads=args.min_downloads, canonical_sweep=not args.no_canonical_sweep, architecture=args.architecture, + refresh_profiles=args.refresh_profiles, ) diff --git a/transformer_lens/tools/model_registry/registry_io.py b/transformer_lens/tools/model_registry/registry_io.py index dfe075e6a3..d52d87a6eb 100644 --- a/transformer_lens/tools/model_registry/registry_io.py +++ b/transformer_lens/tools/model_registry/registry_io.py @@ -11,6 +11,11 @@ from pathlib import Path from typing import Callable, Optional +from transformer_lens.benchmarks.text_quality_profiles import ( + P4_SCORING_VERSION, + is_default_profile, +) + from .verification import VerificationHistory, VerificationRecord logger = logging.getLogger(__name__) @@ -174,7 +179,14 @@ def _get_tl_version() -> Optional[str]: try: import transformer_lens - return getattr(transformer_lens, "__version__", None) + version = getattr(transformer_lens, "__version__", None) + if version: + return str(version) + # The package exports no __version__; installed-distribution + # metadata is the fallback (dev installs record 0.0.0). + from importlib.metadata import version as dist_version + + return dist_version("transformer-lens") except Exception: return None @@ -186,6 +198,7 @@ def update_model_status( note: Optional[str] = None, phase_scores: Optional[dict[int, Optional[float]]] = None, sanitize_fn: Optional[Callable[[Optional[str]], Optional[str]]] = None, + prompt_profile: Optional[str] = None, ) -> bool: """Update a single model entry in supported_models.json. @@ -202,6 +215,10 @@ def update_model_status( note: Optional note for skip/fail reason phase_scores: Phase score dict {1: float, 2: float, 3: float, 4: float} sanitize_fn: Optional callable to sanitize note strings + prompt_profile: Phase-4 prompt profile actually used (e.g. + "task:translation@en-de"). Sparse: the default "continuation" + removes the key (clearing a stale non-default value), None (no + Phase-4 result) leaves it untouched. Returns: True if entry was found/created and updated @@ -235,6 +252,12 @@ def update_model_status( entry[key] = phase_scores[phase_num] elif key not in entry: entry[key] = None + if prompt_profile is not None and is_default_profile(prompt_profile): + entry.pop("prompt_profile", None) + elif prompt_profile is not None: + entry["prompt_profile"] = prompt_profile + if 4 in phase_scores: + entry["p4_scoring_version"] = P4_SCORING_VERSION # Reorder keys so phase scores are always in numerical order _KEY_ORDER = [ "architecture_id", @@ -243,6 +266,8 @@ def update_model_status( "verified_date", "metadata", "note", + "prompt_profile", + "p4_scoring_version", "phase1_score", "phase2_score", "phase3_score", @@ -281,6 +306,20 @@ def update_model_status( "phase9_score": phase_scores.get(9), } ) + new_entry = data["models"][-1] + extras: list[tuple[str, object]] = [] + if prompt_profile is not None and not is_default_profile(prompt_profile): + extras.append(("prompt_profile", prompt_profile)) + if phase_scores.get(4) is not None: + extras.append(("p4_scoring_version", P4_SCORING_VERSION)) + if extras: + # Keep key position consistent with _KEY_ORDER (after "note"). + items = list(new_entry.items()) + idx = [k for k, _ in items].index("note") + 1 + for offset, pair in enumerate(extras): + items.insert(idx + offset, pair) + new_entry.clear() + new_entry.update(items) updated = True if updated: @@ -296,12 +335,28 @@ def update_model_status( return updated +def registry_prompt_profile(model_id: str) -> Optional[str]: + """Stored prompt_profile for a model, or None. Uncached read: the sweep + rewrites the registry between models.""" + try: + data = load_supported_models_raw() + except Exception: + return None + for entry in data.get("models", []): + if entry.get("model_id") == model_id: + profile = entry.get("prompt_profile") + return profile if isinstance(profile, str) else None + return None + + def add_verification_record( model_id: str, arch_id: str, notes: Optional[str] = None, verified_by: str = "verify_models", sanitize_fn: Optional[Callable[[Optional[str]], Optional[str]]] = None, + prompt_profile: Optional[str] = None, + p4_scoring_version: Optional[int] = None, ) -> None: """Append a VerificationRecord to verification_history.json. @@ -325,6 +380,8 @@ def add_verification_record( verified_by=verified_by, transformerlens_version=_get_tl_version(), notes=notes, + prompt_profile=prompt_profile, + p4_scoring_version=p4_scoring_version, ) history = load_verification_history() diff --git a/transformer_lens/tools/model_registry/schemas.py b/transformer_lens/tools/model_registry/schemas.py index 6cd941c154..8b0b00c0c5 100644 --- a/transformer_lens/tools/model_registry/schemas.py +++ b/transformer_lens/tools/model_registry/schemas.py @@ -64,6 +64,8 @@ class ModelEntry: verified_date: Date when verification was performed metadata: Optional metadata from HuggingFace note: Optional note (skip/fail reason, e.g. "Estimated 48 GB exceeds 16 GB limit") + prompt_profile: Phase-4 prompt profile used (e.g. "task:translation@en-de"); + omitted from JSON for the default continuation profile phase1_score: Benchmark Phase 1 score (HF vs Bridge), 0-100 or None phase2_score: Benchmark Phase 2 score (Bridge vs HT unprocessed), 0-100 or None phase3_score: Benchmark Phase 3 score (Bridge vs HT processed), 0-100 or None @@ -79,6 +81,8 @@ class ModelEntry: verified_date: Optional[date] = None metadata: Optional[ModelMetadata] = None note: Optional[str] = None + prompt_profile: Optional[str] = None + p4_scoring_version: Optional[int] = None phase1_score: Optional[float] = None phase2_score: Optional[float] = None phase3_score: Optional[float] = None @@ -88,8 +92,9 @@ class ModelEntry: phase9_score: Optional[float] = None def to_dict(self) -> dict: - """Convert to a JSON-serializable dictionary.""" - return { + """Convert to a JSON-serializable dictionary. prompt_profile is sparse: + omitted when None so default-profile entries carry no key.""" + result = { "architecture_id": self.architecture_id, "model_id": self.model_id, "status": self.status, @@ -104,6 +109,18 @@ def to_dict(self) -> dict: "phase8_score": self.phase8_score, "phase9_score": self.phase9_score, } + extras: list[tuple[str, object]] = [] + if self.prompt_profile is not None: + extras.append(("prompt_profile", self.prompt_profile)) + if self.p4_scoring_version is not None: + extras.append(("p4_scoring_version", self.p4_scoring_version)) + if extras: + note_index = list(result).index("note") + 1 + items = list(result.items()) + for offset, pair in enumerate(extras): + items.insert(note_index + offset, pair) + result = dict(items) + return result @classmethod def from_dict(cls, data: dict) -> "ModelEntry": @@ -128,6 +145,8 @@ def from_dict(cls, data: dict) -> "ModelEntry": verified_date=verified_date, metadata=metadata, note=data.get("note"), + prompt_profile=data.get("prompt_profile"), + p4_scoring_version=data.get("p4_scoring_version"), phase1_score=data.get("phase1_score"), phase2_score=data.get("phase2_score"), phase3_score=data.get("phase3_score"), diff --git a/transformer_lens/tools/model_registry/validate.py b/transformer_lens/tools/model_registry/validate.py index 0151959a9c..d070303ea9 100644 --- a/transformer_lens/tools/model_registry/validate.py +++ b/transformer_lens/tools/model_registry/validate.py @@ -290,6 +290,31 @@ def _validate_model_entry(data: dict, path: str) -> list[ValidationError]: if "note" in data and data["note"] is not None: errors.extend(_validate_string(data["note"], f"{path}.note", min_length=1)) + # p4_scoring_version (optional sparse int; absent = old GPT-2 scale) + if "p4_scoring_version" in data and data["p4_scoring_version"] is not None: + version = data["p4_scoring_version"] + if not isinstance(version, int) or isinstance(version, bool) or version < 2: + errors.append( + ValidationError(f"{path}.p4_scoring_version", "must be an int >= 2", version) + ) + + # prompt_profile (optional sparse string; must parse as a profile spec) + if "prompt_profile" in data and data["prompt_profile"] is not None: + errors.extend( + _validate_string(data["prompt_profile"], f"{path}.prompt_profile", min_length=1) + ) + if isinstance(data["prompt_profile"], str): + try: + from transformer_lens.benchmarks.text_quality_profiles import ( + ProfileSpec, + ) + + ProfileSpec.parse(data["prompt_profile"]) + except ValueError as e: + errors.append( + ValidationError(f"{path}.prompt_profile", str(e), data["prompt_profile"]) + ) + # verified_date (optional date string) if "verified_date" in data and data["verified_date"] is not None: errors.extend( diff --git a/transformer_lens/tools/model_registry/verification.py b/transformer_lens/tools/model_registry/verification.py index f402aef435..f6734cf4a6 100644 --- a/transformer_lens/tools/model_registry/verification.py +++ b/transformer_lens/tools/model_registry/verification.py @@ -29,6 +29,10 @@ class VerificationRecord: architecture_id: str = "Unknown" verified_by: Optional[str] = None transformerlens_version: Optional[str] = None + # P4 verdict flips are undiagnosable without knowing which profile and + # scoring scale produced the record. + prompt_profile: Optional[str] = None + p4_scoring_version: Optional[int] = None notes: Optional[str] = None invalidated: bool = False invalidation_reason: Optional[str] = None @@ -41,6 +45,8 @@ def to_dict(self) -> dict: "verified_date": self.verified_date.isoformat(), "verified_by": self.verified_by, "transformerlens_version": self.transformerlens_version, + "prompt_profile": self.prompt_profile, + "p4_scoring_version": self.p4_scoring_version, "notes": self.notes, "invalidated": self.invalidated, "invalidation_reason": self.invalidation_reason, @@ -55,6 +61,8 @@ def from_dict(cls, data: dict) -> "VerificationRecord": verified_date=date.fromisoformat(data["verified_date"]), verified_by=data.get("verified_by"), transformerlens_version=data.get("transformerlens_version"), + prompt_profile=data.get("prompt_profile"), + p4_scoring_version=data.get("p4_scoring_version"), notes=data.get("notes"), invalidated=data.get("invalidated", False), invalidation_reason=data.get("invalidation_reason"), diff --git a/transformer_lens/tools/model_registry/verify_models.py b/transformer_lens/tools/model_registry/verify_models.py index 66ebcca592..df549bf8a1 100644 --- a/transformer_lens/tools/model_registry/verify_models.py +++ b/transformer_lens/tools/model_registry/verify_models.py @@ -35,6 +35,10 @@ from pathlib import Path from typing import Optional +from transformer_lens.benchmarks.text_quality_profiles import ( + P4_SCORING_VERSION, + p4_pass_threshold, +) from transformer_lens.utilities.heterogeneous_config import het_safe_view # Exit code used for graceful interrupts (Ctrl+C). The wrapper script @@ -394,6 +398,7 @@ def estimate_benchmark_memory_gb( dtype: str = "float32", phases: Optional[list[int]] = None, use_hf_reference: bool = True, + device: str = "cpu", ) -> float: """Estimate peak memory needed for benchmark suite. @@ -420,8 +425,12 @@ def estimate_benchmark_memory_gb( bpp = bytes_per_param.get(dtype, 4) model_size_gb = n_params * bpp / (1024**3) - # GPT-2 scorer overhead (loaded during Phase 4) - gpt2_overhead_gb = 0.5 + # Phase-4 judge overhead: measured 2.33 GB RSS loading Qwen2.5-0.5B fp32 + # on CPU (494M params). Kept slightly above the measurement; over-counting + # is the safe direction. + # The CPU-pinned judge never occupies accelerator memory; charging it to + # a cuda budget produces spurious VRAM skips. + judge_overhead_gb = 2.5 if device == "cpu" else 0.0 # Activation/framework overhead as a fraction of model size overhead_fraction = 0.2 @@ -441,8 +450,8 @@ def estimate_benchmark_memory_gb( # Bridge + HookedTransformer = 2 copies phase_peaks.append(model_size_gb * 2.0 * (1 + overhead_fraction)) elif p == 4: - # Bridge + GPT-2 scorer - phase_peaks.append(model_size_gb * (1 + overhead_fraction) + gpt2_overhead_gb) + # Bridge + judge + phase_peaks.append(model_size_gb * (1 + overhead_fraction) + judge_overhead_gb) return max(phase_peaks) if phase_peaks else model_size_gb @@ -595,6 +604,18 @@ def _extract_phase_scores(results: list) -> dict[int, Optional[float]]: return scores +def _extract_prompt_profile(results: list) -> Optional[str]: + """Effective Phase-4 prompt profile from the benchmark details, or None + when no Phase-4 result exists. The default "continuation" is reported so + the registry write can clear a stale non-default key.""" + for result in results: + if result.phase == 4 and result.details: + profile = result.details.get("prompt_profile") + if isinstance(profile, str): + return profile + return None + + # Per-phase minimum score thresholds (0-100). # Phase 1: Core correctness (bridge vs HF) — must pass everything. # Phase 2: Hook/cache/gradient tests — most should pass. @@ -604,7 +625,9 @@ def _extract_phase_scores(results: list) -> dict[int, Optional[float]]: 1: 100.0, 2: 75.0, 3: 75.0, - 4: 50.0, + # Phase 4 floor == the benchmark pass line; a gap between them lets a + # failing score carry a clean "completed" note. + 4: p4_pass_threshold(), 7: 75.0, 8: 75.0, 9: 75.0, @@ -760,15 +783,89 @@ def _build_verified_note( else: issue_parts.append(f"P{phase}={score}%") + p4_uncovered = next( + ( + r.message + for r in all_results + if r.phase == 4 + and r.severity == BenchmarkSeverity.SKIPPED + and r.message.startswith("P4 skipped:") + ), + None, + ) + suffix = "" + if p4_uncovered: + # Keep the gap visible in the registry until prompt coverage is added. + reason = p4_uncovered.split("—")[0].replace("P4 skipped:", "").strip() + suffix = f"; P4 skipped (uncovered: {reason} — file a coverage issue)" + if issue_parts and low_text_quality: return ( f"Full verification completed with issues, low text quality: {'; '.join(issue_parts)}" + + suffix ) if issue_parts: - return f"Full verification completed with issues: {'; '.join(issue_parts)}" + return f"Full verification completed with issues: {'; '.join(issue_parts)}" + suffix if low_text_quality: - return "Full verification completed with issues, low text quality" - return "Full verification completed" + return "Full verification completed with issues, low text quality" + suffix + return "Full verification completed" + suffix + + +def _preserved_issue_suffix(model_id: str, eff_phases) -> str: + """Sub-100 scores from phases not re-run this pass stay visible in the + note; a partial pass must not overwrite tracked residue.""" + from transformer_lens.tools.model_registry.registry_io import ( + load_supported_models_raw, + ) + + try: + entry = next( + ( + m + for m in load_supported_models_raw().get("models", []) + if m.get("model_id") == model_id + ), + None, + ) + except OSError: + return "" + if entry is None: + return "" + residue = [] + for phase in (2, 3, 7, 8, 9): + if phase in (eff_phases or []): + continue + score = entry.get(f"phase{phase}_score") + if score is not None and score < 100.0: + residue.append(f"P{phase}={score}%") + if not residue: + return "" + return f" (prior issues retained: {', '.join(residue)})" + + +def _p1_only_core_note(p4_score, all_results: list) -> str: + """Note for a core run where P1 passed but P4 did not contribute a pass. + + A skipped P4 is a coverage gap, not a quality failure — the stale + (possibly old-scale) score must not be relabeled "poor".""" + from transformer_lens.benchmarks.utils import BenchmarkSeverity + + p4_skip_msg = next( + ( + r.message + for r in all_results + if r.phase == 4 + and r.severity == BenchmarkSeverity.SKIPPED + and r.message.startswith("P4 skipped:") + ), + None, + ) + if p4_skip_msg is not None: + reason = p4_skip_msg.split("—")[0].replace("P4 skipped:", "").strip() + return f"Core verification passed; P4 skipped ({reason})" + if p4_score is None: + return "Core verification passed, but text quality benchmark errored. Needs review" + return f"Core verification passed, but text quality poor (P4={p4_score}). Needs review" def _clear_hf_cache(quiet: bool = False) -> None: @@ -779,8 +876,16 @@ def _clear_hf_cache(quiet: bool = False) -> None: if not cache_dir.exists(): return + from transformer_lens.benchmarks.text_quality import JUDGE_MODEL_ID + + # The pinned Phase-4 judge is needed by every run; deleting it here would + # force a re-download per family. + judge_dir = "models--" + JUDGE_MODEL_ID.replace("/", "--") + freed = 0 for blobs_dir in cache_dir.glob("models--*/blobs"): + if blobs_dir.parent.name == judge_dir: + continue for blob in blobs_dir.iterdir(): try: size = blob.stat().st_size @@ -867,21 +972,25 @@ def verify_models( # phases stays None = full verification for the model. - # Pre-load the GPT-2 scoring model for Phase 4 so it persists across all - # models in the batch instead of being loaded and destroyed for each one. - _scoring_model = None - _scoring_tokenizer = None + # Pre-load the Phase-4 judge so it persists across all models in the batch + # instead of being loaded and destroyed for each one. + _judge_model = None + _judge_tokenizer = None if phases is None or 4 in phases: try: - from transformer_lens.benchmarks.text_quality import _load_scoring_model + from transformer_lens.benchmarks.text_quality import ( + JUDGE_MODEL_ID, + JUDGE_REVISION, + load_judge, + ) - _scoring_model, _scoring_tokenizer = _load_scoring_model("gpt2", device) + _judge_model, _judge_tokenizer = load_judge() if not quiet: - print("Pre-loaded GPT-2 scoring model for Phase 4") + print(f"Pre-loaded Phase 4 judge {JUDGE_MODEL_ID}@{JUDGE_REVISION[:8]}") except Exception as e: if not quiet: - print(f"Warning: Could not pre-load GPT-2 scorer: {e}") - print(" Phase 4 will load its own scorer per model.") + print(f"Warning: Could not pre-load Phase 4 judge: {e}") + print(" Phase 4 will load its own judge per model.") total = len(candidates) for i, candidate in enumerate(candidates, 1): @@ -953,7 +1062,7 @@ def verify_models( # Step 2: Check memory estimated_mem = estimate_benchmark_memory_gb( - n_params, dtype, phases=phases_to_run, use_hf_reference=use_hf_reference + n_params, dtype, phases=phases_to_run, use_hf_reference=use_hf_reference, device=device ) candidate.estimated_memory_gb = estimated_mem if not quiet: @@ -985,7 +1094,14 @@ def verify_models( } torch_dtype = _dtype_map[dtype] + from transformer_lens.benchmarks.text_quality_profiles import resolve_profile + from transformer_lens.tools.model_registry.registry_io import ( + registry_prompt_profile, + ) + + resolved_profile = str(resolve_profile(model_id, arch, registry_prompt_profile(model_id))) if not quiet: + print(f" Prompt profile: {resolved_profile}") print(f" Running phases {phases} in a single benchmark call...") try: all_results = run_benchmark_suite( @@ -997,8 +1113,9 @@ def verify_models( verbose=not quiet, phases=phases_to_run, trust_remote_code=needs_remote_code, - scoring_model=_scoring_model, - scoring_tokenizer=_scoring_tokenizer, + judge_model=_judge_model, + judge_tokenizer=_judge_tokenizer, + prompt_profile=resolved_profile, ) except Exception as e: error_msg = str(e) @@ -1095,7 +1212,9 @@ def verify_models( if p1_pass and p4_pass and p7_pass and p8_pass: partial_status = STATUS_VERIFIED - partial_note = "Core verification completed" + partial_note = "Core verification completed" + _preserved_issue_suffix( + model_id, eff_phases + ) elif p1_pass and p4_pass and not p7_pass: p7_score = filtered_scores.get(7) if p7_score is None: @@ -1112,9 +1231,7 @@ def verify_models( ) elif p1_pass: partial_status = STATUS_VERIFIED - partial_note = ( - "Core verification passed, but text quality poor. Needs review" - ) + partial_note = _p1_only_core_note(p4, all_results) else: # P1 failed — build a descriptive failure note partial_status = STATUS_FAILED @@ -1151,6 +1268,7 @@ def verify_models( status=partial_status, phase_scores=filtered_scores, note=partial_note, + prompt_profile=_extract_prompt_profile(all_results), ) # A provisional run was not numerically verified; do not write a # verification-history record (VerificationHistory.is_verified() @@ -1161,6 +1279,8 @@ def verify_models( arch, notes=partial_note, sanitize_fn=_sanitize_note, + prompt_profile=_extract_prompt_profile(all_results), + p4_scoring_version=(P4_SCORING_VERSION if 4 in filtered_scores else None), ) if partial_status == STATUS_FAILED: progress.failed.append(model_id) @@ -1205,6 +1325,7 @@ def verify_models( written_status, phase_scores=phase_scores, note=note, + prompt_profile=_extract_prompt_profile(all_results), ) # Provisional runs are not numerically verified — no history record # (is_verified() would otherwise report them as verified). @@ -1213,6 +1334,8 @@ def verify_models( model_id, arch, notes=note, + prompt_profile=_extract_prompt_profile(all_results), + p4_scoring_version=(P4_SCORING_VERSION if 4 in phase_scores else None), ) if is_provisional: progress.provisional.append(model_id) @@ -1235,12 +1358,15 @@ def verify_models( note=note, phase_scores=phase_scores, sanitize_fn=_sanitize_note, + prompt_profile=_extract_prompt_profile(all_results), ) add_verification_record( model_id, arch, notes=note, sanitize_fn=_sanitize_note, + prompt_profile=_extract_prompt_profile(all_results), + p4_scoring_version=(P4_SCORING_VERSION if 4 in phase_scores else None), ) progress.failed.append(model_id) @@ -1275,9 +1401,9 @@ def verify_models( _save_checkpoint(progress) # Clean up pre-loaded scoring model - if _scoring_model is not None: - del _scoring_model - del _scoring_tokenizer + if _judge_model is not None: + del _judge_model + del _judge_tokenizer gc.collect() return progress @@ -1289,6 +1415,7 @@ def _print_dry_run( max_memory_gb: float, phases: Optional[list[int]] = None, use_hf_reference: bool = True, + device: str = "cpu", ) -> None: """Print what would be tested in a dry run.""" print(f"\nDry run: {len(candidates)} models would be tested") @@ -1314,7 +1441,11 @@ def _print_dry_run( try: n_params = estimate_model_params(c.model_id) mem = estimate_benchmark_memory_gb( - n_params, dtype, phases=phases_to_run, use_hf_reference=use_hf_reference + n_params, + dtype, + phases=phases_to_run, + use_hf_reference=use_hf_reference, + device=device, ) status = "OK" if mem <= max_memory_gb else "SKIP (too large)" if mem > max_memory_gb: @@ -1562,6 +1693,7 @@ def main() -> None: max_memory_gb, phases=args.phases, use_hf_reference=not args.no_hf_reference, + device=args.device, ) return