diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 4cb68a5241..d40428a301 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -277,7 +277,7 @@ jobs: coverage-test: name: Full Code Coverage Test runs-on: ubuntu-latest - # Suite runs under pytest-xdist on a 2-vCPU runner (~45 min with swap); + # Suite runs under pytest-xdist on a 2-vCPU runner (~35 min with swap); # 75 catches a real hang well below the old serial 90. timeout-minutes: 75 steps: @@ -285,14 +285,20 @@ jobs: - name: Add swap space # The 2-vCPU runner has ~7GB RAM; two xdist workers each holding # torch + resident models overflow it. Swap absorbs the model-load - # spikes (mirrors the notebook-checks job). + # spikes. 8G left the suite's heavy tail one xdist scheduling roll + # from the runner killing the step at ~95% (identical SHAs pass and + # fail), so 16G — on the ~65G /mnt temp disk, keeping the OS disk + # free for the uv env and model caches. run: | sudo swapoff /swapfile 2>/dev/null || true sudo rm -f /swapfile - sudo fallocate -l 8G /swapfile - sudo chmod 600 /swapfile - sudo mkswap /swapfile - sudo swapon /swapfile + sudo swapoff /mnt/swapfile 2>/dev/null || true + sudo rm -f /mnt/swapfile + sudo fallocate -l 16G /mnt/swapfile + sudo chmod 600 /mnt/swapfile + sudo mkswap /mnt/swapfile + sudo swapon /mnt/swapfile + free -h && df -h / /mnt - name: Install uv uses: astral-sh/setup-uv@v7 with: @@ -358,6 +364,15 @@ jobs: # Worker count knob (this runner is 2-vCPU, so -n auto = 2). If swap # still can't hold peak memory, drop to "-n 1" or grow the swapfile. XDIST_ARGS: "-n auto --dist loadscope" + - name: Memory post-mortem + # The tail-of-suite kills leave no cause in the step log; dmesg names + # the killer (kernel OOM, systemd-oomd, or neither) for the next one. + if: failure() + run: | + sudo dmesg | tail -n 80 || true + free -h || true + swapon --show || true + df -h / /mnt || true - name: Build check run: uv build - name: Upload Coverage Report Artifact diff --git a/.gitignore b/.gitignore index 570bd085ca..090cbb72da 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ docs/source/generated !.claude/commands/ .adapter-progress.json transformer_lens/tools/model_registry/data/verification_checkpoint.json +venv/ diff --git a/demos/BERT.ipynb b/demos/BERT.ipynb index 63404a85af..e8670d454b 100644 --- a/demos/BERT.ipynb +++ b/demos/BERT.ipynb @@ -307,7 +307,7 @@ "output_type": "stream", "text": [ "Prompt: ['The [MASK] is bright today.', 'She [MASK] to the store.', 'The dog [MASK] the ball.']\n", - "Prediction: \"['Prediction 0: sun', 'Prediction 1: returned', 'Prediction 2: has']\"\n" + "Prediction: \"['Prediction 0: sun', 'Prediction 1: went', 'Prediction 2: caught']\"\n" ] } ], @@ -499,7 +499,13 @@ "\n", "inputs = tokenizer(sentence_a, sentence_b, return_tensors=\"pt\")\n", "device = next(nsp.parameters()).device\n", - "predictions = nsp(inputs[\"input_ids\"].to(device), return_type=\"predictions\")\n", + "# token_type_ids mark where sentence A ends and B begins — without them the NSP\n", + "# head sees one undifferentiated span and can return the wrong verdict.\n", + "predictions = nsp(\n", + " inputs[\"input_ids\"].to(device),\n", + " token_type_ids=inputs[\"token_type_ids\"].to(device),\n", + " return_type=\"predictions\",\n", + ")\n", "\n", "print(f\"Sentence A: {sentence_a}\")\n", "print(f\"Sentence B: {sentence_b}\")\n", diff --git a/demos/Grokking_Demo.ipynb b/demos/Grokking_Demo.ipynb index 0eb0f88fe3..b2046db581 100644 --- a/demos/Grokking_Demo.ipynb +++ b/demos/Grokking_Demo.ipynb @@ -784,7 +784,9 @@ "model_checkpoints = []\n", "checkpoint_epochs = []\n", "if TRAIN_MODEL:\n", - " for epoch in tqdm.tqdm(range(num_epochs)):\n", + "# mininterval throttles tqdm below Jupyter's IOPub rate limit while keeping a\n", + "# visible progress bar.\n", + " for epoch in tqdm.tqdm(range(num_epochs), mininterval=2):\n", " train_logits = model(train_data)\n", " train_loss = loss_fn(train_logits, train_labels)\n", " train_loss.backward()\n", @@ -1867,53 +1869,18 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "key_fourier_embed torch.Size([8, 128])\n" - ] - }, - { - "data": { - "text/html": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "key_freqs = [17, 25, 32, 47]\n", - "key_freq_indices = [33, 34, 49, 50, 63, 64, 93, 94]\n", + "# Derive the key frequencies from this run's embedding instead of hardcoding\n", + "# values from a past run — they vary with seed and training length.\n", + "fourier_norms = (fourier_basis @ W_E).norm(dim=-1)\n", + "key_freq_indices = [\n", + " i for i, norm in enumerate(fourier_norms) if i > 0 and norm > fourier_norms.max() / 4\n", + "]\n", + "key_freqs = sorted({(i + 1) // 2 for i in key_freq_indices})\n", + "print(\"key_freqs\", key_freqs)\n", "fourier_embed = fourier_basis @ W_E\n", "key_fourier_embed = fourier_embed[key_freq_indices]\n", "print(\"key_fourier_embed\", key_fourier_embed.shape)\n", diff --git a/demos/doc_sanitize.cfg b/demos/doc_sanitize.cfg index fd4388894d..4926ba521f 100644 --- a/demos/doc_sanitize.cfg +++ b/demos/doc_sanitize.cfg @@ -37,3 +37,7 @@ replace: \1 [regex7] regex: [^\n]*DeprecationWarning:(?=\n\nHookedTransformer is deprecated and will be removed in 4\.0\. Use TransformerBridge\.boot_transformers\(\.\.\.\) instead, then call enable_compatibility_mode\(\) for HookedTransformer-equivalent numerics\.) replace: DeprecationWarning: + +[regex8] +regex: /(?:var|tmp|private)/\S* +replace: TMP-PATH diff --git a/docs/source/content/contributing.md b/docs/source/content/contributing.md index 20e10f5ce1..02cadb3990 100644 --- a/docs/source/content/contributing.md +++ b/docs/source/content/contributing.md @@ -328,7 +328,7 @@ set -a; source .env; set +a uv run python -m transformer_lens.tools.model_registry.verify_models --model ``` -`verify_models` runs phases 1–4 (forward correctness vs HF, hook firing + gradients, weight processing, generation quality) and updates `data/supported_models.json` with the resulting status and per-phase scores. We recommend running `--dry-run` first to project memory and parameter count without loading the model, and verifying one model at a time — concurrent loads tend to OOM a single device. +`verify_models` runs phases 1–4 (forward correctness vs HF, hook firing + gradients, weight processing, text-generation quality) and updates `data/supported_models.json` with the resulting status and per-phase scores. We recommend running `--dry-run` first to project memory and parameter count without loading the model, and verifying one model at a time — concurrent loads tend to OOM a single device. Running with `--no-hf-reference` skips the HuggingFace numerical comparison (Phase 1 becomes structural-only). A passing run is then recorded as **provisional** (status 4), which does *not* count as verified — re-run without the flag for a real HF-compared verification. @@ -341,11 +341,11 @@ It's worth reading the per-phase scores in addition to the final status — the | 1 | 100% | — | Verification fails | | 2 | 75% | `logits_equivalence`, `loss_equivalence` | Verification fails | | 3 | 75% | `logits_equivalence`, `loss_equivalence` | Verification fails | -| 4 | 50% | — | **Non-gating** — below 50% adds `"low text quality"` to the registry `note`; never fails verification. | +| 4 | 54.5% (measured pass line, `p4_pass_threshold()`) | — | **Non-gating** — below the line adds a `"text quality poor (P4=…)"` note; never fails verification. | | 7 | 75% | `multimodal_forward` | Verification fails. A NULL score also fails. | | 8 | 75% | `audio_forward` | Verification fails. A NULL score also fails. | -Phase 4 is intentionally lenient — it's a coherence metric, not a correctness check. A sub-100% Phase-4 score on a small parity-test model can still indicate a real adapter bug that the gates don't catch (missing `preprocess_weights` fold, wrong `default_prepend_bos`, and so on); the model can pass verification overall and still be worth a manual look. +Phase 4 prompts each model with its resolved prompt profile (chat template, translation, code, own-language continuation, ...) and scores the generation against a known-good reference with one pinned multilingual judge, via the perplexity ratio `PPL(generated)/PPL(reference)`. It's intentionally lenient — a coherence metric, not a correctness check. A sub-100% Phase-4 score on a small parity-test model can still indicate a real adapter bug that the gates don't catch (missing `preprocess_weights` fold, wrong `default_prepend_bos`, and so on); the model can pass verification overall and still be worth a manual look. If verification fails by `~1e-3` or more against the HF reference, the bisection workflow lives at [Debugging Numerical Divergence](debugging_numerical_divergence.md). diff --git a/docs/source/content/jacobian_lens_fitting.md b/docs/source/content/jacobian_lens_fitting.md index 8c19095f82..6323c7bae3 100644 --- a/docs/source/content/jacobian_lens_fitting.md +++ b/docs/source/content/jacobian_lens_fitting.md @@ -318,6 +318,49 @@ exact values will not necessarily transfer. time." Here `k` is an **upper bound**: `support` returns *at most* `k` active vectors (often fewer), never `k` padded with zero-coefficient slots. +### Occupancy and fraction of variance + +Two statistics turn the honesty bullets above into numbers you can measure. Both build on J-lens +vector dictionaries and sparse supports, but occupancy uses its own projection-residual recurrence. + +`occupancy` estimates **how many J-lens vectors are meaningfully active** in a single activation — +the quantity behind the paper's `k <= 25`. At each step it admits the unused atom with the greatest +signed, norm-normalized correlation with the current residual, projects the activation onto the full +selected span, and sets the next residual to `x - Pi_S x`. This is the same per-step correlation +*rule* as `decompose`, but `decompose` recurses on a nonnegative coefficient-fit residual and may +stop early, whereas occupancy selects exactly `max_atoms` atoms, so their supports need not match. +Occupancy records the per-step captured variance `||Pi_S x||^2 / ||x||^2` and compares that curve +against the same occupancy recurrence on `num_control_dictionaries` random unit-norm dictionaries. +The occupancy is the step of **maximum separation** between the real and averaged-control +*cumulative* captured variance — the point past which further vectors add no more than random +directions would. It is deterministic given `seed` and needs no threshold. + +```python +occ = lens.occupancy(model, "The Eiffel Tower is in the city of", layer=6, position=-1) +occ.occupancy # int: meaningfully-active vector count (a small positive integer) +occ.marginal_captured_variance # [max_atoms] real per-step captured-variance gains (for plotting) +occ.control_captured_variance # [max_atoms] averaged random-control gains +``` + +`fraction_of_variance` profiles the **J-space share of activation variance over a prompt corpus**. +For each `(layer, position)` at or past `skip_first` (mirroring the fit's early-position skip) it +records `||j_space_component||^2 / ||activation||^2` — the `selected_support` span projection, +matching the paper's appendix operationalization, **not** the nonnegative `reconstruction`. Per +layer it reports the `median` of those fractions and the `pooled` ratio +`sum(||j_space||^2) / sum(||activation||^2)`. +Each token tensor must represent one prompt and have shape `[1, seq]`; `skip_first` must be +non-negative even when explicit `positions` override its sampling behavior. + +```python +profile = lens.fraction_of_variance(model, prompts, layers=[3, 6], k=8) +profile.median # {layer: median fraction} -- the paper's "median 6-7%" quantity +profile.pooled # {layer: pooled ratio in [0, 1]} +``` + +Both are **shape** claims on open weights: expect a small occupancy and a small variance fraction, +but do not expect the paper's closed-model figures (see *Interpreting the numbers honestly* above) +to transfer numerically. + The full-vocabulary dictionary is cached on the model's device and is vocabulary-sized (gigabytes for large models); release it with `lens.clear_device_cache()`. diff --git a/docs/source/content/migrating_to_v3.md b/docs/source/content/migrating_to_v3.md index f5047bf2fe..65881ac18c 100644 --- a/docs/source/content/migrating_to_v3.md +++ b/docs/source/content/migrating_to_v3.md @@ -122,7 +122,7 @@ For the full mapping of legacy → canonical names and the expected tensor shape Two semantic differences inside `enable_compatibility_mode()` worth knowing if you are porting activation-patching, DLA, or attribution-patching code: -- **`blocks.{i}.hook_mlp_in` fires pre-ln2** (matching legacy `HookedTransformer`). Use `bridge.set_use_hook_mlp_in(True)` to enable it — setting `cfg.use_hook_mlp_in = True` directly is honored when blocks share the bridge's `cfg`, but the setter is the supported entry point. The pre-ln2 placement means cached values from one run can be patched into another and re-flow through `ln2 → mlp` consistently across the bridge and `HookedTransformer`. +- **`blocks.{i}.hook_mlp_in` fires pre-ln2** (matching legacy `HookedTransformer`). Enable it with `bridge.set_use_hook_mlp_in(True)` or `bridge.cfg.use_hook_mlp_in = True`; direct config assignment routes through the same validation and propagation path as the setter. The pre-ln2 placement means cached values from one run can be patched into another and re-flow through `ln2 → mlp` consistently across the bridge and `HookedTransformer`. - **`hook_q_input` / `hook_k_input` / `hook_v_input` / `hook_attn_in`** also fire pre-ln1 in compat mode. On the per-head LN application that follows, the bridge routes through the raw HF norm rather than the `NormalizationBridge` wrapper, so `ln1`'s sub-hooks (`hook_in`, `hook_normalized`, `hook_scale`) do **not** fire once per head the way legacy `LayerNormPre` would. Q/K/V projections downstream still match legacy numerically; only the intermediate LN sub-hook firing is suppressed. Post-norm architectures (OLMo 2, BERT-style encoders) and MLA blocks (DeepSeek V2/V3/R1) do not participate in the pre-ln1 capture — `MLABlockBridge` does not expose those aliases, and post-norm models would read the post-attention residual instead of the block input. @@ -144,7 +144,35 @@ If your code only touches these APIs, the migration is genuinely just the loadin ### BERT Next Sentence Prediction -`BertNextSentencePrediction` is not ported to `TransformerBridge`. Keep using `HookedEncoder` + `BertNextSentencePrediction` for NSP workflows. The bridge's BERT adapter does load NSP HuggingFace checkpoints (it rewires the unembed to `cls.seq_relationship`), but the high-level NSP API – sentence-pair tokenization, `[CLS]` pooling, "sequential"/"not sequential" decoding — is not exposed. If this is feature is something you'd like added to TransformerBridge, please file an issue. +NSP runs on the bridge today — load the NSP head via `model_class` and pass the +sentence-pair tokenization through: + +```python +from transformers import AutoTokenizer, BertForNextSentencePrediction +from transformer_lens.model_bridge import TransformerBridge + +tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") +nsp = TransformerBridge.boot_transformers( + "google-bert/bert-base-cased", + model_class=BertForNextSentencePrediction, +) +nsp.enable_compatibility_mode() + +inputs = tokenizer("A man walked into a grocery store.", "He bought an apple.", return_tensors="pt") +nsp(inputs["input_ids"], token_type_ids=inputs["token_type_ids"], return_type="predictions") +# 'The sentences are sequential' +``` + +**Pass `token_type_ids`.** They are what tells BERT where the first sentence ends +and the second begins; without them the NSP head scores a single undifferentiated +span and can return the wrong verdict (on the pair above, dropping them collapses +the logits from ±4.37 to ±0.58, and a genuinely non-sequential pair flips to +"sequential"). With them, the bridge reproduces the raw HuggingFace NSP logits +exactly. + +The legacy `BertNextSentencePrediction` wrapper is deprecated and cannot wrap a +`TransformerBridge` — it reaches for `HookedEncoder`-only internals +(`encoder_output`, `pooler`, `nsp_head`). Use the recipe above instead. ### New in 3.x: streaming generation diff --git a/scripts/phase4_review.py b/scripts/phase4_review.py new file mode 100644 index 0000000000..3b6e98fbc9 --- /dev/null +++ b/scripts/phase4_review.py @@ -0,0 +1,72 @@ +"""Registry-wide Phase-4 review: which verified models' stored scores predate +the profile rework and deserve a re-run. + +phase4_score is a mixed-scale column: entries stamped p4_scoring_version=2 +were measured with the pinned-judge reference-ratio scoring (pass line 56); +unstamped entries carry the old GPT-2 absolute-perplexity scale (pass line 85) +and are never compared against the new line — they are re-run candidates. +Read-only. +""" + +import argparse + +from transformer_lens.benchmarks.text_quality_profiles import ( + P4_SCORING_VERSION, + resolve_profile, +) +from transformer_lens.tools.model_registry.registry_io import load_supported_models_raw + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--below", type=float, default=None, help="Only scores below this") + parser.add_argument("--limit", type=int, default=None, help="Max rows per section") + args = parser.parse_args() + + current: list = [] + stale: list = [] + for entry in load_supported_models_raw().get("models", []): + if entry.get("status") != 1 or entry.get("phase4_score") is None: + continue + score = entry["phase4_score"] + if args.below is not None and score >= args.below: + continue + profile = str( + resolve_profile( + entry["model_id"], entry.get("architecture_id"), entry.get("prompt_profile") + ) + ) + row = (profile != "continuation", score, entry["model_id"], profile) + if entry.get("p4_scoring_version") == P4_SCORING_VERSION: + current.append(row) + else: + stale.append(row) + + # Profile-changed first, then ascending score: measurement changed most. + for rows in (current, stale): + rows.sort(key=lambda r: (not r[0], r[1])) + if args.limit: + current = current[: args.limit] + stale = stale[: args.limit] + + print( + f"{len(current)} scored on the current scale (v{P4_SCORING_VERSION}); " + f"{len(stale)} on the old GPT-2 scale (re-run candidates)\n" + ) + for title, rows in ( + (f"v{P4_SCORING_VERSION} (reference-ratio scale, pass 56)", current), + ("v1 (GPT-2 scale — scores NOT comparable to the new pass line)", stale), + ): + if not rows: + continue + changed = sum(1 for r in rows if r[0]) + print(f"== {title}: {len(rows)} models, {changed} non-default profiles") + print(f"{'score':>6} {'profile':<28} model") + for is_changed, score, model_id, profile in rows: + marker = "*" if is_changed else " " + print(f"{score:6.1f}{marker} {profile:<28} {model_id}") + print() + + +if __name__ == "__main__": + main() diff --git a/scripts/text_quality_judge_bakeoff.py b/scripts/text_quality_judge_bakeoff.py new file mode 100644 index 0000000000..95aed50529 --- /dev/null +++ b/scripts/text_quality_judge_bakeoff.py @@ -0,0 +1,464 @@ +"""Bake off perplexity judges for the reworked Phase-4 text-quality benchmark. + +Scores two small causal LMs as candidate PPL judges: for each of 9 languages +(8 natural + "code"), build a fluent corpus from ``text_quality_profiles`` and +six deterministic corruptions per fluent string (shuffle/repeat/charnoise x2/ +crosslang x2). The judge that best separates fluent from corrupted text by +ROC AUC, worst-language-first, wins; its R_FAIL/R_GOOD thresholds and +per-reference PPLs are then emitted for the real benchmark to consume. + +Run: uv run python scripts/text_quality_judge_bakeoff.py + uv run python scripts/text_quality_judge_bakeoff.py --languages en,fr --models gpt2 +""" +from __future__ import annotations + +import argparse +import gc +import json +import math +import random +import statistics +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import numpy as np +import torch +from huggingface_hub import HfApi +from transformers import AutoModelForCausalLM, AutoTokenizer + +from transformer_lens.benchmarks.text_quality_profiles import ( + CHAT_PROMPTS, + CONTINUATION_PROMPTS, + PIVOT_SENTENCES, +) + +# All PIVOT_SENTENCES languages plus code, so no scored language is left +# uncalibrated (it/nl/pt/hi were absent from the original judge selection run). +DEFAULT_LANGUAGES = [ + "en", + "fr", + "es", + "de", + "it", + "nl", + "pt", + "zh", + "ar", + "ru", + "ja", + "hi", + "code", +] +DEFAULT_MODELS = ["bigscience/bloom-560m", "Qwen/Qwen2.5-0.5B"] + +NO_SPACE_LANGS = {"zh", "ja"} +CHARNOISE_RATES = (0.15, 0.4) +CROSSLANG_RATES = (0.3, 0.7) + +OUT_JSON = Path("judge_reference_ppls.json") # cwd; override with --out + +# --------------------------------------------------------------------------- +# Fluent corpus +# --------------------------------------------------------------------------- + + +def build_fluent_corpus(languages: list[str]) -> dict[str, list[str]]: + """Fluent strings per language: pivot sentences + continuation/chat references.""" + corpus: dict[str, list[str]] = {} + for lang in languages: + if lang == "code": + corpus[lang] = [p.reference for p in CONTINUATION_PROMPTS["code"]] + continue + texts = list(PIVOT_SENTENCES.get(lang, ())) + texts += [p.reference for p in CONTINUATION_PROMPTS.get(lang, ())] + texts += [p.reference for p in CHAT_PROMPTS.get(lang, ())] + corpus[lang] = texts + return corpus + + +# --------------------------------------------------------------------------- +# Corruptions (deterministic given a shared random.Random) +# --------------------------------------------------------------------------- + + +def tokenize_units(text: str, lang: str) -> list[str]: + """Words for space-delimited languages, characters for zh/ja.""" + return list(text) if lang in NO_SPACE_LANGS else text.split() + + +def join_units(units: list[str], lang: str) -> str: + return "".join(units) if lang in NO_SPACE_LANGS else " ".join(units) + + +def corrupt_shuffle(text: str, lang: str, rng: random.Random) -> str: + """Permute word (or char) order.""" + units = tokenize_units(text, lang) + rng.shuffle(units) + return join_units(units, lang) + + +def corrupt_repeat(text: str, lang: str) -> str: + """Repeat the first 3 words (5 chars for zh/ja) until the original length.""" + units = tokenize_units(text, lang) + if not units: + return text + n = 5 if lang in NO_SPACE_LANGS else 3 + seed = units[:n] or units + out = [seed[i % len(seed)] for i in range(len(units))] + return join_units(out, lang) + + +def corrupt_charnoise(text: str, rate: float, rng: random.Random) -> str: + """Swap-adjacent-or-delete at `rate` of character positions.""" + chars = list(text) + out: list[str] = [] + i = 0 + while i < len(chars): + if rng.random() < rate: + if i + 1 < len(chars) and rng.random() < 0.5: + out.append(chars[i + 1]) + out.append(chars[i]) + i += 2 + continue + i += 1 # delete + continue + out.append(chars[i]) + i += 1 + return "".join(out) + + +def corrupt_crosslang( + text: str, lang: str, rate: float, other_units: list[str], rng: random.Random +) -> str: + """Replace `rate` of tokens with tokens drawn from another language's fluent text.""" + units = tokenize_units(text, lang) + if not units or not other_units: + return text + n_replace = min(len(units), max(1, round(rate * len(units)))) + idxs = rng.sample(range(len(units)), n_replace) + out = units[:] + for idx in idxs: + out[idx] = rng.choice(other_units) + return join_units(out, lang) + + +@dataclass +class CorruptedSample: + """One corrupted variant of a fluent source string.""" + + lang: str + source_text: str + kind: str + severity: Optional[float] + text: str + + +def build_corruptions( + languages: list[str], fluent_corpus: dict[str, list[str]], seed: int = 42 +) -> dict[str, list[CorruptedSample]]: + """6 corrupted variants per fluent string, generated once and shared by both candidates.""" + rng = random.Random(seed) + other_pool: dict[str, list[str]] = {} + for i, lang in enumerate(languages): + next_lang = languages[(i + 1) % len(languages)] + other_text = " ".join(fluent_corpus.get(next_lang, [])) + other_pool[lang] = tokenize_units(other_text, lang) + + by_lang: dict[str, list[CorruptedSample]] = {lang: [] for lang in languages} + for lang in languages: + for text in fluent_corpus[lang]: + samples = by_lang[lang] + samples.append( + CorruptedSample(lang, text, "shuffle", None, corrupt_shuffle(text, lang, rng)) + ) + samples.append(CorruptedSample(lang, text, "repeat", None, corrupt_repeat(text, lang))) + for rate in CHARNOISE_RATES: + samples.append( + CorruptedSample( + lang, text, "charnoise", rate, corrupt_charnoise(text, rate, rng) + ) + ) + for rate in CROSSLANG_RATES: + samples.append( + CorruptedSample( + lang, + text, + "crosslang", + rate, + corrupt_crosslang(text, lang, rate, other_pool[lang], rng), + ) + ) + return by_lang + + +# --------------------------------------------------------------------------- +# PPL scoring +# --------------------------------------------------------------------------- + + +def _with_retry(fn, *args, **kwargs): # type: ignore[no-untyped-def] + """One retry after 60s on a 429/rate-limit error.""" + try: + return fn(*args, **kwargs) + except Exception as exc: # noqa: BLE001 + msg = str(exc) + if "429" in msg or "rate limit" in msg.lower(): + print(f"429 hit, retrying in 60s: {msg}", file=sys.stderr) + time.sleep(60) + return fn(*args, **kwargs) + raise + + +def score_text(text: str, tokenizer, model) -> Optional[dict]: # type: ignore[no-untyped-def] + """NLL-based PPL for one string; None if tokenization has <2 tokens.""" + enc = tokenizer(text, return_tensors="pt") + ids = enc["input_ids"] + n_tokens = int(ids.shape[1]) + if n_tokens < 2: + return None + t0 = time.perf_counter() + with torch.no_grad(): + out = model(input_ids=ids, labels=ids) + dt = time.perf_counter() - t0 + ppl = math.exp(out.loss.item()) + unk_id = tokenizer.unk_token_id + unk_count = int((ids == unk_id).sum().item()) if unk_id is not None else None + return {"ppl": ppl, "n_tokens": n_tokens, "unk_count": unk_count, "dt": dt} + + +# --------------------------------------------------------------------------- +# AUC (hand-rolled, rank-based Mann-Whitney) +# --------------------------------------------------------------------------- + + +def auc_score(neg_scores: list[float], pos_scores: list[float]) -> float: + """P(pos > neg) via rank-sum; positive = corrupted, negative = fluent.""" + n_pos, n_neg = len(pos_scores), len(neg_scores) + if n_pos == 0 or n_neg == 0: + return float("nan") + combined = [(s, 0) for s in neg_scores] + [(s, 1) for s in pos_scores] + combined.sort(key=lambda x: x[0]) + n = len(combined) + ranks = [0.0] * n + i = 0 + while i < n: + j = i + while j < n and combined[j][0] == combined[i][0]: + j += 1 + avg_rank = (i + 1 + j) / 2.0 # 1-indexed, averaged over the tie block + for k in range(i, j): + ranks[k] = avg_rank + i = j + rank_sum_pos = sum(r for r, (_, lbl) in zip(ranks, combined) if lbl == 1) + return (rank_sum_pos - n_pos * (n_pos + 1) / 2) / (n_pos * n_neg) + + +# --------------------------------------------------------------------------- +# Per-candidate run +# --------------------------------------------------------------------------- + + +def run_candidate( + model_id: str, + languages: list[str], + fluent_corpus: dict[str, list[str]], + corrupted_by_lang: dict[str, list[CorruptedSample]], +) -> dict: + print(f"Loading {model_id} ...") + tokenizer = _with_retry(AutoTokenizer.from_pretrained, model_id) + model = _with_retry(AutoModelForCausalLM.from_pretrained, model_id, dtype=torch.float32) + model.eval() + + fluent_ppl: dict[str, dict[str, float]] = {} + lang_stats: dict[str, dict] = {} + total_dt = 0.0 + total_forwards = 0 + + for lang in languages: + fluent_ppl[lang] = {} + ln_fluent: list[float] = [] + has_unk = tokenizer.unk_token_id is not None + unk_count = 0 + unk_total = 0 + + for text in fluent_corpus[lang]: + res = score_text(text, tokenizer, model) + if res is None: + continue + total_forwards += 1 + total_dt += res["dt"] + fluent_ppl[lang][text] = res["ppl"] + ln_fluent.append(math.log(res["ppl"])) + if has_unk: + unk_count += res["unk_count"] + unk_total += res["n_tokens"] + + ln_corrupt: list[float] = [] + ln_ratios: list[float] = [] + ratio_raw: list[float] = [] + for sample in corrupted_by_lang[lang]: + if sample.source_text not in fluent_ppl[lang]: + continue # source string itself was skipped (too short) + res = score_text(sample.text, tokenizer, model) + if res is None: + continue + total_forwards += 1 + total_dt += res["dt"] + ppl_src = fluent_ppl[lang][sample.source_text] + ln_corrupt.append(math.log(res["ppl"])) + ln_ratios.append(math.log(res["ppl"]) - math.log(ppl_src)) + ratio_raw.append(res["ppl"] / ppl_src) + if has_unk: + unk_count += res["unk_count"] + unk_total += res["n_tokens"] + + lang_stats[lang] = { + "auc": auc_score(ln_fluent, ln_corrupt), + "median_ln_ratio": float(np.median(ln_ratios)) if ln_ratios else float("nan"), + "unk_rate": (unk_count / unk_total) if has_unk and unk_total else None, + "mean_fluent_ppl": ( + float(np.mean(list(fluent_ppl[lang].values()))) + if fluent_ppl[lang] + else float("nan") + ), + "ratio_raw": ratio_raw, + } + + avg_forward_time = total_dt / total_forwards if total_forwards else float("nan") + del model + gc.collect() + return { + "fluent_ppl": fluent_ppl, + "lang_stats": lang_stats, + "avg_forward_time": avg_forward_time, + } + + +# --------------------------------------------------------------------------- +# Winner selection + constants +# --------------------------------------------------------------------------- + + +def pick_winner(results: dict[str, dict], languages: list[str]) -> tuple[str, str]: + def min_auc(name: str) -> float: + return min(results[name]["lang_stats"][l]["auc"] for l in languages) + + def spread(name: str) -> float: + vals = [results[name]["lang_stats"][l]["median_ln_ratio"] for l in languages] + return max(vals) - min(vals) + + def speed(name: str) -> float: + return results[name]["avg_forward_time"] + + names = sorted(results, key=min_auc, reverse=True) + best = min_auc(names[0]) + tied = [n for n in names if best - min_auc(n) <= 0.01] + if len(tied) == 1: + return tied[0], "highest minimum per-language AUC" + + best_spread = min(spread(n) for n in tied) + tied2 = [n for n in tied if spread(n) == best_spread] + if len(tied2) == 1: + return tied2[0], "min-AUC tie -> smaller cross-language ln-ratio spread" + + winner = min(tied2, key=speed) + return winner, "min-AUC tie -> spread tie -> faster wall-clock per forward" + + +def compute_r_fail(winner_stats: dict, languages: list[str]) -> float: + """Geo-mean over languages of the MEDIAN corrupted/fluent ratio. + + A low percentile degenerates below 1 in weak-separation languages (some + corruptions do not raise perplexity there), which would invert the log + mapping; the median is the robust "typical broken output" anchor. This is + the exact derivation of the shipped JUDGE_R_FAIL.""" + per_lang = [] + for lang in languages: + raw = winner_stats["lang_stats"][lang]["ratio_raw"] + if raw: + per_lang.append(float(np.median(raw))) + return statistics.geometric_mean(per_lang) + + +def compute_r_good(winner_stats: dict, languages: list[str]) -> float: + per_lang = [] + for lang in languages: + vals = list(winner_stats["fluent_ppl"][lang].values()) + ratios = [vals[i] / vals[j] for i in range(len(vals)) for j in range(len(vals)) if i != j] + if ratios: + per_lang.append(float(np.percentile(ratios, 90))) + return statistics.geometric_mean(per_lang) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--languages", default=",".join(DEFAULT_LANGUAGES)) + p.add_argument("--models", default=",".join(DEFAULT_MODELS)) + p.add_argument("--out", type=Path, default=OUT_JSON, help="Where to write reference PPLs") + return p.parse_args() + + +def main() -> None: + args = parse_args() + languages = [l.strip() for l in args.languages.split(",") if l.strip()] + models = [m.strip() for m in args.models.split(",") if m.strip()] + + api = HfApi() + for model_id in models: + info = _with_retry(api.model_info, model_id) + print(f"{model_id} revision sha: {info.sha}") + + fluent_corpus = build_fluent_corpus(languages) + corrupted_by_lang = build_corruptions(languages, fluent_corpus) + + results: dict[str, dict] = {} + for model_id in models: + results[model_id] = run_candidate(model_id, languages, fluent_corpus, corrupted_by_lang) + + header = f"{'candidate':28s} {'lang':6s} {'auc':>7s} {'med_ln_ratio':>13s} {'unk_rate':>9s} {'mean_ppl':>10s}" + print("\n" + header) + print("-" * len(header)) + for model_id in models: + for lang in languages: + st = results[model_id]["lang_stats"][lang] + unk_str = f"{st['unk_rate']:.4f}" if st["unk_rate"] is not None else "n/a" + print( + f"{model_id:28s} {lang:6s} {st['auc']:7.3f} {st['median_ln_ratio']:13.3f} " + f"{unk_str:>9s} {st['mean_fluent_ppl']:10.2f}" + ) + + winner, reason = pick_winner(results, languages) + print(f"\nwinner: {winner} ({reason})") + for model_id in models: + print( + f" {model_id}: avg forward wall-clock {results[model_id]['avg_forward_time']*1000:.2f} ms" + ) + + r_fail = compute_r_fail(results[winner], languages) + r_good = compute_r_good(results[winner], languages) + print(f"JUDGE_R_FAIL = {r_fail:.1f}") + print(f"JUDGE_R_GOOD = {r_good:.2f}") + print(f"pass line score(R_GOOD) = {100 - 100 * math.log(r_good) / math.log(r_fail):.1f}") + + flagged = [ + lang + for lang in languages + if all(results[m]["lang_stats"][lang]["auc"] < 0.8 for m in models) + ] + print(f"languages with AUC < 0.8 for BOTH candidates: {flagged or 'none'}") + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(results[winner]["fluent_ppl"], ensure_ascii=False, indent=2)) + print(f"\nreference PPLs written to {args.out}") + + +if __name__ == "__main__": + main() diff --git a/tests/QUARANTINES.md b/tests/QUARANTINES.md index 049c050701..89d835d4e0 100644 --- a/tests/QUARANTINES.md +++ b/tests/QUARANTINES.md @@ -79,14 +79,11 @@ Big-model adapter tests use `@pytest.mark.slow`, CI tier filters `-m "not slow"` | Path | Reason | Issue | |---|---|---| -| [`unit/model_bridge/test_bridge_generate_no_tokenizer.py`:30,128](unit/model_bridge/test_bridge_generate_no_tokenizer.py) | `skipif(_MACOS_ARM64)` — KV-cache NaN | Upstream PyTorch/HF on M-series Macs | -| [`integration/model_bridge/test_bridge_generate_stopping_criteria.py`](integration/model_bridge/test_bridge_generate_stopping_criteria.py) | `skipif(_MACOS_ARM64)`, KV-cache NaN (one `use_past_kv_cache=True` test) | Upstream PyTorch/HF on M-series Macs | | [`acceptance/test_hooked_transformer.py`](acceptance/test_hooked_transformer.py) | `redwood_attn_2l` (2 tests) — `ArthurConmy/redwood_tokenizer`'s merges name a token missing from its vocab (`Ġpati`), rejected by tokenizers >= 0.20 on both the fast and slow paths | Third-party repo; the weights load fine, only the tokenizer is unusable | -**Un-skip:** when upstream resolves. Don't bypass — produces NaN logits. The redwood skip is -evaluated at collection by actually attempting the load, so it disappears on its own if the repo -is fixed or the tokenizers constraint relaxes; substituting a different tokenizer is not a fix -(it would change token ids and invalidate the pinned expected loss). +**Un-skip:** evaluated at collection by actually attempting the load, so it disappears on its own +if the repo is fixed or the tokenizers constraint relaxes; substituting a different tokenizer is +not a fix (it would change token ids and invalidate the pinned expected loss). --- @@ -94,22 +91,6 @@ is fixed or the tokenizers constraint relaxes; substituting a different tokenize No modules are currently quarantined this way. -**Resolved 2026-08-17.** `acceptance/test_hooked_transformer.py`, `test_hooked_encoder.py` and -`test_hooked_encoder_decoder.py` had carried module-level `pytest.mark.skip(reason="CI test -pollution")` since #1129. Re-running them found no pollution: each passes alone, and the full -acceptance tier with all three enabled is green (231 passed, 39 skipped). What the skips were -hiding was four genuine failures, all now fixed at the source: - -| Was failing | Actual cause | -|---|---| -| `test_bert_block` | transformers 5.x returns a tensor from `BertLayer.forward`, so the test's `[0]` took batch element 0 instead of tuple element 0 | -| `test_bloom_similarity_*` (×2) | the HF fixture loaded bloom at its checkpoint dtype (fp16) while TL loads fp32 — the comparison measured HF's own fp16 error (0.259 log-softmax against *itself*), not TL | -| `test_model[redwood_attn_2l]`, `test_from_pretrained_no_processing[redwood_attn_2l]` | `ArthurConmy/redwood_tokenizer` has merges referencing a token absent from its vocab, which tokenizers >= 0.20 rejects; see the per-test skip below | - -Two silent TransformerLens bugs also lived in this blind spot the whole time: T5's decoder -self-attention was never causally masked, and its relative-position bias used the encoder's -bucketing. Both are fixed. Keep these modules enabled. - --- ## Technical debt — individual diff --git a/tests/acceptance/model_bridge/test_run_with_cache_batch.py b/tests/acceptance/model_bridge/test_run_with_cache_batch.py index 3600c42ce7..8462a498de 100644 --- a/tests/acceptance/model_bridge/test_run_with_cache_batch.py +++ b/tests/acceptance/model_bridge/test_run_with_cache_batch.py @@ -7,6 +7,19 @@ import torch +from transformer_lens.utilities import get_attention_mask + + +def _last_real_token_positions(model, prompts: list[str]) -> torch.Tensor: + tokens = model.to_tokens(prompts) + attention_mask = get_attention_mask( + model.tokenizer, + tokens, + prepend_bos=getattr(model.cfg, "default_prepend_bos", True), + ) + positions = torch.arange(tokens.shape[1], device=tokens.device).expand_as(tokens) + return positions.masked_fill(attention_mask == 0, -1).max(dim=1).values + def test_run_with_cache_batch_matches_individual(gpt2_bridge): """Batched run_with_cache logits at the last real token should match per-prompt runs.""" @@ -23,9 +36,9 @@ def test_run_with_cache_batch_matches_individual(gpt2_bridge): # Batched run batched_logits, _ = gpt2_bridge.run_with_cache(prompts) - # With left-padding forced internally, position -1 is the last real token - for i in range(len(prompts)): - batched_last = batched_logits[i, -1, :] + last_real_positions = _last_real_token_positions(gpt2_bridge, prompts) + for i, position in enumerate(last_real_positions): + batched_last = batched_logits[i, position, :] assert torch.allclose( individual_logits[i], batched_last, atol=1e-4 ), f"Prompt {i} logit mismatch between individual and batched run_with_cache" @@ -54,11 +67,11 @@ def capture_individual(tensor, hook): # Batched run captured_batched = [] + last_real_positions = _last_real_token_positions(gpt2_bridge, prompts) def capture_batched(tensor, hook): - # For left-padded batch, last real token is at position -1 for all - for i in range(tensor.shape[0]): - captured_batched.append(tensor[i, -1, :].detach().clone()) + for i, position in enumerate(last_real_positions): + captured_batched.append(tensor[i, position, :].detach().clone()) gpt2_bridge.run_with_hooks( prompts, diff --git a/tests/integration/benchmarks/__init__.py b/tests/integration/benchmarks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/benchmarks/test_text_quality_profiles.py b/tests/integration/benchmarks/test_text_quality_profiles.py new file mode 100644 index 0000000000..1e1f96fa64 --- /dev/null +++ b/tests/integration/benchmarks/test_text_quality_profiles.py @@ -0,0 +1,171 @@ +"""End-to-end Phase-4 profile scoring against real models and the real judge. + +Each test guards a profile path that unit stubs cannot: the Marian test keeps +seq2seq whole-output scoring (pre-profile P4 masked a prompt "continuation" +that seq2seq output does not have and scored 0); the Florence-2 test keeps the +caption path (text-only prompts yield a bare EOS on image-conditioned models); +the judge tests pin the revision and the fluent-vs-corrupted separation the +bake-off measured. +""" + +import pytest + +pytest.importorskip("transformers") + +from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + +def _boot(model_id, **kwargs): + from transformer_lens.model_bridge import TransformerBridge + + try: + return TransformerBridge.boot_transformers(model_id, device="cpu", **kwargs) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"{model_id} unavailable offline: {exc}") + + +@pytest.fixture(scope="module") +def judge(): + from transformer_lens.benchmarks.text_quality import load_judge + + try: + return load_judge() + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"judge unavailable offline: {exc}") + + +def test_translation_profile_marian(judge): + """Seq2seq output is standalone (a translation), not a continuation of the + prompt; the translation profile must score it whole, against the pivot + reference, in the direction parsed from the model id.""" + bridge = _boot("Helsinki-NLP/opus-mt-nl-en") + assert bridge.original_model.config.is_encoder_decoder # precondition + + judge_model, judge_tokenizer = judge + result = benchmark_text_quality( + bridge, + "task:translation@nl-en", + judge_model=judge_model, + judge_tokenizer=judge_tokenizer, + ) + assert result.details is not None, result.message + assert result.details["prompt_profile"] == "task:translation@nl-en" + # A working translator of 3 short pivot sentences must land well above the + # broken floor (score 0 = judge's typical-corruption perplexity ratio). + assert result.details["score"] > 50.0, result.details + + +def test_caption_profile_florence2(judge): + """Florence-2 emits a bare EOS for text-only prompts; P4 must drive real + image-conditioned captions and score them under the caption profile.""" + pytest.importorskip("PIL") + bridge = _boot("florence-community/Florence-2-base", trust_remote_code=True) + + judge_model, judge_tokenizer = judge + result = benchmark_text_quality( + bridge, + "continuation", # deliberately wrong: the caption adjustment must win + judge_model=judge_model, + judge_tokenizer=judge_tokenizer, + ) + assert result.details is not None, result.message + assert result.details["prompt_profile"] == "caption" + assert result.details["score"] > 0.0 + + +def test_chat_profile_templates_and_scores(judge): + """Chat models are scored through their own template (prepend_bos=False — + the template supplies BOS); output must not be the template markers.""" + bridge = _boot("Qwen/Qwen2.5-0.5B-Instruct") + + judge_model, judge_tokenizer = judge + result = benchmark_text_quality( + bridge, + "chat", + judge_model=judge_model, + judge_tokenizer=judge_tokenizer, + ) + assert result.details is not None, result.message + assert result.details["prompt_profile"] == "chat" + assert "<|im_start|>" not in result.details["generated_text"] + assert result.details["score"] > 50.0, result.details + + +def test_fluent_vs_shuffled_separation_end_to_end(judge): + """The full scoring chain must separate real model output from word salad: + any break (mask slip, ratio inversion, penalty loss) collapses the gap.""" + import random + + from transformer_lens.benchmarks.text_quality import ( + _compute_repetition_penalty, + _judge_perplexity, + _ratio_to_score, + ) + from transformer_lens.benchmarks.text_quality_profiles import CONTINUATION_PROMPTS + + judge_model, judge_tokenizer = judge + entry = CONTINUATION_PROMPTS["en"][0] + # A DISTINCT on-topic fluent paraphrase, not the reference itself: ref/ref + # is identically 1 -> 100 and would pass with the judge deleted. + # Measured: ppl 8.3 vs ref 5.1 -> score 83.2 (judge conditioned on the + # relativity prompt correctly rejects off-topic fluent text). + fluent = ( + " measurements of time and distance depend on the observer's motion," + " so no single frame of reference is absolute." + ) + words = entry.reference.split() + random.Random(42).shuffle(words) + shuffled = " ".join(words) + + ref_ppl, err = _judge_perplexity(entry.reference, entry.prompt, judge_tokenizer, judge_model) + assert err is None + fluent_ppl, err = _judge_perplexity(fluent, entry.prompt, judge_tokenizer, judge_model) + assert err is None + fluent_score = _ratio_to_score(fluent_ppl / ref_ppl) * _compute_repetition_penalty(fluent) + shuf_ppl, err = _judge_perplexity(shuffled, entry.prompt, judge_tokenizer, judge_model) + assert err is None + shuffled_score = _ratio_to_score(shuf_ppl / ref_ppl) * _compute_repetition_penalty(shuffled) + + assert fluent_score >= 60.0, fluent_score + assert shuffled_score < 50.0, (shuf_ppl, ref_ppl) + assert fluent_score - shuffled_score >= 30.0 + + +def test_reference_perplexities_match_pinned_values(judge): + """Judge-revision/reference drift guard: the judge's perplexity on a few + fixed reference strings must match values measured at bake-off time + (2026-08-20, Qwen2.5-0.5B@060db649, fp32 CPU). A judge unpin or a silent + reference edit moves these.""" + judge_model, judge_tokenizer = judge + from transformer_lens.benchmarks.text_quality import _judge_perplexity + from transformer_lens.benchmarks.text_quality_profiles import PIVOT_SENTENCES + + pinned = { + ("en", 0): 30.79, + ("fr", 0): 81.85, + ("zh", 0): 53.15, + } + for (lang, idx), expected in pinned.items(): + ppl, err = _judge_perplexity(PIVOT_SENTENCES[lang][idx], "", judge_tokenizer, judge_model) + assert err is None + assert ppl == pytest.approx(expected, rel=0.15), (lang, idx, ppl) + + +def test_continuation_references_share_a_scale(judge): + """Per-language reference PPLs must sit within 3.5x of the language + median: an outlier reference makes its prompt's bar proportionally looser + (the old en[3] measured 31.3 vs median 8.9 and handed gpt2 a clamp-100 on + output worse than its 62-scoring sibling prompt).""" + judge_model, judge_tokenizer = judge + from transformer_lens.benchmarks.text_quality import _judge_perplexity + from transformer_lens.benchmarks.text_quality_profiles import CONTINUATION_PROMPTS + + for lang, prompts in CONTINUATION_PROMPTS.items(): + ppls = [] + for pp in prompts: + ppl, err = _judge_perplexity(pp.reference, pp.prompt, judge_tokenizer, judge_model) + assert err is None, (lang, err) + ppls.append(ppl) + median = sorted(ppls)[len(ppls) // 2] + for i, ppl in enumerate(ppls): + assert ppl <= 3.5 * median, (lang, i, round(ppl, 1), round(median, 1)) diff --git a/tests/integration/model_bridge/compatibility/test_bridge_cache_behavior.py b/tests/integration/model_bridge/compatibility/test_bridge_cache_behavior.py index 0a347cac49..506f84b6af 100644 --- a/tests/integration/model_bridge/compatibility/test_bridge_cache_behavior.py +++ b/tests/integration/model_bridge/compatibility/test_bridge_cache_behavior.py @@ -128,9 +128,8 @@ class TestCacheEqualityWithHookedTransformer: def test_cache_values_match(self, bridge_compat, reference_ht): """Cache activations should match between bridge and HookedTransformer. - Note: Raw attention scores use different masking sentinels: - HookedTransformer uses -inf, Bridge uses torch.finfo(dtype).min. - Unmasked scores and resulting patterns should still match. + Compatibility mode normalizes masked attention scores to -inf, matching + HookedTransformer. Unmasked scores and resulting patterns should match. """ prompt = "Hello World!" _, bridge_cache = bridge_compat.run_with_cache(prompt) @@ -148,7 +147,7 @@ def test_cache_values_match(self, bridge_compat, reference_ht): ), f"Shape mismatch for {hook}: {ht_act.shape} vs {bridge_act.shape}" if hook == "blocks.0.attn.hook_attn_scores": - # Different masking sentinels — compare only unmasked positions + # Compare the informative, unmasked scores separately. masked = torch.isinf(ht_act) unmasked = ~masked assert torch.allclose( diff --git a/tests/integration/model_bridge/helpers.py b/tests/integration/model_bridge/helpers.py index 19e271323b..ebf5cfbf9c 100644 --- a/tests/integration/model_bridge/helpers.py +++ b/tests/integration/model_bridge/helpers.py @@ -1,8 +1,36 @@ """Shared helpers for bridge integration tests.""" +import copy + import torch +def make_tiny_pair(hf_config, arch_name, *, loader=None): + """Seeded tiny (bridge, ref) pair sharing identical weights. + + ``loader(config) -> model`` builds each side (default + ``AutoModelForCausalLM.from_config``); ref keeps the seeded init, hf gets a + state-dict copy, and the bridge wraps hf with eager attention on cpu. + """ + from transformers import AutoModelForCausalLM + + from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, + ) + + if loader is None: + loader = AutoModelForCausalLM.from_config + hf_config._attn_implementation = "eager" + torch.manual_seed(42) + ref = loader(hf_config).eval() + hf = loader(copy.deepcopy(hf_config)).eval() + hf.load_state_dict(ref.state_dict()) + bridge = build_bridge_from_module( + hf, arch_name, hf_config=copy.deepcopy(hf_config), tokenizer=None, device="cpu" + ).eval() + return bridge, ref + + def assert_bridge_matches_hf(bridge, *args, atol: float = 1e-5, **kwargs) -> None: """Assert the bridge's logits match its wrapped HF model on the same inputs. diff --git a/tests/integration/model_bridge/test_attention_score_sentinel.py b/tests/integration/model_bridge/test_attention_score_sentinel.py new file mode 100644 index 0000000000..424198df58 --- /dev/null +++ b/tests/integration/model_bridge/test_attention_score_sentinel.py @@ -0,0 +1,67 @@ +"""Compatibility-mode attention-score sentinel regression coverage.""" + +import torch + +SCORES = "blocks.0.attn.hook_attn_scores" +PATTERN = "blocks.0.attn.hook_pattern" + + +def test_gpt2_compatibility_scores_use_negative_infinity( + gpt2_bridge_compat, gpt2_hooked_processed +) -> None: + """GPT-2's direct HF mask is normalized before the compatibility hook.""" + tokens = gpt2_hooked_processed.to_tokens("The capital of France is") + _, bridge_cache = gpt2_bridge_compat.run_with_cache(tokens, names_filter=[SCORES]) + _, hooked_cache = gpt2_hooked_processed.run_with_cache(tokens, names_filter=[SCORES]) + + bridge_scores, hooked_scores = bridge_cache[SCORES], hooked_cache[SCORES] + causal_mask = torch.isneginf(hooked_scores) + assert causal_mask.any() + assert torch.isneginf(bridge_scores[causal_mask]).all() + torch.testing.assert_close( + bridge_scores[~causal_mask], hooked_scores[~causal_mask], rtol=0, atol=0 + ) + + +def test_gpt2_left_padding_uses_negative_infinity_and_finite_patterns( + gpt2_bridge_compat, +) -> None: + """Fully masked pad queries are zeroed after softmax in compatibility mode.""" + long = gpt2_bridge_compat.to_tokens("The capital of France is") + short = gpt2_bridge_compat.to_tokens("Paris") + n_pad = long.shape[1] - short.shape[1] + padded_short = torch.cat([torch.zeros_like(long[:, :n_pad]), short], dim=1) + tokens = torch.cat([long, padded_short], dim=0) + attention_mask = torch.cat( + [ + torch.ones_like(long), + torch.cat([torch.zeros_like(long[:, :n_pad]), torch.ones_like(short)], dim=1), + ], + dim=0, + ) + + _, cache = gpt2_bridge_compat.run_with_cache( + tokens, attention_mask=attention_mask, names_filter=[SCORES, PATTERN] + ) + scores, pattern = cache[SCORES], cache[PATTERN] + key_padding = ~attention_mask.bool()[:, None, None, :] + causal = torch.triu(torch.ones(long.shape[1], long.shape[1], dtype=torch.bool), diagonal=1)[ + None, None + ] + masked = (key_padding | causal).expand_as(scores) + + assert torch.isneginf(scores[masked]).all() + assert torch.isfinite(pattern).all() + + +def test_gpt2_mixed_dtype_mask_is_normalized_before_addition(gpt2_bridge_compat) -> None: + """A lower-precision HF mask sentinel must survive score upcasting.""" + scores = torch.zeros(1, 1, 2, 2, dtype=torch.float32) + attention_mask = torch.zeros_like(scores, dtype=torch.float16) + attention_mask[..., 0, 1] = torch.finfo(torch.float16).min + + actual = gpt2_bridge_compat.blocks[0].attn._apply_reconstruct_attention_mask( + scores, attention_mask, seq_len=2 + ) + + assert torch.isneginf(actual[..., 0, 1]).all() diff --git a/tests/integration/model_bridge/test_attention_weight_accessors.py b/tests/integration/model_bridge/test_attention_weight_accessors.py index 5390471f86..8afc91dbd5 100644 --- a/tests/integration/model_bridge/test_attention_weight_accessors.py +++ b/tests/integration/model_bridge/test_attention_weight_accessors.py @@ -51,7 +51,7 @@ def llama_bridge(): @pytest.fixture(scope="module") -def gpt2_bridge(): +def tiny_gpt2_bridge(): """Tiny GPT-2: square Conv1D c_proj (control — no-transpose path must stay correct).""" from transformers import GPT2Config, GPT2LMHeadModel @@ -116,8 +116,8 @@ def test_w_v_uses_kv_heads(self, llama_bridge): class TestConv1DAccessorParity: - def test_w_o_reproduces_c_proj(self, gpt2_bridge): - bridge, hf_model = gpt2_bridge + def test_w_o_reproduces_c_proj(self, tiny_gpt2_bridge): + bridge, hf_model = tiny_gpt2_bridge attn = bridge.blocks[0].attn w_o = attn.W_O assert w_o.shape == (4, 16, 64) @@ -126,3 +126,59 @@ def test_w_o_reproduces_c_proj(self, gpt2_bridge): expected = c_proj(z.reshape(2, 64)) actual = torch.einsum("bhd,hdm->bm", z, w_o) + attn.b_O assert torch.allclose(actual, expected, atol=1e-5) + + +class TestWeightCircuitsGQA: + """QK/OV/composition circuits expand grouped K/V to n_heads (issue #1553). + + Pre-fix, every property below raised at FactoredMatrix construction on GQA + models because the grouped [n_kv_heads] axis cannot broadcast against the + per-query-head [n_heads] axis. + """ + + def test_qk_factors_align_to_query_heads(self, llama_bridge): + bridge, _ = llama_bridge + QK = bridge.QK + assert QK.A.shape == (2, 4, 64, 16) + assert QK.B.shape == (2, 4, 16, 64) + # Query head h reads kv head h // (n_heads // n_kv_heads). + for h in range(4): + assert torch.equal(QK.B[0, h], bridge.blocks[0].attn.W_K[h // 2].T) + + def test_ov_factors_align_to_query_heads(self, llama_bridge): + bridge, _ = llama_bridge + OV = bridge.OV + assert OV.A.shape == (2, 4, 64, 16) + assert OV.B.shape == (2, 4, 16, 64) + for h in range(4): + assert torch.equal(OV.A[1, h], bridge.blocks[1].attn.W_V[h // 2]) + + def test_for_attn_layers_variants_align(self, llama_bridge): + bridge, _ = llama_bridge + indices, QK = bridge.QK_for_attn_layers() + assert indices == [0, 1] + assert QK.A.shape == (2, 4, 64, 16) + assert QK.B.shape == (2, 4, 16, 64) + _, OV = bridge.OV_for_attn_layers() + assert OV.A.shape == (2, 4, 64, 16) + assert OV.B.shape == (2, 4, 16, 64) + + @pytest.mark.parametrize("mode", ["Q", "K", "V"]) + def test_composition_scores_cover_all_query_heads(self, llama_bridge, mode): + bridge, _ = llama_bridge + result = bridge.all_composition_scores(mode) + assert result.scores.shape == (2, 4, 2, 4) + assert len(result.head_labels) == 8 + + def test_raw_kv_stacks_stay_grouped(self, llama_bridge): + bridge, _ = llama_bridge + assert bridge.W_K.shape == (2, 2, 64, 16) + assert bridge.W_V.shape == (2, 2, 64, 16) + + def test_mha_circuits_untouched(self, tiny_gpt2_bridge): + bridge, _ = tiny_gpt2_bridge + QK, OV = bridge.QK, bridge.OV + assert torch.equal(QK.A, bridge.W_Q) + assert torch.equal(QK.B, bridge.W_K.transpose(-2, -1)) + assert torch.equal(OV.A, bridge.W_V) + assert torch.equal(OV.B, bridge.W_O) diff --git a/tests/integration/model_bridge/test_batched_generate_position_ids.py b/tests/integration/model_bridge/test_batched_generate_position_ids.py new file mode 100644 index 0000000000..a4e6224242 --- /dev/null +++ b/tests/integration/model_bridge/test_batched_generate_position_ids.py @@ -0,0 +1,133 @@ +"""position_ids handling during batched-list generation. + +Batched list input is left-padded internally, so each row's real tokens start at +a different offset. The bridge derives position_ids for that, but only models +that neither reject the kwarg nor derive positions themselves may receive them +(#1626). + +The cached decoding path needs the same gate as the prompt path. Every branch +there supplies position_ids, including a ``total_len - 1`` fallback that counts +pad slots, so gating only the prompt derivation diverts a refused model into the +fallback instead of leaving it alone. + +OPT is the vehicle for the refused case: ``OPTLearnedPositionalEmbedding`` +consumes the attention mask and derives its own positions, so the gate declines +it, while its forward would happily accept the kwarg and use it. +""" + +from __future__ import annotations + +import functools + +import pytest +import torch + +GREEDY = dict(max_new_tokens=4, do_sample=False, verbose=False) +PROMPTS = ["The capital of France is the city of", "Hi"] + + +@pytest.fixture(scope="module") +def opt_bridge(): + """A model the gate refuses. Its positional embedding reads the mask.""" + from transformer_lens.model_bridge import TransformerBridge + + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-OPTForCausalLM", device="cpu", dtype=torch.float32 + ) + bridge.eval() + return bridge + + +def _stack_logits(output) -> torch.Tensor: + logits = output.logits + return torch.stack(list(logits)) if isinstance(logits, (list, tuple)) else logits + + +def _position_ids_per_step(bridge, use_past_kv_cache: bool) -> list: + """The position_ids each forward actually received, one entry per step.""" + seen: list = [] + original = bridge.original_model.forward + + # functools.wraps so the gate still sees the real signature; a bare + # (*args, **kwargs) spy would look like it accepts position_ids. + @functools.wraps(original) + def _spy(*args, **kwargs): + supplied = kwargs.get("position_ids") + seen.append(None if supplied is None else supplied.tolist()) + return original(*args, **kwargs) + + bridge.original_model.forward = _spy + try: + bridge.generate(list(PROMPTS), use_past_kv_cache=use_past_kv_cache, **GREEDY) + finally: + bridge.original_model.forward = original + return seen + + +def test_gate_refuses_opt(opt_bridge) -> None: + """Guards the premise of the tests below: OPT must be the refused case.""" + assert opt_bridge._accepts_derived_position_ids() is False + + +def test_refused_model_generates_identically_with_and_without_cache(opt_bridge) -> None: + """Cached decoding must not change the answer. + + Compared on logits rather than decoded text on purpose: greedy argmax + absorbs the drift and the strings match even when the positions are wrong. + """ + cached = opt_bridge.generate( + list(PROMPTS), use_past_kv_cache=True, output_logits=True, **GREEDY + ) + uncached = opt_bridge.generate( + list(PROMPTS), use_past_kv_cache=False, output_logits=True, **GREEDY + ) + + torch.testing.assert_close(_stack_logits(cached), _stack_logits(uncached), rtol=0, atol=1e-5) + + +def test_refused_model_receives_no_position_ids_on_cached_steps(opt_bridge) -> None: + """The mechanism, not just the symptom. + + The fallback supplies a per-batch constant, so a coarser check can miss it; + assert the kwarg never reaches a model that derives positions itself. + """ + assert _position_ids_per_step(opt_bridge, use_past_kv_cache=True) == [None] * ( + GREEDY["max_new_tokens"] + ) + + +def test_accepted_model_still_receives_per_row_position_ids(distilgpt2_bridge) -> None: + """The gate must not disarm the models it was never meant to exclude.""" + if distilgpt2_bridge.tokenizer.pad_token_id is None: + distilgpt2_bridge.tokenizer.pad_token = distilgpt2_bridge.tokenizer.eos_token + + seen = _position_ids_per_step(distilgpt2_bridge, use_past_kv_cache=True) + + assert seen[0] is not None, "prompt step must still receive derived positions" + cached_steps = [step for step in seen[1:] if step is not None] + assert len(cached_steps) == len(seen) - 1, "cached steps must still be supplied" + # Row 1 ("Hi") is left-padded, so its position must be strictly lower than + # row 0's. A pad-counting fallback would give both rows the same value. + first_cached = cached_steps[0] + assert first_cached[1][0] < first_cached[0][0], first_cached + + +def test_accepted_model_generates_identically_with_and_without_cache(distilgpt2_bridge) -> None: + """Control for the parity property on a model the gate allows. + + Looser than the OPT case at 1e-3. Cached decoding and full recompute sum in + different orders, which on distilgpt2's logit scale of ~132 shows as 1.4e-04, + or 1e-06 relative. The regression this guards moves logits by ~0.3, so the + margin is still more than two orders of magnitude. + """ + if distilgpt2_bridge.tokenizer.pad_token_id is None: + distilgpt2_bridge.tokenizer.pad_token = distilgpt2_bridge.tokenizer.eos_token + + cached = distilgpt2_bridge.generate( + list(PROMPTS), use_past_kv_cache=True, output_logits=True, **GREEDY + ) + uncached = distilgpt2_bridge.generate( + list(PROMPTS), use_past_kv_cache=False, output_logits=True, **GREEDY + ) + + torch.testing.assert_close(_stack_logits(cached), _stack_logits(uncached), rtol=0, atol=1e-3) diff --git a/tests/integration/model_bridge/test_bert_verification_fixes.py b/tests/integration/model_bridge/test_bert_verification_fixes.py new file mode 100644 index 0000000000..b637432e2c --- /dev/null +++ b/tests/integration/model_bridge/test_bert_verification_fixes.py @@ -0,0 +1,93 @@ +"""The four defects that kept google-bert/bert-base-cased at FAILED since June. + +1. The tokenizer fallback installs bos_token='<|endoftext|>' on BERT (which has + none) and to_tokens string-prepended it, WordPiece-shredding it into 8 subword + tokens at the front of every input. +2. The tokenizer_prepends_bos probe compared position 0 against that fake BOS id + instead of recognizing [CLS], desyncing the bridge from HookedTransformer. +3. The component harness fed float tensors to token_type_embed (an nn.Embedding). +4. Phase 2 loaded the masked LM into HookedTransformer — a causal decoder — and + graded the bridge against a bidirectional model run under a causal mask. +""" + +import pytest +import torch + +from transformer_lens.model_bridge import TransformerBridge + +MODEL = "google-bert/bert-base-cased" +TEXT = ( + "Natural language processing tasks, such as question answering, " + "machine translation, reading comprehension, and summarization, " + "are typically approached with supervised learning." +) + + +@pytest.fixture(scope="module") +def bert(): + return TransformerBridge.boot_transformers(MODEL, device="cpu") + + +def test_probe_recognizes_cls_as_prepended_bos(bert) -> None: + assert bert.cfg.tokenizer_prepends_bos is True + + +def test_to_tokens_is_clean_wordpiece(bert) -> None: + tokens = bert.to_tokens(TEXT) + assert tokens.shape[1] == 32, tokens.shape + decoded = bert.tokenizer.convert_ids_to_tokens(tokens[0]) + assert decoded[:2] == ["[CLS]", "Natural"], decoded[:6] + + +def test_prepend_false_strips_cls_matching_hooked_transformer(bert) -> None: + assert bert.to_tokens(TEXT, prepend_bos=False).shape[1] == 31 + + +def test_non_atomic_bos_is_never_string_prepended(bert) -> None: + """Defense in depth: even with the flag forced to the manual-prepend path, + a BOS the tokenizer would shred must not be prepended.""" + original = bert.cfg.tokenizer_prepends_bos + bert.cfg.tokenizer_prepends_bos = False + try: + tokens = bert.to_tokens(TEXT, prepend_bos=True) + finally: + bert.cfg.tokenizer_prepends_bos = original + assert tokens.shape[1] == 32, tokens.shape # 40 when the shred happens + decoded = bert.tokenizer.convert_ids_to_tokens(tokens[0]) + assert decoded[1] == "Natural", decoded[:6] + + +def test_component_harness_feeds_ints_to_embedding_tables(bert) -> None: + from transformers import AutoModelForMaskedLM + + from transformer_lens.benchmarks.component_benchmark import benchmark_all_components + + hf = AutoModelForMaskedLM.from_pretrained(MODEL, dtype=torch.float32).eval() + result = benchmark_all_components(bert, hf) + assert result.passed, result.message + assert "token_type_embed" not in str(result.details), result.details + + +def test_phase2_never_grades_against_a_causal_reference() -> None: + """A masked LM loaded into HookedTransformer runs bidirectional weights under + a causal mask — every comparison against it is noise. Phase 2 must skip that + reference; numerical checks fall back to the Phase 1 HF logits.""" + from transformer_lens.benchmarks.main_benchmark import run_benchmark_suite + from transformer_lens.benchmarks.utils import BenchmarkSeverity + + results = run_benchmark_suite( + model_name=MODEL, + device="cpu", + phases=[1, 2], + use_hf_reference=True, + use_ht_reference=True, + enable_compatibility_mode=False, + verbose=False, + track_memory=False, + ) + hard_failures = [ + r + for r in results + if not r.passed and r.severity not in (BenchmarkSeverity.SKIPPED, BenchmarkSeverity.WARNING) + ] + assert not hard_failures, [f"{r.name}: {r.message[:90]}" for r in hard_failures] diff --git a/tests/integration/model_bridge/test_bloom_gated_hooks.py b/tests/integration/model_bridge/test_bloom_gated_hooks.py new file mode 100644 index 0000000000..21ce1a609d --- /dev/null +++ b/tests/integration/model_bridge/test_bloom_gated_hooks.py @@ -0,0 +1,44 @@ +"""Bloom's gated attention hooks must fire like every other joint-QKV bridge. + +`BloomAttentionBridge` overrides both `forward` and `_reconstruct_attention`, and +the overrides projected Q/K/V directly and always took the plain output +projection. `hook_result`, `hook_q_input`, `hook_k_input`, `hook_v_input` and +`hook_attn_in` therefore never fired — enabling `use_attn_result` on a Bloom +model silently produced nothing. +""" + +import pytest +import torch + +from transformer_lens.benchmarks import benchmark_gated_hooks_fire +from transformer_lens.model_bridge import TransformerBridge + +MODEL = "bigscience/bloom-560m" +PROMPT = "The theory of relativity explains that the speed of light" + + +@pytest.fixture(scope="module") +def bridge(): + return TransformerBridge.boot_transformers(MODEL, device="cpu") + + +def test_every_gated_hook_fires(bridge) -> None: + result = benchmark_gated_hooks_fire(bridge, PROMPT) + assert result.passed, result.message + fired = (result.details or {}).get("fired_counts", {}) + assert fired and all(count > 0 for count in fired.values()), fired + + +@pytest.mark.parametrize("flag", ["use_attn_result", "use_split_qkv_input", "use_attn_in"]) +def test_gated_paths_preserve_the_output(bridge, flag: str) -> None: + """The fork re-parameterizes the same math, so logits must not move beyond + the op-order noise a correct implementation shows (gpt2 1e-4, pythia 4e-3).""" + tokens = bridge.to_tokens(PROMPT) + with torch.no_grad(): + baseline = bridge(tokens).float() + setattr(bridge.cfg, flag, True) + try: + gated = bridge(tokens).float() + finally: + setattr(bridge.cfg, flag, False) + torch.testing.assert_close(gated, baseline, atol=5e-3, rtol=1e-3) diff --git a/tests/integration/model_bridge/test_bridge_generate_stopping_criteria.py b/tests/integration/model_bridge/test_bridge_generate_stopping_criteria.py index 92256c4d90..7dacaf8002 100644 --- a/tests/integration/model_bridge/test_bridge_generate_stopping_criteria.py +++ b/tests/integration/model_bridge/test_bridge_generate_stopping_criteria.py @@ -18,14 +18,11 @@ cached-eager-attention path can NaN (issue #1322). """ -import platform import pytest import torch from transformers import StoppingCriteria, StoppingCriteriaList -_MACOS_ARM64 = platform.system() == "Darwin" and platform.machine() == "arm64" - # Common kwargs for the greedy, macOS-safe, token-returning generate calls below. _GEN = dict(do_sample=False, use_past_kv_cache=False, return_type="tokens", verbose=False) @@ -225,7 +222,6 @@ def test_batched_generation_stops(bridge_with_pad): assert torch.equal(out1, out2), "batched greedy generation must be deterministic" -@pytest.mark.skipif(_MACOS_ARM64, reason="Upstream macOS-arm64 KV-cache NaN, see issue #1322.") def test_stop_string_with_kv_cache(bridge): """stop_strings also works on the default KV-cache path (not only the no-cache path).""" tokens = bridge.to_tokens("The quick brown") diff --git a/tests/integration/model_bridge/test_bridge_integration.py b/tests/integration/model_bridge/test_bridge_integration.py index 3e2a8a79ec..9ae02e4068 100644 --- a/tests/integration/model_bridge/test_bridge_integration.py +++ b/tests/integration/model_bridge/test_bridge_integration.py @@ -573,73 +573,6 @@ def test_get_params_configuration_mismatch(): bridge.cfg.n_layers = original_n_layers -def test_get_params_multi_query_attention_reshaping(): - """Test Multi-Query Attention weight reshaping logic without requiring a large model. - - This test verifies that the get_params function can correctly handle different - weight shapes that occur in Multi-Query Attention architectures, where K and V - weights have different shapes than Q weights. - """ - model_name = "gpt2" - bridge = TransformerBridge.boot_transformers(model_name) - - # Get the original attention layer to modify - original_attn = bridge.blocks[0].attn - original_k_weight = original_attn.k.weight.clone() - original_v_weight = original_attn.v.weight.clone() - - try: - # Test case 1: Simulate MQA where K and V have shape [d_head, d_model] - # instead of [d_model, d_model] - d_head = bridge.cfg.d_head - d_model = bridge.cfg.d_model - - # Create MQA-style K and V weights with shape [d_head, d_model] - mqa_k_weight = torch.randn( - d_head, d_model, dtype=original_k_weight.dtype, device=original_k_weight.device - ) - mqa_v_weight = torch.randn( - d_head, d_model, dtype=original_v_weight.dtype, device=original_v_weight.device - ) - - # Temporarily replace the weights - original_attn.k.weight.data = mqa_k_weight - original_attn.v.weight.data = mqa_v_weight - - # This should work without raising exceptions - params_dict = bridge.get_params() - - # Verify the weights were reshaped correctly - # For MQA: K and V should be expanded from [d_head, d_model] to [n_heads, d_model, d_head] (same as Q) - k_param = params_dict["blocks.0.attn.W_K"] - v_param = params_dict["blocks.0.attn.W_V"] - - expected_shape = (bridge.cfg.n_heads, bridge.cfg.d_model, bridge.cfg.d_head) - assert ( - k_param.shape == expected_shape - ), f"K weight should be reshaped to {expected_shape}, got {k_param.shape}" - assert ( - v_param.shape == expected_shape - ), f"V weight should be reshaped to {expected_shape}, got {v_param.shape}" - - # Verify that all heads contain the transposed MQA weight (due to transpose + expand operation) - expected_k_per_head = mqa_k_weight.transpose(0, 1) # [d_head, d_model] -> [d_model, d_head] - expected_v_per_head = mqa_v_weight.transpose(0, 1) # [d_head, d_model] -> [d_model, d_head] - - for head_idx in range(bridge.cfg.n_heads): - assert torch.allclose( - k_param[head_idx], expected_k_per_head - ), f"K head {head_idx} should match transposed MQA weight" - assert torch.allclose( - v_param[head_idx], expected_v_per_head - ), f"V head {head_idx} should match transposed MQA weight" - - finally: - # Always restore original weights - original_attn.k.weight.data = original_k_weight - original_attn.v.weight.data = original_v_weight - - def test_TransformerBridge_hooks_backward_hooks(): """Test that TransformerBridge.hooks() correctly registers backward hooks. diff --git a/tests/integration/model_bridge/test_bridge_layer_past_cache.py b/tests/integration/model_bridge/test_bridge_layer_past_cache.py new file mode 100644 index 0000000000..31261dc7e9 --- /dev/null +++ b/tests/integration/model_bridge/test_bridge_layer_past_cache.py @@ -0,0 +1,72 @@ +"""Architectures that name the KV cache `layer_past` must still populate it. + +GPT-NeoX, GPT-J, Bloom, Falcon, MPT, CodeGen and GPT-BigCode take the cache as +`layer_past`; everything modern takes `past_key_values`. Reading only the latter +left the cache empty, so each decode step attended to itself alone and generation +silently ignored the prompt — pythia-1.4b answered every prompt with +" the first time.\\n\\n\\n...". +""" + +import pytest +import torch +from transformers import DynamicCache + +from transformer_lens.model_bridge import TransformerBridge + +MODEL = "EleutherAI/pythia-70m" # GPTNeoX: takes `layer_past` +PROMPT = "The theory of relativity explains that" + + +@pytest.fixture(scope="module") +def bridge(): + return TransformerBridge.boot_transformers(MODEL, device="cpu") + + +def test_forward_populates_a_layer_past_cache(bridge) -> None: + cache = DynamicCache() + tokens = bridge.to_tokens(PROMPT) + with torch.no_grad(): + bridge(tokens, past_key_values=cache, use_cache=True) + assert ( + cache.get_seq_length() == tokens.shape[1] + ), f"cache holds {cache.get_seq_length()} of {tokens.shape[1]} tokens" + + +def test_cached_generation_matches_uncached(bridge) -> None: + """The cache is an optimization: it must not change what is generated.""" + outputs = {} + for use_cache in (True, False): + torch.manual_seed(42) + outputs[use_cache] = bridge.generate( + PROMPT, + max_new_tokens=15, + do_sample=False, + verbose=False, + use_past_kv_cache=use_cache, + ) + assert outputs[True] == outputs[False], f"cached={outputs[True]!r}\nuncached={outputs[False]!r}" + + +def test_generation_depends_on_the_prompt(bridge) -> None: + """Engagement check: an empty cache made every prompt yield the same text, + which prompt-independent output is the loudest symptom of.""" + torch.manual_seed(42) + a = bridge.generate( + "The theory of relativity explains that", + max_new_tokens=12, + do_sample=False, + verbose=False, + use_past_kv_cache=True, + ) + torch.manual_seed(42) + b = bridge.generate( + "Modern computing relies heavily on", + max_new_tokens=12, + do_sample=False, + verbose=False, + use_past_kv_cache=True, + ) + assert ( + a[len("The theory of relativity explains that") :] + != b[len("Modern computing relies heavily on") :] + ) diff --git a/tests/integration/model_bridge/test_bridge_qk_ov_vs_hooked_gqa.py b/tests/integration/model_bridge/test_bridge_qk_ov_vs_hooked_gqa.py new file mode 100644 index 0000000000..998003419f --- /dev/null +++ b/tests/integration/model_bridge/test_bridge_qk_ov_vs_hooked_gqa.py @@ -0,0 +1,54 @@ +"""GQA weight-circuit parity between TransformerBridge and HookedTransformer. + +Acceptance for https://github.com/TransformerLensOrg/TransformerLens/issues/1553: +bridge.QK/OV on a GQA model must match HookedTransformer.QK/OV (whose +GroupedQueryAttention repeat_interleaves grouped K/V) within fp tolerance. +Qwen2-0.5B (14 query heads, 2 kv heads) is a small ungated GQA model both +systems support; it is not CI-cached, hence @pytest.mark.slow. +""" + +import pytest +import torch + +from transformer_lens import HookedTransformer +from transformer_lens.model_bridge.bridge import TransformerBridge + +MODEL = "Qwen/Qwen2-0.5B" + + +@pytest.mark.slow +class TestGQAWeightCircuitParity: + @pytest.fixture(scope="class") + def bridge_and_hooked(self): + bridge = TransformerBridge.boot_transformers(MODEL, device="cpu") + hooked = HookedTransformer.from_pretrained_no_processing(MODEL, device="cpu") + return bridge, hooked + + def test_model_is_gqa(self, bridge_and_hooked): + bridge, _ = bridge_and_hooked + assert bridge.cfg.n_key_value_heads is not None + assert bridge.cfg.n_key_value_heads < bridge.cfg.n_heads + + def test_qk_factors_match_hooked(self, bridge_and_hooked): + bridge, hooked = bridge_and_hooked + torch.testing.assert_close(bridge.QK.A, hooked.QK.A) + torch.testing.assert_close(bridge.QK.B, hooked.QK.B) + + def test_ov_factors_match_hooked(self, bridge_and_hooked): + bridge, hooked = bridge_and_hooked + torch.testing.assert_close(bridge.OV.A, hooked.OV.A) + torch.testing.assert_close(bridge.OV.B, hooked.OV.B) + + def test_qk_product_matches_hooked(self, bridge_and_hooked): + bridge, hooked = bridge_and_hooked + torch.testing.assert_close(bridge.QK.A[0] @ bridge.QK.B[0], hooked.QK.A[0] @ hooked.QK.B[0]) + + def test_for_attn_layers_match_hooked(self, bridge_and_hooked): + bridge, hooked = bridge_and_hooked + indices, QK = bridge.QK_for_attn_layers() + assert indices == list(range(bridge.cfg.n_layers)) + torch.testing.assert_close(QK.A, hooked.QK.A) + torch.testing.assert_close(QK.B, hooked.QK.B) + _, OV = bridge.OV_for_attn_layers() + torch.testing.assert_close(OV.A, hooked.OV.A) + torch.testing.assert_close(OV.B, hooked.OV.B) diff --git a/tests/integration/model_bridge/test_cohere2_nope_no_warning.py b/tests/integration/model_bridge/test_cohere2_nope_no_warning.py new file mode 100644 index 0000000000..71214936a7 --- /dev/null +++ b/tests/integration/model_bridge/test_cohere2_nope_no_warning.py @@ -0,0 +1,50 @@ +"""Cohere2 NoPE layers must not warn about missing position_embeddings. + +Full-attention (global) layers deliberately null position_embeddings — RoPE is +sliding-window-only on Cohere2 — so the base bridge's missing-RoPE RuntimeWarning +is spurious there. +""" + +from __future__ import annotations + +import warnings + +import pytest +import torch +from transformers import AutoModelForCausalLM, Cohere2Config + +from transformer_lens.model_bridge.sources import build_bridge_from_module + + +@pytest.fixture(scope="module") +def cohere2_bridge(): + cfg = Cohere2Config( + vocab_size=64, + hidden_size=32, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + intermediate_size=64, + max_position_embeddings=32, + sliding_window=4, + sliding_window_pattern=2, + layer_types=["sliding_attention", "full_attention"], + ) + cfg._attn_implementation = "eager" + torch.manual_seed(0) + hf = AutoModelForCausalLM.from_config(cfg).eval() + return build_bridge_from_module( + hf, "Cohere2ForCausalLM", hf_config=cfg, tokenizer=None, device="cpu" + ).eval() + + +def test_nope_layer_forward_emits_no_rope_warning(cohere2_bridge) -> None: + tokens = torch.tensor([[1, 2, 3, 4, 5]]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with torch.no_grad(): + cohere2_bridge.run_with_cache(tokens, names_filter=["blocks.1.attn.hook_pattern"]) + rope_warnings = [ + w for w in caught if issubclass(w.category, RuntimeWarning) and "RoPE" in str(w.message) + ] + assert rope_warnings == [], [str(w.message) for w in rope_warnings] diff --git a/tests/integration/model_bridge/test_encdec_string_generation.py b/tests/integration/model_bridge/test_encdec_string_generation.py new file mode 100644 index 0000000000..6fa8df4bc5 --- /dev/null +++ b/tests/integration/model_bridge/test_encdec_string_generation.py @@ -0,0 +1,202 @@ +"""bridge.generate(str) on encoder-decoder models must tokenize with the +tokenizer's native recipe. to_tokens' decoder-style BOS policy injected a +stray and dropped the trailing , corrupting encoder input — m2m100 +degenerated into token loops; Marian/T5 degraded silently. + +A tiny-random M2M100 is used because its lang-code recipe genuinely differs +from to_tokens output (Marian's happens to coincide, so it cannot +discriminate); random weights are fine — greedy decoding is deterministic, so +outputs match iff the encoder input matches. +""" + +import pytest +import torch + +pytest.importorskip("transformers") + + +def test_m2m100_string_generation_matches_native_recipe(): + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + text = "Ik moet nu echt gaan slapen." + native_ids = bridge.tokenizer(text, return_tensors="pt")["input_ids"] + to_tokens_ids = bridge.to_tokens(text) + assert ( + native_ids[0].tolist() != to_tokens_ids[0].tolist() + ), "precondition: recipes must differ or this test cannot discriminate" + # Assert on the tokens generate() actually consumed (random tiny weights + # emit input-independent output, so generated text cannot discriminate). + _, fed = bridge.generate( + text, max_new_tokens=4, temperature=0.0, return_type="tokens", return_input_tokens=True + ) + assert isinstance(fed, torch.Tensor) + assert fed[0].tolist() == native_ids[0].tolist(), (fed[0].tolist(), native_ids[0].tolist()) + + +def test_m2m100_batched_list_generation_matches_native_recipe(): + """The list-input branch had the same corruption (unpatched in the first + fix): batched generate on M2M100/MBart fed to_tokens-mangled encoder + input. Both rows must match the tokenizer's own padded batch encoding.""" + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + texts = ["Ik moet nu echt gaan slapen.", "Ik kan niet zo leven."] + native = bridge.tokenizer(texts, return_tensors="pt", padding=True)["input_ids"] + _, fed = bridge.generate( + texts, max_new_tokens=4, temperature=0.0, return_type="tokens", return_input_tokens=True + ) + assert isinstance(fed, torch.Tensor) + assert fed.tolist() == native.tolist(), (fed.tolist(), native.tolist()) + + +def test_generation_config_forced_bos_applied_by_default(): + """HF's generate() applies generation_config defaults; bart-large-cnn pins + forced_bos_token_id=0 there and its summaries degrade without it. The + bridge must honor the config value when the caller passes none.""" + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + forced = 7 + bridge.original_model.generation_config.forced_bos_token_id = forced + out = bridge.generate( + "Ik moet nu echt gaan slapen.", max_new_tokens=4, temperature=0.0, return_type="tokens" + ) + assert out[0, 1].item() == forced + + +def test_generation_config_min_length_suppresses_early_eos(): + """bart-large-cnn pins min_length=56 in its generation config; HF's + generate() suppresses EOS until then. Without it the bridge loop can EOS + on step one and emit an empty summary (observed live, scored 0).""" + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + from unittest import mock + + from transformer_lens import utilities as tl_utils + + eos = bridge.original_model.config.eos_token_id + # Sample EOS whenever its logit is finite: the loop's -inf suppression is + # then the ONLY thing that can delay it, so this discriminates exactly + # that mechanism (tiny-random weights never prefer EOS on their own). + real_sample = tl_utils.sample_logits + + def eos_greedy(logits, **kwargs): + out = real_sample(logits, **kwargs) + finite = torch.isfinite(logits[:, eos]) + out[finite] = eos + return out + + bridge.original_model.generation_config.min_length = 10 + with mock.patch.object(tl_utils, "sample_logits", eos_greedy): + out = bridge.generate( + "Ik moet nu echt gaan slapen.", + max_new_tokens=16, + temperature=0.0, + return_type="tokens", + stop_at_eos=True, + ) + decoder_part = out[0, 1:].tolist() + # Without suppression EOS lands at decoder position 1; with it, no EOS + # before the floor and EOS immediately after it lifts. + assert not any(t == eos for t in decoder_part[:8]), decoder_part + assert eos in decoder_part, decoder_part + + +def test_generation_config_no_repeat_ngram_applied(): + """bart-large-cnn pins no_repeat_ngram_size=3; HF applies it by default. + Without it greedy decoding falls into a BOS attractor (observed live: + empty summary, scored 0). Force an attractor token and assert the + processor breaks the loop.""" + from unittest import mock + + from transformer_lens import utilities as tl_utils + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + attractor = 5 + real_sample = tl_utils.sample_logits + + def prefer_attractor(logits, **kwargs): + out = real_sample(logits, **kwargs) + allowed = torch.isfinite(logits[:, attractor]) + out[allowed] = attractor + return out + + bridge.original_model.generation_config.no_repeat_ngram_size = 2 + with mock.patch.object(tl_utils, "sample_logits", prefer_attractor): + out = bridge.generate( + "Ik moet nu echt gaan slapen.", + max_new_tokens=8, + temperature=0.0, + return_type="tokens", + stop_at_eos=False, + ) + seq = out[0].tolist() + runs = [seq[i] == seq[i + 1] == attractor for i in range(len(seq) - 1)] + # A (5,5) bigram may occur once, but 5,5,5 requires repeating it — banned. + assert not any( + seq[i] == seq[i + 1] == seq[i + 2] == attractor for i in range(len(seq) - 2) + ), seq + + +def test_batched_unequal_rows_match_solo_generation(): + """Id equality can't see mask handling: the batched enc-dec path fed + native ids but no attention mask, so the short row of an unequal batch + attended over pads. Greedy decoding of the short prompt must be identical + batched and solo.""" + from transformer_lens.model_bridge import TransformerBridge + + try: + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-M2M100ForConditionalGeneration", device="cpu" + ) + except (OSError, ConnectionError, TimeoutError) as exc: + pytest.skip(f"tiny-random-m2m100 unavailable offline: {exc}") + + short = "Ik slaap." + long = "Ik moet nu echt heel snel gaan slapen want het is al veel te laat geworden." + # Logits-level: argmax can survive unmasked pads on a tiny model, the + # step-0 distribution cannot. + solo = bridge.generate( + short, max_new_tokens=2, temperature=0.0, return_type="tokens", output_logits=True + ) + batched = bridge.generate( + [short, long], max_new_tokens=2, temperature=0.0, return_type="tokens", output_logits=True + ) + solo_step0 = solo.logits[0][0] + batched_step0_row0 = batched.logits[0][0] + assert torch.allclose(solo_step0, batched_step0_row0, atol=1e-4), float( + (solo_step0 - batched_step0_row0).abs().max() + ) diff --git a/tests/integration/model_bridge/test_generate_attention_mask.py b/tests/integration/model_bridge/test_generate_attention_mask.py new file mode 100644 index 0000000000..7fbe82b71c --- /dev/null +++ b/tests/integration/model_bridge/test_generate_attention_mask.py @@ -0,0 +1,274 @@ +"""Generation from an already-padded prompt. + +``generate()`` had no way to be told which prompt tokens are padding, so a +pre-padded tensor generated as though its pads were real context: every real +token's position was shifted and the continuation diverged from the same prompt +unpadded. See #1612. + +Two routes now work. ``attention_mask`` states the padding explicitly, and +``padding_side`` — which the bridge accepted but never applied to token input — +reads it off the pad token. Only the explicit mask can express an interior gap +or a pad id that also occurs as a real token. +""" + +from __future__ import annotations + +import copy + +import pytest +import torch + +GREEDY = dict(max_new_tokens=5, do_sample=False, verbose=False) + + +@pytest.fixture(scope="module") +def prompt(distilgpt2_bridge) -> torch.Tensor: + return distilgpt2_bridge.to_tokens("The capital of France is") + + +@pytest.fixture(scope="module") +def unpadded_continuation(distilgpt2_bridge, prompt) -> list[int]: + return distilgpt2_bridge.generate(prompt, **GREEDY)[0, prompt.shape[1] :].tolist() + + +def _left_pad(bridge, tokens: torch.Tensor, n_pad: int) -> tuple[torch.Tensor, torch.Tensor]: + pad_id = bridge.tokenizer.pad_token_id + if pad_id is None: + pad_id = bridge.tokenizer.eos_token_id + padded = torch.cat([torch.full((1, n_pad), pad_id, dtype=tokens.dtype), tokens], dim=1) + mask = torch.cat( + [torch.zeros(1, n_pad, dtype=torch.long), torch.ones(1, tokens.shape[1], dtype=torch.long)], + dim=1, + ) + return padded, mask + + +@pytest.mark.parametrize("use_past_kv_cache", [True, False]) +@pytest.mark.parametrize("n_pad", [1, 3, 7]) +def test_attention_mask_recovers_the_unpadded_continuation( + distilgpt2_bridge, prompt, unpadded_continuation, n_pad, use_past_kv_cache +) -> None: + """The whole point: padding a prompt must not change what it generates.""" + padded, mask = _left_pad(distilgpt2_bridge, prompt, n_pad) + + out = distilgpt2_bridge.generate( + padded, attention_mask=mask, use_past_kv_cache=use_past_kv_cache, **GREEDY + ) + + assert out[0, n_pad + prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_without_a_mask_the_pads_are_treated_as_context( + distilgpt2_bridge, prompt, unpadded_continuation +) -> None: + """The unfixed behaviour, pinned so a regression is visible rather than silent. + + padding_side defaults to "right", so leading pads are not recognised and the + continuation drifts. This is the case #1612 reported. + """ + padded, _ = _left_pad(distilgpt2_bridge, prompt, 4) + + out = distilgpt2_bridge.generate(padded, **GREEDY) + + assert out[0, 4 + prompt.shape[1] :].tolist() != unpadded_continuation + + +def test_padding_side_left_is_applied_to_token_input( + distilgpt2_bridge, prompt, unpadded_continuation +) -> None: + """generate() has always documented a padding_side argument, but applied it + only when tokenizing string or list input. For a token tensor it was inert.""" + padded, _ = _left_pad(distilgpt2_bridge, prompt, 4) + + out = distilgpt2_bridge.generate(padded, padding_side="left", **GREEDY) + + assert out[0, 4 + prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_padding_side_is_restored_afterwards(distilgpt2_bridge, prompt) -> None: + """The tokenizer is shared across a session, so the override must not leak.""" + before = distilgpt2_bridge.tokenizer.padding_side + padded, _ = _left_pad(distilgpt2_bridge, prompt, 3) + + distilgpt2_bridge.generate(padded, padding_side="left", **GREEDY) + + assert distilgpt2_bridge.tokenizer.padding_side == before + + +def test_explicit_mask_wins_over_the_padding_side_heuristic( + distilgpt2_bridge, prompt, unpadded_continuation +) -> None: + """A caller who states the padding must not be second-guessed by the pad-token + scan, which here would mask nothing because padding_side is "right".""" + padded, mask = _left_pad(distilgpt2_bridge, prompt, 4) + + out = distilgpt2_bridge.generate(padded, attention_mask=mask, padding_side="right", **GREEDY) + + assert out[0, 4 + prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_interior_gap_needs_the_explicit_mask(distilgpt2_bridge, prompt) -> None: + """padding_side can only describe padding at one edge. A masked-out token in + the middle shifts later positions just the same, and only a mask says so.""" + pad_id = distilgpt2_bridge.tokenizer.eos_token_id + gapped = prompt.clone() + gapped[0, 2] = pad_id + mask = torch.ones_like(prompt) + mask[0, 2] = 0 + compact = torch.cat([prompt[:, :2], prompt[:, 3:]], dim=1) + + reference = distilgpt2_bridge.generate(compact, **GREEDY)[0, compact.shape[1] :].tolist() + via_mask = distilgpt2_bridge.generate(gapped, attention_mask=mask, **GREEDY)[ + 0, prompt.shape[1] : + ].tolist() + + assert via_mask == reference + + +def test_rows_padded_to_different_lengths(distilgpt2_bridge) -> None: + """Each row must generate what it would alone, whatever its own pad count.""" + long_prompt = distilgpt2_bridge.to_tokens("The capital of France is") + short_prompt = distilgpt2_bridge.to_tokens("Hello") + width = max(long_prompt.shape[1], short_prompt.shape[1]) + + rows, masks = [], [] + for tokens in (long_prompt, short_prompt): + padded, mask = _left_pad(distilgpt2_bridge, tokens, width - tokens.shape[1]) + rows.append(padded) + masks.append(mask) + + out = distilgpt2_bridge.generate( + torch.cat(rows, dim=0), attention_mask=torch.cat(masks, dim=0), **GREEDY + ) + + for index, tokens in enumerate((long_prompt, short_prompt)): + solo = distilgpt2_bridge.generate(tokens, **GREEDY)[0, tokens.shape[1] :].tolist() + assert out[index, width:].tolist() == solo + + +def test_the_mask_reaches_every_step_not_just_the_first(distilgpt2_bridge, prompt) -> None: + """Before #1612 an attention_mask kwarg was absorbed into **multimodal_kwargs, + which are merged into the forward kwargs on step 0 only. That made the first + token come out right and every later one wrong, which is worse to debug than a + uniform failure. Each step must see a mask covering the prompt plus the tokens + generated so far. + """ + n_pad = 3 + padded, mask = _left_pad(distilgpt2_bridge, prompt, n_pad) + prompt_width = padded.shape[1] + seen: list[torch.Tensor | None] = [] + + original = distilgpt2_bridge.original_model.forward + + def _spy(*args, **kwargs): + seen.append(kwargs.get("attention_mask")) + return original(*args, **kwargs) + + distilgpt2_bridge.original_model.forward = _spy + try: + distilgpt2_bridge.generate(padded, attention_mask=mask, **GREEDY) + finally: + distilgpt2_bridge.original_model.forward = original + + assert len(seen) == GREEDY["max_new_tokens"] + for step, observed in enumerate(seen): + assert observed is not None, f"step {step} received no attention_mask" + assert observed.shape[1] == prompt_width + step + # The prompt's padding stays masked however far generation has run. + assert observed[0, :n_pad].sum() == 0 + assert observed[0, n_pad:].all() + + +def test_unpadded_generation_is_unchanged(distilgpt2_bridge, prompt, unpadded_continuation) -> None: + """An all-ones mask is what the model assumes anyway, so supplying one must + be a no-op rather than a second code path.""" + out = distilgpt2_bridge.generate(prompt, attention_mask=torch.ones_like(prompt), **GREEDY) + + assert out[0, prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_string_and_list_input_still_work(distilgpt2_bridge) -> None: + """The list path builds its own mask; neither route may regress.""" + if distilgpt2_bridge.tokenizer.pad_token_id is None: + distilgpt2_bridge.tokenizer.pad_token = distilgpt2_bridge.tokenizer.eos_token + + solo = distilgpt2_bridge.generate("The capital of France is", **GREEDY) + batched = distilgpt2_bridge.generate(["The capital of France is", "Hi"], **GREEDY) + + assert isinstance(solo, str) and solo.startswith("The capital of France is") + assert batched[0] == solo + + +def test_mask_shape_must_match_the_prompt(distilgpt2_bridge, prompt) -> None: + """generate() extends the mask itself, so a pre-extended one is a mistake + worth naming rather than broadcasting into something unintended.""" + with pytest.raises(ValueError, match="does not match the prompt shape"): + distilgpt2_bridge.generate( + prompt, attention_mask=torch.ones(1, prompt.shape[1] + 5, dtype=torch.long), **GREEDY + ) + + +def test_padding_side_without_a_tokenizer_is_an_error(distilgpt2_bridge, prompt) -> None: + """A bridge booted without a tokenizer has nothing to read the padding from, so + padding_side would be inert — leaving exactly the bug this module is about, but + silently. attention_mask still works there and the message must say so.""" + bridge = copy.copy(distilgpt2_bridge) + bridge.tokenizer = None + assert bridge.tokenizer is None and distilgpt2_bridge.tokenizer is not None + padded, _ = _left_pad(distilgpt2_bridge, prompt, 3) + + with pytest.raises(ValueError, match="this bridge has none"): + bridge.generate(padded, padding_side="left", **GREEDY) + + +def test_padding_side_without_a_pad_token_is_an_error(distilgpt2_bridge, prompt) -> None: + """Same reasoning for a tokenizer that has no pad id to scan for.""" + padded, _ = _left_pad(distilgpt2_bridge, prompt, 3) + tokenizer = distilgpt2_bridge.tokenizer + original_pad = tokenizer.pad_token_id + tokenizer.pad_token_id = None + try: + with pytest.raises(ValueError, match="pad_token_id"): + distilgpt2_bridge.generate(padded, padding_side="left", **GREEDY) + finally: + tokenizer.pad_token_id = original_pad + + +def test_a_tokenizerless_bridge_still_accepts_an_explicit_mask( + distilgpt2_bridge, prompt, unpadded_continuation +) -> None: + """The alternative the error points at has to actually work.""" + bridge = copy.copy(distilgpt2_bridge) + bridge.tokenizer = None + n_pad = 3 + padded, mask = _left_pad(distilgpt2_bridge, prompt, n_pad) + + out = bridge.generate(padded, attention_mask=mask, **GREEDY) + + assert out[0, n_pad + prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_inputs_embeds_forwards_the_mask_untouched(distilgpt2_bridge, prompt) -> None: + """There are no token positions to correct on the embeds path, but processors + emit an attention_mask alongside their other outputs and callers pass the lot + straight through. Before this parameter existed that mask reached the model via + **multimodal_kwargs, so it must still arrive rather than raise. + """ + embeds = distilgpt2_bridge.original_model.get_input_embeddings()(prompt) + mask = torch.ones_like(prompt) + seen: list[torch.Tensor | None] = [] + + original = distilgpt2_bridge.original_model.forward + + def _spy(*args, **kwargs): + seen.append(kwargs.get("attention_mask")) + return original(*args, **kwargs) + + distilgpt2_bridge.original_model.forward = _spy + try: + distilgpt2_bridge.generate(embeds, attention_mask=mask, **GREEDY) + finally: + distilgpt2_bridge.original_model.forward = original + + assert seen and seen[0] is not None + torch.testing.assert_close(seen[0], mask) diff --git a/tests/integration/model_bridge/test_glm4v_adapter.py b/tests/integration/model_bridge/test_glm4v_adapter.py index 176ebc9804..86b393da3f 100644 --- a/tests/integration/model_bridge/test_glm4v_adapter.py +++ b/tests/integration/model_bridge/test_glm4v_adapter.py @@ -114,6 +114,29 @@ def grab(tensor, hook): class TestGlm4vGeneration: + def test_generate_after_multimodal_forward(self, glm4v_bridge, snapshot_path): + """A multimodal forward caches mRoPE rope_deltas on the HF module; text-only + generate must not add that stale delta to its cached-step positions.""" + from PIL import Image + from transformers import AutoProcessor + + proc = AutoProcessor.from_pretrained(snapshot_path) + img = Image.new("RGB", (112, 112), "red") + messages = [ + { + "role": "user", + "content": [{"type": "image"}, {"type": "text", "text": "Describe"}], + } + ] + text = proc.apply_chat_template(messages, add_generation_prompt=True) + inputs = dict(proc(text=[text], images=[img], return_tensors="pt")) + bridge_inputs = {k: v for k, v in inputs.items() if k != "input_ids"} + with torch.no_grad(): + glm4v_bridge(inputs["input_ids"], **bridge_inputs) + + text_out = glm4v_bridge.generate("Hello", max_new_tokens=5, do_sample=False, verbose=False) + assert isinstance(text_out, str) + def test_generate(self, glm4v_bridge): text = glm4v_bridge.generate("Hello", max_new_tokens=5, do_sample=False, verbose=False) assert isinstance(text, str) diff --git a/tests/integration/model_bridge/test_left_padding_positions.py b/tests/integration/model_bridge/test_left_padding_positions.py new file mode 100644 index 0000000000..46d9ab8ff5 --- /dev/null +++ b/tests/integration/model_bridge/test_left_padding_positions.py @@ -0,0 +1,367 @@ +"""Regression tests for left-padding position handling in TransformerBridge. + +A causal LM's logits at a sequence's real token positions must not depend on how +that sequence is padded, provided the caller supplies the matching attention_mask. +Left padding shifts every real token's absolute position, so position_ids have to +be derived from the mask; without that the bridge silently returns wrong logits +and a wrong loss. See #1609. + +Right padding is included as a control: causality already protects it, so it was +never affected and must stay that way. +""" + +from __future__ import annotations + +import pytest +import torch + +from transformer_lens import utilities as utils + +PAD_ID = 0 + + +def _pad(tokens: torch.Tensor, n_pad: int, side: str) -> tuple[torch.Tensor, torch.Tensor]: + """Pad `tokens` on `side`, returning (padded_tokens, attention_mask).""" + pads = torch.full((tokens.shape[0], n_pad), PAD_ID, dtype=tokens.dtype) + ones = torch.ones(tokens.shape, dtype=torch.long) + zeros = torch.zeros((tokens.shape[0], n_pad), dtype=torch.long) + if side == "left": + return torch.cat([pads, tokens], dim=1), torch.cat([zeros, ones], dim=1) + return torch.cat([tokens, pads], dim=1), torch.cat([ones, zeros], dim=1) + + +def _mixed_batch( + tokens: torch.Tensor, n_pad: int +) -> tuple[torch.Tensor, torch.Tensor, tuple, tuple]: + """A batch of one right-padded, one left-padded and one unpadded row.""" + width = tokens.shape[1] + n_pad + right, m_right = _pad(tokens, n_pad, "right") + left, m_left = _pad(tokens, n_pad, "left") + plain = torch.arange(20, 20 + width, dtype=tokens.dtype).unsqueeze(0) + m_plain = torch.ones(1, width, dtype=torch.long) + batch = torch.cat([right, left, plain], dim=0) + mask = torch.cat([m_right, m_left, m_plain], dim=0) + return batch, mask, (right, m_right), (plain, m_plain) + + +def _spy_on_position_ids(bridge, tokens_in: torch.Tensor, mask: torch.Tensor): + """Run a forward, returning the position_ids the wrapped model actually saw.""" + seen: dict[str, object] = {} + original = bridge.original_model.forward + + def _spy(*args, **kwargs): + seen["position_ids"] = kwargs.get("position_ids") + return original(*args, **kwargs) + + bridge.original_model.forward = _spy + try: + with torch.no_grad(): + bridge(tokens_in, attention_mask=mask, return_type="logits") + finally: + bridge.original_model.forward = original + return seen["position_ids"] + + +@pytest.fixture(scope="module") +def tokens(distilgpt2_bridge) -> torch.Tensor: + return distilgpt2_bridge.to_tokens("The capital of France is") + + +@pytest.mark.parametrize("side", ["left", "right"]) +@pytest.mark.parametrize("n_pad", [1, 3, 5]) +def test_logits_are_invariant_to_padding(distilgpt2_bridge, tokens, side, n_pad) -> None: + """Padding must not change the logits at a sequence's real positions.""" + padded, mask = _pad(tokens, n_pad, side) + real = slice(n_pad, None) if side == "left" else slice(None, tokens.shape[1]) + + with torch.no_grad(): + baseline = distilgpt2_bridge(tokens, return_type="logits") + actual = distilgpt2_bridge(padded, attention_mask=mask, return_type="logits")[:, real] + + assert torch.isfinite(actual).all() + torch.testing.assert_close(actual, baseline, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("side", ["left", "right"]) +def test_compat_mode_logits_are_invariant_to_padding(distilgpt2_bridge_compat, side: str) -> None: + """enable_compatibility_mode() promises HookedTransformer-equivalent numerics, + which this property is part of.""" + tokens = distilgpt2_bridge_compat.to_tokens("The capital of France is") + n_pad = 3 + padded, mask = _pad(tokens, n_pad, side) + real = slice(n_pad, None) if side == "left" else slice(None, tokens.shape[1]) + + with torch.no_grad(): + baseline = distilgpt2_bridge_compat(tokens, return_type="logits") + actual = distilgpt2_bridge_compat(padded, attention_mask=mask, return_type="logits")[ + :, real + ] + + torch.testing.assert_close(actual, baseline, rtol=1e-3, atol=1e-3) + + +def test_derived_position_ids_match_hooked_transformer(distilgpt2_bridge, tokens) -> None: + """The derived positions must be the ones HookedTransformer would use, i.e. the + shared get_offset_position_ids helper rather than a parallel derivation.""" + n_pad = 3 + padded, mask = _pad(tokens, n_pad, "left") + expected = utils.get_offset_position_ids(0, mask) + + with torch.no_grad(): + derived = distilgpt2_bridge(padded, attention_mask=mask, return_type="logits") + supplied = distilgpt2_bridge( + padded, attention_mask=mask, position_ids=expected, return_type="logits" + ) + + torch.testing.assert_close(derived, supplied, rtol=1e-5, atol=1e-5) + + +def test_explicit_position_ids_take_precedence(distilgpt2_bridge, tokens) -> None: + """A caller-supplied position_ids must not be overwritten by the derivation.""" + n_pad = 3 + padded, mask = _pad(tokens, n_pad, "left") + derived_positions = utils.get_offset_position_ids(0, mask) + shifted = derived_positions + 1 # deliberately different, but still in range + + with torch.no_grad(): + default = distilgpt2_bridge(padded, attention_mask=mask, return_type="logits") + overridden = distilgpt2_bridge( + padded, attention_mask=mask, position_ids=shifted, return_type="logits" + ) + + assert not torch.allclose(default, overridden, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("gap", [slice(3, 5), slice(1, 2)]) +def test_interior_mask_gap_uses_derived_positions(distilgpt2_bridge, tokens, gap) -> None: + """A mask gap that is not leading padding still shifts later positions, so the + derivation must fire for any mask, not only ones starting with a pad.""" + mask = torch.ones(tokens.shape, dtype=torch.long) + mask[0, gap] = 0 + gapped = tokens.clone() + gapped[0, gap] = PAD_ID + expected = utils.get_offset_position_ids(0, mask) + + with torch.no_grad(): + derived = distilgpt2_bridge(gapped, attention_mask=mask, return_type="logits") + supplied = distilgpt2_bridge( + gapped, attention_mask=mask, position_ids=expected, return_type="logits" + ) + + torch.testing.assert_close(derived, supplied, rtol=1e-5, atol=1e-5) + + +@pytest.mark.parametrize("mask_kind", ["all_ones", "right_padded"]) +def test_no_position_ids_injected_when_unnecessary(distilgpt2_bridge, tokens, mask_kind) -> None: + """Masks whose attended tokens already sit at their default positions must not + get position_ids injected: it is a no-op at best, and models whose forward does + not accept position_ids would raise. + """ + if mask_kind == "all_ones": + passed, mask = tokens, torch.ones(tokens.shape, dtype=torch.long) + else: + passed, mask = _pad(tokens, 3, "right") + + assert _spy_on_position_ids(distilgpt2_bridge, passed, mask) is None + + +def test_unshifted_rows_keep_default_positions(distilgpt2_bridge, tokens) -> None: + """The decision is per row, not per batch: only rows whose mask moves an + attended token get derived positions, so the others stay on plain arange.""" + n_pad = 3 + batch, mask, _, _ = _mixed_batch(tokens, n_pad) + seen = _spy_on_position_ids(distilgpt2_bridge, batch, mask) + + assert seen is not None + arange = torch.arange(batch.shape[1]) + torch.testing.assert_close(seen[0], arange) # right-padded + torch.testing.assert_close(seen[2], arange) # unpadded + torch.testing.assert_close(seen[1], utils.get_offset_position_ids(0, mask)[1]) # left-padded + + +def test_one_left_padded_row_does_not_perturb_its_neighbours(distilgpt2_bridge, tokens) -> None: + """Derived positions for one row must not change its unshifted neighbours.""" + n_pad = 3 + batch, mask, (right, m_right), (plain, m_plain) = _mixed_batch(tokens, n_pad) + control_batch = torch.cat([right, right, plain], dim=0) + control_mask = torch.cat([m_right, m_right, m_plain], dim=0) + + with torch.no_grad(): + mixed = distilgpt2_bridge(batch, attention_mask=mask, return_type="logits") + control = distilgpt2_bridge( + control_batch, attention_mask=control_mask, return_type="logits" + ) + unpadded = distilgpt2_bridge(tokens, return_type="logits") + + # Matching batch shapes isolate derived-position handling from BLAS kernel + # changes caused by comparing batched and single-row matrix multiplications. + torch.testing.assert_close(mixed[0:1], control[0:1], rtol=0, atol=1e-6) + torch.testing.assert_close(mixed[2:3], control[2:3], rtol=0, atol=1e-6) + # ...while the row that did need correcting still gets it. + torch.testing.assert_close(mixed[1:2, n_pad:], unpadded, rtol=1e-3, atol=1e-3) + + +def test_float_attention_mask_is_accepted(distilgpt2_bridge, tokens) -> None: + """Derived positions index an embedding table, so a float 0/1 mask must not + produce float position_ids.""" + n_pad = 3 + padded, mask = _pad(tokens, n_pad, "left") + + with torch.no_grad(): + baseline = distilgpt2_bridge(tokens, return_type="logits") + actual = distilgpt2_bridge(padded, attention_mask=mask.float(), return_type="logits") + + torch.testing.assert_close(actual[:, n_pad:], baseline, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize( + "mask_row", + [ + pytest.param([0, 0, 0, 0, 0, 0], id="all_masked"), + pytest.param([0, 0, 0, 0, 0, 1], id="single_real_token"), + pytest.param([1, 0, 1, 0, 1, 1], id="two_interior_gaps"), + pytest.param([0, 1, 1, 1, 1, 0], id="padded_both_ends"), + ], +) +def test_degenerate_masks_do_not_crash(distilgpt2_bridge, tokens, mask_row) -> None: + """Shapes the happy path never reaches. An all-masked row in particular must + not inject anything: every position is a pad, so nothing is displaced.""" + mask = torch.tensor([mask_row[: tokens.shape[1]]], dtype=torch.long) + positions = _spy_on_position_ids(distilgpt2_bridge, tokens, mask) + + if mask.sum() == 0: + assert positions is None + else: + expected = utils.get_offset_position_ids(0, mask) + torch.testing.assert_close(positions, expected) + + +def test_four_dimensional_mask_is_left_alone(distilgpt2_bridge, tokens) -> None: + """A 4-D mask is an additive attention bias, not a 0/1 padding mask, so the + cumsum derivation is meaningless on it.""" + seq = tokens.shape[1] + mask = torch.ones(1, 1, seq, seq) + assert _spy_on_position_ids(distilgpt2_bridge, tokens, mask) is None + + +def test_inputs_embeds_are_left_alone(distilgpt2_bridge, tokens) -> None: + """Float input is pre-computed embeddings; there are no token positions to + derive and the batch/seq layout is not guaranteed to match the mask.""" + embeds = distilgpt2_bridge.original_model.get_input_embeddings()(tokens) + _, mask = _pad(tokens[:, :-2], 2, "left") + assert _spy_on_position_ids(distilgpt2_bridge, embeds, mask) is None + + +def test_gradients_flow_through_a_left_padded_forward(distilgpt2_bridge, tokens) -> None: + """The derivation must not detach the graph or poison the loss with pads.""" + n_pad = 3 + padded, mask = _pad(tokens, n_pad, "left") + + loss = distilgpt2_bridge(padded, attention_mask=mask, return_type="loss") + loss.backward() + grad = distilgpt2_bridge.original_model.get_input_embeddings().weight.grad + try: + assert torch.isfinite(loss) + assert grad is not None and torch.isfinite(grad).all() and (grad != 0).any() + finally: + distilgpt2_bridge.zero_grad(set_to_none=True) + + +def test_run_with_cache_matches_forward_under_left_padding(distilgpt2_bridge, tokens) -> None: + """run_with_cache routes through a different kwarg-filtering path, so the + injection has to survive it identically.""" + n_pad = 3 + padded, mask = _pad(tokens, n_pad, "left") + + with torch.no_grad(): + direct = distilgpt2_bridge(padded, attention_mask=mask, return_type="logits") + cached, activations = distilgpt2_bridge.run_with_cache(padded, attention_mask=mask) + + torch.testing.assert_close(direct, cached, rtol=0, atol=1e-6) + assert len(activations) > 0 + + +@pytest.fixture(scope="module") +def opt_bridge(): + """OPT is the one supported architecture whose positional embedding consumes + the attention mask, so it must be left to derive positions for itself.""" + from transformer_lens.model_bridge import TransformerBridge + + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-OPTForCausalLM", device="cpu", dtype=torch.float32 + ) + bridge.eval() + return bridge + + +def test_self_deriving_model_is_left_alone(opt_bridge) -> None: + """OPTLearnedPositionalEmbedding derives from the mask already, and uses its + own convention (-1) for padded slots. Overriding it buys no correctness and + silently changes the padded slots, so the bridge must stay out of the way. + """ + ids = torch.arange(20, 26).unsqueeze(0) + n_pad = 3 + padded, mask = _pad(ids, n_pad, "left") + + assert opt_bridge._accepts_derived_position_ids() is False + assert _spy_on_position_ids(opt_bridge, padded, mask) is None + + with torch.no_grad(): + bridge_out = opt_bridge(padded, attention_mask=mask, return_type="logits") + hf_out = opt_bridge.original_model(input_ids=padded, attention_mask=mask).logits + unpadded = opt_bridge(ids, attention_mask=torch.ones_like(ids), return_type="logits") + + # Deferring to OPT keeps the bridge exactly on HF, and OPT's own derivation + # already delivers the padding-invariance this module is about. + torch.testing.assert_close(bridge_out, hf_out, rtol=0, atol=1e-6) + torch.testing.assert_close(bridge_out[:, n_pad:], unpadded, rtol=1e-4, atol=1e-4) + + +def test_cached_step_with_left_padding(distilgpt2_bridge, tokens) -> None: + """With a KV cache the mask spans past+new while input_ids is only the new + token, so the derivation must be offset back to the tokens being passed. + + Prefill goes through the bridge (return_type="logits_and_cache") so the + cached keys and values are built under the same position convention the step + uses. Stepping off a cache prefilled by raw HF mixes two conventions and is + not equivalent to anything. + """ + n_pad = 3 + padded, mask = _pad(tokens, n_pad, "left") + new_token = torch.tensor([[318]]) + extended = torch.cat([mask, torch.ones(1, 1, dtype=torch.long)], dim=1) + + with torch.no_grad(): + _, cache = distilgpt2_bridge(padded, attention_mask=mask, return_type="logits_and_cache") + step = distilgpt2_bridge( + new_token, attention_mask=extended, past_key_values=cache, return_type="logits" + ) + # Ground truth: the same prompt and token with no padding at all. + unpadded = distilgpt2_bridge(torch.cat([tokens, new_token], dim=1), return_type="logits") + + assert step.shape[:2] == (1, 1) + torch.testing.assert_close(step, unpadded[:, -1:], rtol=1e-3, atol=1e-3) + + +def test_cached_decoding_with_left_padding_matches_full_recompute( + distilgpt2_bridge, tokens +) -> None: + """Several cached steps in a row: each must land on what recomputing the + whole left-padded sequence would give, or the offset drifts with the cache.""" + n_pad = 3 + padded, mask = _pad(tokens, n_pad, "left") + + with torch.no_grad(): + _, cache = distilgpt2_bridge(padded, attention_mask=mask, return_type="logits_and_cache") + sequence, grown = padded, mask + for _ in range(4): + logits = distilgpt2_bridge(sequence, attention_mask=grown, return_type="logits") + next_token = logits[:, -1].argmax(dim=-1, keepdim=True) + sequence = torch.cat([sequence, next_token], dim=1) + grown = torch.cat([grown, torch.ones(1, 1, dtype=torch.long)], dim=1) + + stepped = distilgpt2_bridge( + next_token, attention_mask=grown, past_key_values=cache, return_type="logits" + ) + full = distilgpt2_bridge(sequence, attention_mask=grown, return_type="logits") + torch.testing.assert_close(stepped, full[:, -1:], rtol=1e-3, atol=1e-3) diff --git a/tests/integration/model_bridge/test_llada_adapter.py b/tests/integration/model_bridge/test_llada_adapter.py index 4368615b8b..2078560d96 100644 --- a/tests/integration/model_bridge/test_llada_adapter.py +++ b/tests/integration/model_bridge/test_llada_adapter.py @@ -3,6 +3,7 @@ from __future__ import annotations import copy +import functools import gc import math import weakref @@ -649,6 +650,68 @@ def test_padding_mask_blocks_keys_without_becoming_causal(models: TinyModels) -> ) +def test_left_padding_does_not_inject_unsupported_position_ids(models: TinyModels) -> None: + """The bridge derives position_ids from attention_mask for left-padded input + (#1609), but this forward takes neither position_ids nor **kwargs — as the + released LLaDA remote code does not — so the kwarg would raise TypeError + where the model used to return logits. + """ + tokens = torch.tensor([[63, 63, 5, 7, 9]]) + attention_mask = torch.tensor([[0, 0, 1, 1, 1]]) + with torch.inference_mode(): + reference_logits = models.reference(tokens, attention_mask=attention_mask).logits + bridge_logits = models.bridge(tokens, attention_mask=attention_mask) + torch.testing.assert_close(bridge_logits, reference_logits, rtol=1e-5, atol=1e-6) + + +def test_batched_list_input_does_not_inject_unsupported_position_ids() -> None: + """Batched list input builds its own attention_mask and position_ids so pad + tokens don't contaminate the forward (#1626). The mask is safe for any model; + the position_ids are not, and this forward takes neither them nor **kwargs. + + A local bridge rather than the module fixture: this needs a tokenizer, and + attaching one to the shared instance would leak into the other tests. The + tokenizer is given a BOS so the path under test is reached independently of + BOS handling elsewhere. + """ + local = _build_models() + tokenizer = _offline_tokenizer() + tokenizer.bos_token = "" + local.bridge.tokenizer = tokenizer + + seen: dict = {} + original = local.bridge.original_model.forward + + # functools.wraps so inspect.signature() still resolves to the real forward: + # the gate reads that signature, and a bare (*args, **kwargs) spy would look + # like it accepts position_ids and defeat the check under test. + @functools.wraps(original) + def _spy(*args, **kwargs): + seen.clear() + seen.update(kwargs) + return original(*args, **kwargs) + + local.bridge.original_model.forward = _spy + try: + with torch.inference_mode(): + logits = local.bridge(["token_5 token_7 token_9", "token_5"], return_type="logits") + batched = dict(seen) + with torch.inference_mode(): + local.bridge("token_5 token_7 token_9", return_type="logits") + unbatched = dict(seen) + finally: + local.bridge.original_model.forward = original + + assert logits.shape[0] == 2 + assert "position_ids" not in batched + # The mask is still supplied — withholding it would reintroduce the padding + # contamination this branch exists to prevent. + assert "attention_mask" in batched + # Control: a single unbatched string never reached this branch, so the gate + # must not have changed anything for it either. + assert "position_ids" not in unbatched + + def test_run_with_cache_exposes_hooks_without_hf_output_attentions( models: TinyModels, ) -> None: diff --git a/tests/integration/model_bridge/test_olmo2_attention_input_state.py b/tests/integration/model_bridge/test_olmo2_attention_input_state.py new file mode 100644 index 0000000000..5a89a2f78c --- /dev/null +++ b/tests/integration/model_bridge/test_olmo2_attention_input_state.py @@ -0,0 +1,73 @@ +"""OLMo 2 attention-input fork regression tests using a tiny local HF model.""" + +import pytest +import torch +from transformers import AutoModelForCausalLM +from transformers.models.olmo2 import Olmo2Config + +from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_config_from_hf, +) +from transformer_lens.model_bridge.supported_architectures.olmo2 import ( + Olmo2ArchitectureAdapter, +) + + +class _Tokenizer: + pass + + +def _tiny_olmo2_bridge() -> TransformerBridge: + torch.manual_seed(0) + config = Olmo2Config( + vocab_size=64, + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + max_position_embeddings=32, + ) + config.architectures = ["Olmo2ForCausalLM"] + hf_model = AutoModelForCausalLM.from_config(config).to(torch.float32).eval() + bridge_config = build_bridge_config_from_hf( + hf_model.config, "Olmo2ForCausalLM", "olmo2-tiny", torch.float32 + ) + return TransformerBridge( + hf_model, Olmo2ArchitectureAdapter(bridge_config), tokenizer=_Tokenizer() + ) + + +@pytest.mark.parametrize( + ("setter_name", "input_hook_names"), + [ + ( + "set_use_split_qkv_input", + ( + "blocks.1.attn.hook_q_input", + "blocks.1.attn.hook_k_input", + "blocks.1.attn.hook_v_input", + ), + ), + ("set_use_attn_in", ("blocks.1.attn.hook_attn_in",)), + ], +) +def test_attention_input_fork_does_not_leak_state_between_forwards( + setter_name: str, input_hook_names: tuple[str, ...] +) -> None: + bridge = _tiny_olmo2_bridge() + bridge.enable_compatibility_mode() + getattr(bridge, setter_name)(True) + tokens = torch.tensor([[1, 2, 3, 4, 5]]) + + with torch.no_grad(): + first_logits, first_cache = bridge.run_with_cache(tokens) + second_logits, second_cache = bridge.run_with_cache(tokens) + + torch.testing.assert_close(second_logits, first_logits, rtol=0, atol=0) + for hook_name in input_hook_names: + torch.testing.assert_close(second_cache[hook_name], first_cache[hook_name], rtol=0, atol=0) + for cache in (first_cache, second_cache): + expected = cache["blocks.1.hook_resid_pre"].unsqueeze(2).expand_as(cache[hook_name]) + torch.testing.assert_close(cache[hook_name], expected, rtol=0, atol=0) diff --git a/tests/integration/model_bridge/test_parent_module_traversal.py b/tests/integration/model_bridge/test_parent_module_traversal.py new file mode 100644 index 0000000000..5f60c79305 --- /dev/null +++ b/tests/integration/model_bridge/test_parent_module_traversal.py @@ -0,0 +1,397 @@ +"""Download-free parent traversal coverage across Bridge architecture shapes.""" + +from dataclasses import dataclass +from typing import Any, Callable + +import pytest +import torch +from torch import nn +from transformers import ( + ASTConfig, + ASTForAudioClassification, + BartConfig, + BartForConditionalGeneration, + BertConfig, + BertForMaskedLM, + BertForNextSentencePrediction, + BertForPreTraining, + BloomConfig, + BloomForCausalLM, + GPT2Config, + GPT2LMHeadModel, + GPTNeoXConfig, + GPTNeoXForCausalLM, + HubertConfig, + HubertForCTC, + LlamaConfig, + LlamaForCausalLM, + MistralConfig, + MistralForCausalLM, + MixtralConfig, + MixtralForCausalLM, + T5Config, + T5ForConditionalGeneration, + ViTConfig, + ViTForImageClassification, + ViTModel, +) + +from transformer_lens.model_bridge.sources import build_bridge_from_module + + +@dataclass(frozen=True) +class ArchitectureCase: + name: str + model_type: type[nn.Module] + config_factory: Callable[[], Any] + architecture: str + + +def _bert_config() -> BertConfig: + return BertConfig( + vocab_size=32, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=32, + max_position_embeddings=16, + ) + + +def _vit_config() -> ViTConfig: + return ViTConfig( + image_size=16, + patch_size=4, + num_channels=3, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=32, + num_labels=3, + ) + + +def _bart_config() -> BartConfig: + return BartConfig( + vocab_size=32, + d_model=16, + encoder_layers=1, + decoder_layers=1, + encoder_attention_heads=4, + decoder_attention_heads=4, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=16, + ) + + +def _hubert_config() -> HubertConfig: + return HubertConfig( + vocab_size=32, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=32, + conv_dim=(8,), + conv_stride=(2,), + conv_kernel=(3,), + num_conv_pos_embeddings=4, + num_conv_pos_embedding_groups=2, + ) + + +ARCHITECTURE_CASES = ( + ArchitectureCase( + "gpt2-joint-qkv", + GPT2LMHeadModel, + lambda: GPT2Config( + vocab_size=32, + n_positions=16, + n_ctx=16, + n_embd=16, + n_layer=1, + n_head=4, + n_inner=32, + ), + "GPT2LMHeadModel", + ), + ArchitectureCase( + "bloom-joint-qkv", + BloomForCausalLM, + lambda: BloomConfig(vocab_size=32, hidden_size=16, n_layer=1, n_head=4), + "BloomForCausalLM", + ), + ArchitectureCase( + "gpt-neox-rotary", + GPTNeoXForCausalLM, + lambda: GPTNeoXConfig( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + max_position_embeddings=16, + ), + "GPTNeoXForCausalLM", + ), + ArchitectureCase( + "llama-split-qkv-rope", + LlamaForCausalLM, + lambda: LlamaConfig( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=4, + max_position_embeddings=16, + ), + "LlamaForCausalLM", + ), + ArchitectureCase( + "mistral-gqa", + MistralForCausalLM, + lambda: MistralConfig( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=2, + max_position_embeddings=16, + ), + "MistralForCausalLM", + ), + ArchitectureCase("bert-mlm", BertForMaskedLM, _bert_config, "BertForMaskedLM"), + ArchitectureCase( + "bert-nsp", + BertForNextSentencePrediction, + _bert_config, + "BertForMaskedLM", + ), + ArchitectureCase( + "bert-mlm-nsp", + BertForPreTraining, + _bert_config, + "BertForMaskedLM", + ), + ArchitectureCase( + "t5-encoder-decoder", + T5ForConditionalGeneration, + lambda: T5Config( + vocab_size=32, + d_model=16, + d_kv=4, + d_ff=32, + num_layers=1, + num_decoder_layers=1, + num_heads=4, + ), + "T5ForConditionalGeneration", + ), + ArchitectureCase( + "bart-encoder-decoder", + BartForConditionalGeneration, + _bart_config, + "BartForConditionalGeneration", + ), + ArchitectureCase( + "mixtral-moe", + MixtralForCausalLM, + lambda: MixtralConfig( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=2, + num_local_experts=2, + num_experts_per_tok=1, + max_position_embeddings=16, + ), + "MixtralForCausalLM", + ), + ArchitectureCase( + "vit-vision", + ViTForImageClassification, + _vit_config, + "ViTForImageClassification", + ), + ArchitectureCase("vit-bare-pooler", ViTModel, _vit_config, "ViTModel"), + ArchitectureCase( + "hubert-audio", + HubertForCTC, + _hubert_config, + "HubertForCTC", + ), + ArchitectureCase( + "ast-audio-classifier", + ASTForAudioClassification, + lambda: ASTConfig( + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=32, + patch_size=4, + frequency_stride=4, + time_stride=4, + max_length=16, + num_mel_bins=16, + ), + "ASTForAudioClassification", + ), +) + +ARCHITECTURE_CASE_BY_NAME = {case.name: case for case in ARCHITECTURE_CASES} + + +def _named_identities(named_values: Any) -> dict[int, str]: + return {id(value): name for name, value in named_values} + + +def _assert_same_identities(expected: dict[int, str], actual: dict[int, str]) -> None: + missing = [expected[identity] for identity in expected.keys() - actual.keys()] + unexpected = [actual[identity] for identity in actual.keys() - expected.keys()] + assert actual.keys() == expected.keys(), f"missing={missing}, unexpected={unexpected}" + + +@pytest.mark.parametrize("case", ARCHITECTURE_CASES, ids=lambda case: case.name) +def test_parent_and_direct_traversal_have_identical_state(case: ArchitectureCase) -> None: + config = case.config_factory() + model = case.model_type(config).eval() + bridge = build_bridge_from_module( + model, + case.architecture, + hf_config=config, + dtype=torch.float32, + device="cpu", + model_name=f"tiny-{case.name}", + ) + parent = nn.Module() + parent.add_module("bridge", bridge) + + source_parameters = _named_identities(bridge.original_model.named_parameters()) + direct_parameters = _named_identities(bridge.named_parameters()) + parent_parameters = _named_identities(parent.named_parameters()) + source_buffers = _named_identities(bridge.original_model.named_buffers()) + direct_buffers = _named_identities(bridge.named_buffers()) + parent_buffers = _named_identities(parent.named_buffers()) + + _assert_same_identities(source_parameters, direct_parameters) + _assert_same_identities(direct_parameters, parent_parameters) + _assert_same_identities(source_buffers, direct_buffers) + _assert_same_identities(direct_buffers, parent_buffers) + + +def test_parent_dtype_conversion_updates_container_owned_state() -> None: + bart_config = _bart_config() + bridge = build_bridge_from_module( + BartForConditionalGeneration(bart_config), + "BartForConditionalGeneration", + hf_config=bart_config, + dtype=torch.float32, + device="cpu", + model_name="tiny-bart-container-buffer", + ) + parent = nn.Module() + parent.add_module("bridge", bridge) + + parent.to(torch.float64) + + assert bridge.original_model.final_logits_bias.dtype == torch.float64 + assert id(bridge.original_model.final_logits_bias) in { + id(buffer) for buffer in parent.buffers() + } + + +def test_parent_assign_load_updates_container_owned_state() -> None: + bart_config = _bart_config() + bridge = build_bridge_from_module( + BartForConditionalGeneration(bart_config), + "BartForConditionalGeneration", + hf_config=bart_config, + dtype=torch.float32, + device="cpu", + model_name="tiny-bart-container-buffer-load", + ) + parent = nn.Module() + parent.add_module("bridge", bridge) + state = parent.state_dict() + buffer_key = "bridge._container_state_owners.final_logits_bias" + state[buffer_key] = torch.ones_like(state[buffer_key]) + + parent.load_state_dict(state, strict=True, assign=True) + + assert torch.equal(bridge.original_model.final_logits_bias, torch.ones_like(state[buffer_key])) + assert id(bridge.original_model.final_logits_bias) in { + id(buffer) for buffer in parent.buffers() + } + + +@pytest.mark.parametrize( + ("case_name", "container_path", "state_name", "state_key"), + ( + ("bart-encoder-decoder", "", "final_logits_bias", "final_logits_bias"), + ("hubert-audio", "hubert", "masked_spec_embed", "hubert.masked_spec_embed"), + ), +) +def test_direct_assign_load_stays_current_after_apply( + case_name: str, container_path: str, state_name: str, state_key: str +) -> None: + case = ARCHITECTURE_CASE_BY_NAME[case_name] + config = case.config_factory() + bridge = build_bridge_from_module( + case.model_type(config), + case.architecture, + hf_config=config, + dtype=torch.float32, + device="cpu", + model_name=f"tiny-{case.name}-direct-assign", + ) + original_container = ( + bridge.original_model.get_submodule(container_path) + if container_path + else bridge.original_model + ) + owner_container = ( + bridge._container_state_owners.get_submodule(container_path) + if container_path + else bridge._container_state_owners + ) + replacement = torch.full_like(getattr(original_container, state_name), 7) + + bridge.load_state_dict({state_key: replacement}, strict=False, assign=True) + + assert getattr(owner_container, state_name) is getattr(original_container, state_name) + bridge.cpu() + assert torch.equal(getattr(original_container, state_name), replacement) + + +@pytest.mark.parametrize( + ("case_name", "key_fragment", "expected_keys"), + ( + ("bert-nsp", "pooler", {"pooler.weight", "pooler.bias"}), + ("vit-bare-pooler", "pooler", {"pooler.weight", "pooler.bias"}), + ( + "ast-audio-classifier", + "classifier", + {"classifier_ln.weight", "classifier_ln.bias"}, + ), + ), +) +def test_task_head_state_dict_keys_are_not_reexpanded( + case_name: str, key_fragment: str, expected_keys: set[str] +) -> None: + case = ARCHITECTURE_CASE_BY_NAME[case_name] + config = case.config_factory() + bridge = build_bridge_from_module( + case.model_type(config), + case.architecture, + hf_config=config, + dtype=torch.float32, + device="cpu", + model_name=f"tiny-{case.name}-state-dict-keys", + ) + + actual_keys = {key for key in bridge.state_dict() if key_fragment in key} + assert actual_keys == expected_keys diff --git a/tests/integration/model_bridge/test_seq2seq_benchmark_loss.py b/tests/integration/model_bridge/test_seq2seq_benchmark_loss.py new file mode 100644 index 0000000000..46dd763cd2 --- /dev/null +++ b/tests/integration/model_bridge/test_seq2seq_benchmark_loss.py @@ -0,0 +1,35 @@ +"""Benchmark loss calls must supply labels and a resolvable ablation hook on seq2seq. + +The bridge refuses label-less return_type="loss" for encoder-decoder models +(encoder input_ids are not decoder targets). forward_pass.py was updated when +that guard landed; hook_registration and weight_processing kept the bare call, +so P2 hook_functionality errored on all seven seq2seq architectures. The +ablation hook also targeted blocks.0.* which does not exist on encoder-decoder +bridges, silently no-opping the whole check. +""" + +import pytest + +from transformer_lens.benchmarks.hook_registration import benchmark_hook_functionality +from transformer_lens.benchmarks.utils import BenchmarkSeverity, bridge_self_target_loss +from transformer_lens.model_bridge import TransformerBridge + +TEXT = "translate English to German: Hello world" + + +@pytest.fixture(scope="module") +def t5(): + return TransformerBridge.boot_transformers("google-t5/t5-small", device="cpu") + + +def test_self_target_loss_is_finite_on_seq2seq(t5) -> None: + loss = bridge_self_target_loss(t5, TEXT) + assert loss.ndim == 0 and loss.isfinite() + + +def test_hook_functionality_runs_and_the_ablation_bites(t5) -> None: + result = benchmark_hook_functionality(t5, TEXT) + assert result.passed, result.message + assert result.severity != BenchmarkSeverity.ERROR, result.message + # A vacuous run (unresolvable hook) reports "minimal effect: 0.000000". + assert "minimal effect" not in result.message, result.message diff --git a/tests/integration/model_bridge/test_seq2seq_loss.py b/tests/integration/model_bridge/test_seq2seq_loss.py new file mode 100644 index 0000000000..51d109d9cd --- /dev/null +++ b/tests/integration/model_bridge/test_seq2seq_loss.py @@ -0,0 +1,281 @@ +"""Regression tests for TransformerBridge explicit-label loss semantics.""" + +from __future__ import annotations + +import pytest +import torch +import torch.nn.functional as F +from transformers import ( + BartConfig, + BartForConditionalGeneration, + GPT2Config, + GPT2LMHeadModel, + T5Config, + T5ForConditionalGeneration, +) + +from tests.integration.model_bridge.helpers import make_tiny_pair +from transformer_lens.model_bridge import TransformerBridge + + +@pytest.fixture(scope="module") +def tiny_gpt2_pair() -> tuple[TransformerBridge, torch.nn.Module]: + config = GPT2Config( + vocab_size=32, + n_embd=16, + n_layer=1, + n_head=2, + n_positions=16, + n_ctx=16, + bos_token_id=1, + eos_token_id=2, + pad_token_id=0, + ) + return make_tiny_pair(config, "GPT2LMHeadModel", loader=GPT2LMHeadModel) + + +@pytest.fixture(scope="module", params=("bart", "t5")) +def tiny_seq2seq_pair(request) -> tuple[TransformerBridge, torch.nn.Module]: + if request.param == "bart": + config = BartConfig( + vocab_size=32, + d_model=16, + encoder_layers=1, + decoder_layers=1, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=32, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + decoder_start_token_id=2, + ) + return make_tiny_pair( + config, + "BartForConditionalGeneration", + loader=BartForConditionalGeneration, + ) + + config = T5Config( + vocab_size=32, + d_model=16, + d_kv=8, + d_ff=32, + num_layers=1, + num_decoder_layers=1, + num_heads=2, + pad_token_id=0, + eos_token_id=1, + decoder_start_token_id=0, + ) + return make_tiny_pair( + config, + "T5ForConditionalGeneration", + loader=T5ForConditionalGeneration, + ) + + +def test_seq2seq_loss_and_logits_follow_labels( + tiny_seq2seq_pair: tuple[TransformerBridge, torch.nn.Module], +) -> None: + bridge, reference = tiny_seq2seq_pair + source = torch.tensor([[4, 5, 6, 7, 2]]) + label_batches = ( + torch.tensor([[8, 9, 10, 2, -100]]), + torch.tensor([[11, 12, 13, 2, -100]]), + ) + + bridge_losses = [] + bridge_logits = [] + for labels in label_batches: + with torch.no_grad(): + loss = bridge(source, labels=labels, return_type="loss") + logits = bridge(source, labels=labels, return_type="logits") + reference_output = reference(input_ids=source, labels=labels) + + torch.testing.assert_close(loss, reference_output.loss) + torch.testing.assert_close(logits, reference_output.logits) + bridge_losses.append(loss) + bridge_logits.append(logits) + + assert not torch.allclose(bridge_losses[0], bridge_losses[1]) + assert not torch.allclose(bridge_logits[0], bridge_logits[1]) + + +def test_seq2seq_both_allows_shorter_target( + tiny_seq2seq_pair: tuple[TransformerBridge, torch.nn.Module], +) -> None: + bridge, reference = tiny_seq2seq_pair + source = torch.tensor([[4, 5, 6, 7, 2]]) + labels = torch.tensor([[14, 15, 2]]) + + with torch.no_grad(): + logits, loss = bridge(source, labels=labels, return_type="both") + reference_output = reference(input_ids=source, labels=labels) + + assert logits.shape[:2] == labels.shape + torch.testing.assert_close(logits, reference_output.logits) + torch.testing.assert_close(loss, reference_output.loss) + + +def test_seq2seq_loss_supports_hf_tuple_output( + tiny_seq2seq_pair: tuple[TransformerBridge, torch.nn.Module], +) -> None: + bridge, reference = tiny_seq2seq_pair + source = torch.tensor([[4, 5, 6, 7, 2]]) + labels = torch.tensor([[14, 15, 2]]) + + with torch.no_grad(): + logits, loss = bridge( + source, + labels=labels, + return_type="both", + return_dict=False, + ) + reference_loss, reference_logits, *_ = reference( + input_ids=source, + labels=labels, + return_dict=False, + ) + + torch.testing.assert_close(logits, reference_logits) + torch.testing.assert_close(loss, reference_loss) + + +@pytest.mark.parametrize("return_type", ("loss", "both")) +def test_seq2seq_per_token_loss_matches_unshifted_labels( + tiny_seq2seq_pair: tuple[TransformerBridge, torch.nn.Module], + return_type: str, +) -> None: + bridge, reference = tiny_seq2seq_pair + source = torch.tensor([[4, 5, 6, 7, 2]]) + labels = torch.tensor([[14, 15, 2, -100]]) + + with torch.no_grad(): + output = bridge( + source, + labels=labels, + return_type=return_type, + loss_per_token=True, + ) + reference_logits = reference(input_ids=source, labels=labels).logits + + loss = output[1] if isinstance(output, tuple) else output + expected = F.cross_entropy( + reference_logits.flatten(0, 1), + labels.flatten(), + reduction="none", + ignore_index=-100, + ).view_as(labels) + + assert loss.shape == labels.shape + assert loss[0, -1] == 0 + torch.testing.assert_close(loss, expected) + + +@pytest.mark.parametrize("return_type", ("loss", "both")) +def test_seq2seq_loss_requires_labels( + tiny_seq2seq_pair: tuple[TransformerBridge, torch.nn.Module], + return_type: str, +) -> None: + bridge, _ = tiny_seq2seq_pair + source = torch.tensor([[4, 5, 6, 7, 2]]) + + with pytest.raises(ValueError, match="labels are required"): + bridge(source, return_type=return_type) + + +def test_causal_loss_uses_explicit_labels( + tiny_gpt2_pair: tuple[TransformerBridge, torch.nn.Module], +) -> None: + bridge, reference = tiny_gpt2_pair + input_ids = torch.tensor([[4, 5, 6, 7, 2]]) + labels = torch.full_like(input_ids, 9) + + with torch.no_grad(): + default_loss = bridge(input_ids, return_type="loss") + logits, labeled_loss = bridge(input_ids, labels=labels, return_type="both") + reference_output = reference(input_ids=input_ids, labels=labels) + + assert not torch.allclose(labeled_loss, default_loss) + torch.testing.assert_close(logits, reference_output.logits) + torch.testing.assert_close(labeled_loss, reference_output.loss) + + +def test_causal_labels_support_hf_tuple_output( + tiny_gpt2_pair: tuple[TransformerBridge, torch.nn.Module], +) -> None: + bridge, reference = tiny_gpt2_pair + input_ids = torch.tensor([[4, 5, 6, 7, 2]]) + labels = torch.full_like(input_ids, 9) + + with torch.no_grad(): + logits, loss = bridge( + input_ids, + labels=labels, + return_type="both", + return_dict=False, + ) + reference_loss, reference_logits, *_ = reference( + input_ids=input_ids, + labels=labels, + return_dict=False, + ) + + torch.testing.assert_close(logits, reference_logits) + torch.testing.assert_close(loss, reference_loss) + + +def test_causal_per_token_loss_uses_labels_and_ignore_index( + tiny_gpt2_pair: tuple[TransformerBridge, torch.nn.Module], +) -> None: + bridge, reference = tiny_gpt2_pair + input_ids = torch.tensor([[4, 5, 6, 7, 2]]) + labels = torch.tensor([[9, 8, 7, -100, -100]]) + + with torch.no_grad(): + loss = bridge( + input_ids, + labels=labels, + return_type="loss", + loss_per_token=True, + ) + reference_logits = reference(input_ids=input_ids).logits + + expected = F.cross_entropy( + reference_logits[:, :-1].flatten(0, 1), + labels[:, 1:].flatten(), + reduction="none", + ignore_index=-100, + ).view_as(labels[:, 1:]) + + assert loss.shape == labels[:, 1:].shape + assert torch.count_nonzero(loss[:, 2:]) == 0 + torch.testing.assert_close(loss, expected) + + +def test_causal_explicit_labels_preserve_attention_mask_contract( + tiny_gpt2_pair: tuple[TransformerBridge, torch.nn.Module], +) -> None: + bridge, _ = tiny_gpt2_pair + input_ids = torch.tensor([[4, 5, 6, 0, 0]]) + labels = torch.tensor([[9, 8, 7, 6, 5]]) + attention_mask = torch.tensor([[1, 1, 1, 0, 0]]) + + with torch.no_grad(): + logits = bridge(input_ids, attention_mask=attention_mask, return_type="logits") + loss = bridge( + input_ids, + labels=labels, + attention_mask=attention_mask, + return_type="loss", + ) + + transition_mask = attention_mask[:, :-1].bool() & attention_mask[:, 1:].bool() + expected = F.cross_entropy( + logits[:, :-1][transition_mask], + labels[:, 1:][transition_mask], + ) + torch.testing.assert_close(loss, expected) diff --git a/tests/integration/test_jacobian_lens.py b/tests/integration/test_jacobian_lens.py index ba263c85c4..661942dee6 100644 --- a/tests/integration/test_jacobian_lens.py +++ b/tests/integration/test_jacobian_lens.py @@ -464,6 +464,74 @@ def test_decompose_gpt2_activation_reconstructs_and_is_orthogonal(published_gpt2 assert cosine.abs().item() < 1e-3 +def test_occupancy_gpt2_activation_is_a_small_positive_integer(published_gpt2_lens, gpt2_bridge): + """occupancy on a real GPT-2 activation returns a positive integer within ``[1, max_atoms]``, + with per-step real and control captured-variance curves of the right shape. We assert shape and + bounds -- not the paper's closed-model count (an open-weight observation, recorded on failure). + A reduced control count keeps CI fast.""" + layer, max_atoms = 6, 12 + result = published_gpt2_lens.occupancy( + gpt2_bridge, + PROMPT, + layer=layer, + position=-1, + max_atoms=max_atoms, + num_control_dictionaries=8, + ) + + assert isinstance(result.occupancy, int) + assert 1 <= result.occupancy <= max_atoms, f"occupancy={result.occupancy} max_atoms={max_atoms}" + assert result.marginal_captured_variance.shape == (max_atoms,) + assert result.control_captured_variance.shape == (max_atoms,) + assert result.support.shape == (max_atoms,) + assert (result.support >= 0).all() and (result.support < gpt2_bridge.cfg.d_vocab).all() + + # Cumulative captured variance is a projection ratio: non-decreasing and bounded to [0, 1]. + real_cumulative = result.marginal_captured_variance.cumsum(0) + assert (result.marginal_captured_variance >= -1e-4).all() + assert (real_cumulative >= -1e-4).all() and (real_cumulative <= 1.0 + 1e-4).all() + + # Deterministic given the seed: an identical call reproduces the count and both curves exactly. + repeat = published_gpt2_lens.occupancy( + gpt2_bridge, + PROMPT, + layer=layer, + position=-1, + max_atoms=max_atoms, + num_control_dictionaries=8, + ) + assert repeat.occupancy == result.occupancy + assert torch.equal(repeat.marginal_captured_variance, result.marginal_captured_variance) + assert torch.equal(repeat.control_captured_variance, result.control_captured_variance) + + +def test_fraction_of_variance_gpt2_is_a_small_ratio(published_gpt2_lens, gpt2_bridge): + """fraction_of_variance over a small corpus: each layer's median and pooled ratio land in + ``[0, 1]`` with samples recorded, consistent with the paper's "J-space is a small fraction of + total variance" (an open-weight observation, not a numeric match to the closed-model figures). + FIT_PROMPTS are long enough to clear the default 16-position skip.""" + layers = [3, 6] + profile = published_gpt2_lens.fraction_of_variance(gpt2_bridge, FIT_PROMPTS, layers=layers, k=8) + + assert profile.layers == layers + for layer in layers: + per_position = profile.per_position[layer] + assert per_position.numel() > 0 + # Each fraction is ||projection||^2 / ||activation||^2, a variance ratio in [0, 1]. + assert (per_position >= 0.0).all() and (per_position <= 1.0).all() + assert 0.0 <= profile.median[layer] <= 1.0 + assert 0.0 <= profile.pooled[layer] <= 1.0 + + # positions= overrides the skip_first sweep: one explicit position per prompt over the + # two-prompt corpus exercises the override path and cross-prompt pooling (one sample each). + pinned = published_gpt2_lens.fraction_of_variance( + gpt2_bridge, FIT_PROMPTS, layers=layers, k=8, positions=[-1] + ) + for layer in layers: + assert pinned.per_position[layer].numel() == len(FIT_PROMPTS) + assert 0.0 <= pinned.pooled[layer] <= 1.0 + + @pytest.mark.slow def test_decompose_gemma_activation_is_valid(): """Decompose a real gemma-2-2b-it activation via its published lens (slow: real download).""" diff --git a/tests/unit/benchmarks/test_forward_pass_loss.py b/tests/unit/benchmarks/test_forward_pass_loss.py new file mode 100644 index 0000000000..6b011321a9 --- /dev/null +++ b/tests/unit/benchmarks/test_forward_pass_loss.py @@ -0,0 +1,36 @@ +"""Loss benchmarks must follow the explicit-label Bridge contract.""" + +from typing import Any + +import pytest +import torch + +from transformer_lens.benchmarks.forward_pass import benchmark_loss_equivalence +from transformer_lens.model_bridge import TransformerBridge + + +def test_benchmark_loss_equivalence_supplies_tokenized_self_labels( + monkeypatch: pytest.MonkeyPatch, +) -> None: + bridge = object.__new__(TransformerBridge) + torch.nn.Module.__init__(bridge) + labels = torch.tensor([[1, 2, 3]]) + forward_kwargs: dict[str, Any] = {} + + def to_tokens(text: str, **kwargs: Any) -> torch.Tensor: + assert text == "benchmark text" + return labels + + def forward(input: str, **kwargs: Any) -> torch.Tensor: + assert input == "benchmark text" + forward_kwargs.update(kwargs) + return torch.tensor(1.25) + + monkeypatch.setattr(bridge, "to_tokens", to_tokens) + monkeypatch.setattr(bridge, "forward", forward) + + result = benchmark_loss_equivalence(bridge, "benchmark text", reference_loss=1.25) + + assert result.passed + assert forward_kwargs["labels"] is labels + assert forward_kwargs["return_type"] == "loss" diff --git a/tests/unit/benchmarks/test_text_quality_image_conditioned.py b/tests/unit/benchmarks/test_text_quality_image_conditioned.py index b6b0be4522..25938ed8b6 100644 --- a/tests/unit/benchmarks/test_text_quality_image_conditioned.py +++ b/tests/unit/benchmarks/test_text_quality_image_conditioned.py @@ -1,11 +1,6 @@ -"""Image-conditioned seq2seq (Florence-2) must score P4 from real captions. - -Florence-2 needs pixel_values to generate: given a text-only prompt its decoder -emits a 1-token EOS, so every continuation is "too short" and P4 scored 0 (a -misleading failure for a working model). The fix drives real image-conditioned -caption generation ( on synthetic test images) and scores that -grammatical output instead — a genuine quality signal, not a skip. -""" +"""Caption test images must be real, distinct RGB inputs — averaging caption +scores over identical images would be a fake sample size. (The end-to-end +Florence-2 caption test lives in tests/integration/benchmarks/.)""" import pytest @@ -21,29 +16,3 @@ def test_build_caption_test_images_are_distinct_rgb(): assert all(im.mode == "RGB" and im.size == (224, 224) for im in images) # Distinct backgrounds -> distinct pixel data (averaging over samples is real). assert len({im.tobytes() for im in images}) == 3 - - -def test_florence2_text_quality_scores_image_captions(): - from transformer_lens.benchmarks.text_quality import benchmark_text_quality - from transformer_lens.model_bridge import TransformerBridge - - try: - bridge = TransformerBridge.boot_transformers( - "florence-community/Florence-2-base-ft", device="cpu" - ) - except (OSError, ConnectionError, TimeoutError) as exc: - pytest.skip(f"florence-2 unavailable offline: {exc}") - - # Preconditions: this is the image-conditioned seq2seq path. - assert bridge.original_model.config.is_encoder_decoder - assert getattr(bridge.cfg, "is_multimodal", False) - - result = benchmark_text_quality( - bridge, "The theory of relativity explains that", max_new_tokens=50, device="cpu" - ) - # Pre-fix: "Scoring failed for all prompts" (score absent -> registry P4=0). - assert result.details is not None, result.message - assert "score" in result.details, result.message - assert result.details["score"] > 0 - # Scored the model's actual captions, not the 4 text-only prompts. - assert result.details["num_prompts"] >= 1 diff --git a/tests/unit/benchmarks/test_text_quality_scoring.py b/tests/unit/benchmarks/test_text_quality_scoring.py new file mode 100644 index 0000000000..21c368a6fa --- /dev/null +++ b/tests/unit/benchmarks/test_text_quality_scoring.py @@ -0,0 +1,712 @@ +"""Reference-ratio Phase-4 scoring: the score must be a judge-handicap-free +comparison against a reference completion, with penalties for loops and +truncation, generated via token-level slicing (string-prefix slicing breaks +under chat templates because generate() strips special tokens on decode).""" + +import math +from types import SimpleNamespace + +import pytest +import torch + +pytest.importorskip("transformers") + +from transformer_lens.benchmarks.text_quality import ( + JUDGE_R_FAIL, + _length_penalty, + _ratio_to_score, + benchmark_text_quality, +) +from transformer_lens.benchmarks.utils import BenchmarkResult, BenchmarkSeverity + + +class FakeVocabTokenizer: + """Whitespace tokenizer with a growable vocab and template-marker specials.""" + + chat_template = None + + def __init__(self): + self._vocab: list[str] = [] + self._special: set[int] = set() + self.mask_token = None + + def _id(self, word: str, special: bool = False) -> int: + if word not in self._vocab: + self._vocab.append(word) + idx = self._vocab.index(word) + if special: + self._special.add(idx) + return idx + + def encode_words(self, text: str, special: bool = False) -> list[int]: + return [self._id(w, special) for w in text.split()] + + def __call__(self, text, return_tensors=None): + # Native recipe used for encoder-decoder inputs. + ids = self.encode_words(text) + if return_tensors == "pt": + return {"input_ids": torch.tensor([ids])} + return {"input_ids": ids} + + def decode(self, ids, skip_special_tokens=True): + ids = ids.tolist() if hasattr(ids, "tolist") else list(ids) + words = [self._vocab[i] for i in ids if not (skip_special_tokens and i in self._special)] + return " ".join(words) + + def apply_chat_template(self, messages, add_generation_prompt=True, tokenize=False): + return f"<|im_start|> {messages[0]['content']} <|im_end|>" + + +class FakeBridge: + """Decoder-only bridge stub: generate() echoes prompt ids + canned continuation.""" + + def __init__( + self, continuation="the quick brown fox jumps over the lazy dog today", chat_template=None + ): + self.tokenizer = FakeVocabTokenizer() + self.tokenizer.chat_template = chat_template + self.adapter = SimpleNamespace(supports_generation=True, native_sampler=None) + self.original_model = SimpleNamespace( + config=SimpleNamespace(is_encoder_decoder=False, architectures=["FakeLM"]) + ) + self.cfg = SimpleNamespace(device="cpu", is_multimodal=False, model_name="fake") + self._continuation = continuation + self.generate_calls: list[dict] = [] + self.to_tokens_calls: list = [] + + def to_tokens(self, text, prepend_bos=None, **kwargs): + self.to_tokens_calls.append(prepend_bos) + special = text.startswith("<|im_start|>") + if special: + # Template markers become special ids that decode drops. + ids = [] + for word in text.split(): + is_marker = word.startswith("<|") + ids.append(self.tokenizer._id(word, special=is_marker)) + return torch.tensor([ids]) + return torch.tensor([self.tokenizer.encode_words(text)]) + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + cont_ids = self.tokenizer.encode_words(self._continuation) + return torch.cat([input, torch.tensor([cont_ids])], dim=1) + + +class FakeJudgeTokenizer: + """Word-level judge tokenizer sharing nothing with the bridge's.""" + + def __init__(self): + self._vocab: list[str] = [] + + def __call__(self, text, return_tensors=None): + ids = [] + for w in text.split(): + if w not in self._vocab: + self._vocab.append(w) + ids.append(self._vocab.index(w)) + if return_tensors == "pt": + return {"input_ids": torch.tensor([ids])} + return {"input_ids": ids} + + +class FakeJudge: + """Judge whose loss is a configurable function of the scored token ids. + + Records (masked_context_words, scored_words) per call so tests can assert + what the judge was conditioned on.""" + + def __init__(self, tokenizer: FakeJudgeTokenizer, loss_fn): + self._tokenizer = tokenizer + self._loss_fn = loss_fn + self.calls: list = [] + + def __call__(self, input_ids, labels=None): + pairs = list(zip(input_ids[0].tolist(), labels[0].tolist())) + scored = [int(t) for t, l in pairs if l != -100] + masked = [int(t) for t, l in pairs if l == -100] + words = " ".join(self._tokenizer._vocab[i] for i in scored) + self.calls.append((" ".join(self._tokenizer._vocab[i] for i in masked), words)) + return SimpleNamespace(loss=torch.tensor(self._loss_fn(words))) + + +def _run(bridge, profile="continuation", loss_fn=lambda text: 1.0, **kwargs): + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, loss_fn) + result = benchmark_text_quality( + bridge, profile, judge_model=judge, judge_tokenizer=judge_tokenizer, **kwargs + ) + bridge.judge_calls = judge.calls + return result + + +class TestRatioMath: + def test_ratio_one_scores_100(self): + assert _ratio_to_score(1.0) == 100.0 + + def test_ratio_r_fail_scores_zero(self): + assert _ratio_to_score(JUDGE_R_FAIL) == pytest.approx(0.0, abs=1e-9) + + def test_ratio_sqrt_r_fail_scores_50(self): + """Registry's phase-4 floor of 50 = geometric midpoint of good and broken.""" + assert _ratio_to_score(math.sqrt(JUDGE_R_FAIL)) == pytest.approx(50.0, abs=1e-9) + + def test_ratio_below_one_clamps_to_100(self): + """Beating the reference is not extra credit (loops get there trivially).""" + assert _ratio_to_score(0.2) == 100.0 + + def test_ratio_is_handicap_invariant(self): + """A judge that is k-times worse at some language multiplies BOTH sides' + perplexity, so the score must not move (the old absolute-perplexity + mapping drops by 10 ln k).""" + gen_ppl, ref_ppl, k = 40.0, 25.0, 10.0 + assert _ratio_to_score(gen_ppl / ref_ppl) == pytest.approx( + _ratio_to_score((k * gen_ppl) / (k * ref_ppl)) + ) + + +class TestLengthPenalty: + def test_neutral_at_reference_length(self): + assert _length_penalty(40, 40) == 1.0 + + def test_neutral_band_half_to_triple(self): + """Neutral in [0.5x, 3x] of reference: terse-but-complete answers are + not punished, and the old 25% floor (which never fired in four + sweeps — a contentless chat stub scored 93.6) is gone.""" + assert _length_penalty(20, 40) == 1.0 + assert _length_penalty(120, 40) == 1.0 + + def test_penalizes_below_half(self): + assert _length_penalty(10, 40) == pytest.approx(0.5) + assert _length_penalty(13, 41) == pytest.approx(13 / 20.5) + + def test_penalizes_overlength(self): + """Rambling output the repetition penalty misses: 6x the reference + pays half.""" + assert _length_penalty(240, 40) == pytest.approx(0.5) + + def test_zero_reference_is_neutral(self): + assert _length_penalty(3, 0) == 1.0 + + +class TestBenchmarkPipeline: + def test_fluent_output_scores_high(self): + # Continuation long enough to sit in the length-penalty neutral band + # for every reference; ratio 1 then clamps every prompt to 100. + fluent = "the quick brown fox jumps over the lazy dog today while the sun sets slowly behind the old hills" + result = _run(FakeBridge(continuation=fluent), loss_fn=lambda t: 1.0) + assert result.details is not None + assert result.details["score"] == 100.0 + + def test_looping_output_scores_low_despite_low_ratio(self): + """A degenerate loop has LOW judge perplexity; only the repetition + penalty catches it under ratio scoring.""" + loop = "the cat sat the cat sat the cat sat the cat sat" + result = _run(FakeBridge(continuation=loop), loss_fn=lambda t: 0.1) + assert result.details is not None + assert result.details["score"] < 50.0 + + def test_one_token_output_scores_zero(self): + """Florence-style bare-EOS output: one token is not scoreable text.""" + result = _run(FakeBridge(continuation="x"), loss_fn=lambda t: 1.0) + assert result.details is not None + assert result.details["score"] == 0.0 + + def test_generated_segment_sliced_by_token_count(self): + """Chat-template prompts are not string prefixes of decoded output + (specials are stripped); prompt words must still be excluded from the + judged text.""" + seen: list[str] = [] + + def record(text): + seen.append(text) + return 1.0 + + bridge = FakeBridge(chat_template="{{messages}}") + result = _run(bridge, profile="chat", loss_fn=record) + assert result.details is not None + gen_texts = seen[0::2] # generated, reference alternate + assert all("<|im_start|>" not in t for t in seen) + for text in gen_texts: + assert text == bridge._continuation + + def test_context_mask_does_not_swallow_first_generated_token(self): + """Tokenizing prompt+text as one string lets the tokenizer merge + across the seam, so the context mask swallows the first generated + token; the pieces must be tokenized separately.""" + seen: list[str] = [] + + def record(text): + seen.append(text) + return 1.0 + + bridge = FakeBridge(continuation="zebra jumps over seven quiet green hills today") + _run(bridge, loss_fn=record) + gen_texts = [t for t in seen if "zebra" in t or "jumps" in t] + assert gen_texts, seen + assert all(t.startswith("zebra") for t in gen_texts), gen_texts + + def test_empty_generation_is_danger(self): + result = _run(FakeBridge(continuation=""), loss_fn=lambda t: 1.0) + assert result.severity == BenchmarkSeverity.DANGER + assert result.passed is False + + def test_uncovered_profile_skips_with_coverage_instruction(self): + """A coverage gap must tell the operator to file an issue, never score.""" + result = _run(FakeBridge(), profile="task:translation@en-sw") + assert result.severity == BenchmarkSeverity.SKIPPED + assert "task:translation@en-sw" in result.message + assert "file a TransformerLens issue" in result.message + + def test_chat_without_template_downgrades_to_continuation(self): + bridge = FakeBridge(chat_template=None) + result = _run(bridge, profile="chat") + assert result.details is not None + assert result.details["prompt_profile"] == "continuation" + assert "no chat template" in result.details["profile_adjustment"] + + def test_per_prompt_seed_independent_of_order(self): + """Each prompt's sample stream restarts at the benchmark seed, so a + prompt's output cannot depend on how much RNG earlier prompts consumed.""" + + class RngBridge(FakeBridge): + def generate(self, input, **kwargs): + # Fixed RNG consumption, then a draw: identical across prompts + # only if every prompt's stream restarts at the benchmark seed. + torch.rand(5) + draw = int(torch.randint(0, 10_000, (1,)).item()) + cont = self.tokenizer.encode_words(f"gen{draw} token one two") + return torch.cat([input, torch.tensor([cont])], dim=1) + + seen: list[str] = [] + + def record(text): + seen.append(text) + return 1.0 + + _run(RngBridge(), loss_fn=record) + gen_texts = [t for t in seen if t.startswith("gen")] + assert len(gen_texts) >= 2 + assert len(set(gen_texts)) == 1, gen_texts + + +class TestProfileDataIntegrity: + def test_every_table_entry_is_scoreable(self): + from transformer_lens.benchmarks import text_quality_profiles as p + from transformer_lens.benchmarks.text_quality import _wrong_language + + tables = [ + p.CONTINUATION_PROMPTS, + p.CHAT_PROMPTS, + p.SUMMARIZATION_PROMPTS, + p.INSTRUCTION_PROMPTS, + p.DENOISE_PROMPTS, + ] + for table in tables: + for lang, entries in table.items(): + for entry in entries: + assert entry.prompt.strip() + assert entry.reference.strip() + # Content, not just shape: a reference that self-flags as + # wrong-language hard-zeros its own sample (a fr reference + # did; the shape checks missed it). + if table in (p.CONTINUATION_PROMPTS, p.CHAT_PROMPTS): + assert not _wrong_language(entry.reference, lang), ( + lang, + entry.reference[:50], + ) + + def test_pivot_sentences_index_aligned(self): + from transformer_lens.benchmarks.text_quality_profiles import PIVOT_SENTENCES + + lengths = {lang: len(rows) for lang, rows in PIVOT_SENTENCES.items()} + assert set(lengths.values()) == {3}, lengths + + def test_all_kinds_have_knobs(self): + from transformer_lens.benchmarks import text_quality_profiles as p + + assert set(p.MAX_NEW_TOKENS_BY_KIND) == set(p.PROFILE_KINDS) + + +class TestTranslationWiring: + def test_forced_bos_threaded_for_multilingual_translators(self): + """M2M100/MBart select target language via the first decoder token; + dropping the forced_bos_token_id kwarg silently translates into an + arbitrary language (and the judge would score that fluent text well).""" + + class M2M100Bridge(FakeBridge): + def __init__(self): + super().__init__(continuation="ich muss jetzt wirklich schlafen gehen heute abend") + self.original_model.config.is_encoder_decoder = True + self.tokenizer.get_lang_id = lambda lang: {"de": 777, "en": 700}.get(lang, 0) + self.tokenizer.src_lang = "en" + + def generate(self, input, **kwargs): + # Real enc-dec output shape: [decoder_start] + generated, never + # the echoed source prompt. + self.generate_calls.append(kwargs) + start = torch.tensor([[0]]) + cont_ids = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont_ids])], dim=1) + + bridge = M2M100Bridge() + result = _run(bridge, profile="task:translation@en-de") + assert result.details is not None, result.message + assert all(call.get("forced_bos_token_id") == 777 for call in bridge.generate_calls) + assert bridge.tokenizer.src_lang == "en" + + +class TestReviewGuards: + """Guards for defects found in adversarial review of the rework.""" + + def test_cjk_repetition_penalty_uses_characters(self): + """Whitespace-split n-grams see zh/ja text as one word and never fire — + exactly where the judge rewards loops with low perplexity.""" + from transformer_lens.benchmarks.text_quality import _compute_repetition_penalty + + assert _compute_repetition_penalty("的" * 20) < 0.2 + assert _compute_repetition_penalty("のの" * 10) < 0.3 + fluent_zh = "长城是中国古代伟大的防御工程，每年吸引大量游客。" + assert _compute_repetition_penalty(fluent_zh) > 0.7 + + def test_wrong_language_output_scores_zero(self): + """Ratio scoring measures fluency, not language: fluent English beats a + short German reference and clamps to 100 unless language is checked.""" + bridge = FakeBridge(continuation="the quick brown fox jumps over the lazy dog and the cat") + seen = [] + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, lambda t: (seen.append(t) or 1.0)) + from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + result = benchmark_text_quality( + bridge, "continuation@de", judge_model=judge, judge_tokenizer=judge_tokenizer + ) + assert result.details is not None + assert result.details["score"] == 0.0 + assert "not in 'de'" in result.details["per_prompt"] + + def test_wrong_language_check_passes_correct_language(self): + bridge = FakeBridge( + continuation="der alte Zug ist nicht mit einem neuen Wagen gefahren und die Leute" + ) + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, lambda t: 1.0) + from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + result = benchmark_text_quality( + bridge, "continuation@de", judge_model=judge, judge_tokenizer=judge_tokenizer + ) + assert result.details is not None + assert result.details["score"] > 0.0 + + def test_empty_output_scored_zero_not_dropped(self): + """An empty generation must drag the average down, not vanish from it.""" + + class HalfEmptyBridge(FakeBridge): + def __init__(self): + super().__init__() + self._call = 0 + + def generate(self, input, **kwargs): + self._call += 1 + if self._call % 2 == 0: + return input # no new tokens -> empty continuation + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([input, torch.tensor([cont])], dim=1) + + result = _run(HalfEmptyBridge(), loss_fn=lambda t: 1.0) + assert result.details is not None + assert result.details["num_prompts"] == 4 + assert 40.0 <= result.details["score"] <= 60.0, result.details + + def test_denoise_t5_fill_spliced_into_sentence(self): + """Bare span fragments have judge PPL in the thousands, making the + ratio vacuous; the fill must be judged inside the restored sentence.""" + seen: list[str] = [] + + class DenoiseBridge(FakeBridge): + def __init__(self): + super().__init__(continuation="played happily") + self.original_model.config.is_encoder_decoder = True + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + start = torch.tensor([[0]]) + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont])], dim=1) + + bridge = DenoiseBridge() + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, lambda t: (seen.append(t) or 1.0)) + from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + result = benchmark_text_quality( + bridge, "task:denoise", judge_model=judge, judge_tokenizer=judge_tokenizer + ) + assert result.details is not None + # The bare fill must never reach the judge; every judged text is a + # full restored sentence. + assert seen and all(len(t.split()) >= 8 for t in seen), seen + assert "The children played happily in the park until the sun went down." in seen + + def test_chat_prepend_bos_false_threaded_to_tokenizer(self): + """The chat template supplies its own BOS; to_tokens must receive + prepend_bos=False or the prompt gets a double BOS.""" + bridge = FakeBridge(chat_template="{{messages}}") + _run(bridge, profile="chat") + assert bridge.to_tokens_calls and all(v is False for v in bridge.to_tokens_calls) + + def test_translation_scored_jointly(self): + """Short pivot sentences have unstable judge PPL; the three samples + must be concatenated into one judged pair.""" + + class MarianBridge(FakeBridge): + def __init__(self): + super().__init__(continuation="ik moet nu echt gaan slapen vandaag") + self.original_model.config.is_encoder_decoder = True + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + start = torch.tensor([[0]]) + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont])], dim=1) + + bridge = MarianBridge() + judge_tokenizer = FakeJudgeTokenizer() + judge = FakeJudge(judge_tokenizer, lambda t: 1.0) + from transformer_lens.benchmarks.text_quality import benchmark_text_quality + + result = benchmark_text_quality( + bridge, "task:translation@en-nl", judge_model=judge, judge_tokenizer=judge_tokenizer + ) + assert result.details is not None + assert result.details["num_prompts"] == 1 + assert len(bridge.generate_calls) == 3 # generation stays per-sentence + + def test_task_kinds_generate_greedily(self): + """Users run translators deterministically; sampling variance also + makes a single-sample score unstable. Task kinds must pass + temperature 0.0 while open-ended kinds keep sampling.""" + cont_bridge = FakeBridge() + _run(cont_bridge, profile="continuation") + assert all(c["temperature"] == 0.7 for c in cont_bridge.generate_calls) + + class MarianBridge(FakeBridge): + def __init__(self): + super().__init__(continuation="ik moet nu echt gaan slapen vandaag") + self.original_model.config.is_encoder_decoder = True + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + start = torch.tensor([[0]]) + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont])], dim=1) + + task_bridge = MarianBridge() + _run(task_bridge, profile="task:translation@en-nl") + assert all(c["temperature"] == 0.0 for c in task_bridge.generate_calls) + + def test_encdec_prompt_uses_native_tokenizer_recipe(self): + """Encoder input must follow the tokenizer's own recipe (lang token + + trailing ); to_tokens' BOS policy injects and drops , + which sent m2m100 into a quote-mark loop.""" + BOS, EOS = 901, 902 + + class RecipeTokenizer(FakeVocabTokenizer): + def __call__(self, text, return_tensors=None): + ids = self.encode_words(text) + [EOS] + if return_tensors == "pt": + return {"input_ids": torch.tensor([ids])} + return {"input_ids": ids} + + class RecipeBridge(FakeBridge): + def __init__(self): + super().__init__(continuation="ik moet gaan slapen vandaag echt nu") + self.tokenizer.__class__ = RecipeTokenizer + self.original_model.config.is_encoder_decoder = True + self.seen_inputs: list = [] + + def to_tokens(self, text, prepend_bos=None, **kwargs): + self.to_tokens_calls.append(prepend_bos) + return torch.tensor([[BOS] + self.tokenizer.encode_words(text)]) + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + self.seen_inputs.append(input[0].tolist()) + start = torch.tensor([[0]]) + cont = self.tokenizer.encode_words(self._continuation) + return torch.cat([start, torch.tensor([cont])], dim=1) + + bridge = RecipeBridge() + _run(bridge, profile="task:translation@en-nl") + assert bridge.seen_inputs, "no generation happened" + for ids in bridge.seen_inputs: + assert ids[-1] == EOS, ids + assert BOS not in ids, ids + + def test_dead_encdec_denoise_scores_zero(self): + """An empty span fill must not be spliced into the prompt sentence: + the splice hands a dead enc-dec model the near-reference sentence and + a free 100 (decoder-only dead models already scored 0).""" + + class DeadT5Bridge(FakeBridge): + def __init__(self): + super().__init__() + self.original_model.config.is_encoder_decoder = True + self.tokenizer.mask_token = None + + def generate(self, input, **kwargs): + self.generate_calls.append(kwargs) + return torch.tensor([[0]]) + + result = _run(DeadT5Bridge(), profile="task:denoise") + assert result.details["score"] == 0.0 + assert result.severity == BenchmarkSeverity.DANGER + + +class TestRound2ReviewGuards: + """Guards for the second review round's confirmed findings.""" + + def test_curated_strings_never_self_flag(self): + """The wrong-language detector must accept every curated string in its + own language — a reference that self-flags hard-zeros its sample (a + French reference did, via 'de/et' hitting other languages' sets).""" + from transformer_lens.benchmarks.text_quality import _wrong_language + from transformer_lens.benchmarks.text_quality_profiles import ( + CHAT_PROMPTS, + CONTINUATION_PROMPTS, + PIVOT_SENTENCES, + ) + + offenders = [] + for table in (CONTINUATION_PROMPTS, CHAT_PROMPTS): + for lang, prompts in table.items(): + for pp in prompts: + for text in (pp.prompt, pp.reference): + if _wrong_language(text, lang): + offenders.append((lang, text[:50])) + for lang, sents in PIVOT_SENTENCES.items(): + for s in sents: + if _wrong_language(s, lang): + offenders.append((lang, s[:50])) + assert offenders == [] + + def test_cjk_loop_penalized_despite_space(self): + """A single space in a degenerate CJK loop restored the inert word + path (penalty 1.0 vs 0.053); char mode must key off CJK content.""" + from transformer_lens.benchmarks.text_quality import _compute_repetition_penalty + + assert _compute_repetition_penalty("的的的的的的的的的 的的的的的的的的的的") <= 0.3 + assert _compute_repetition_penalty("我该去睡觉了，因为明天有一个很重要的会议要参加。") > 0.5 + + def test_registry_floor_equals_pass_line(self): + """[floor, pass) previously got passed=False with a clean note; both + numbers must come from the same constant.""" + from transformer_lens.benchmarks.text_quality_profiles import p4_pass_threshold + from transformer_lens.tools.model_registry.verify_models import ( + _MIN_PHASE_SCORES, + ) + + assert _MIN_PHASE_SCORES[4] == p4_pass_threshold() + + def test_judge_cannot_self_score(self): + """Ratio scoring against the judge's own perplexity is self-grading.""" + from transformer_lens.benchmarks.text_quality import JUDGE_MODEL_ID + + result = _run(FakeBridge(), profile="continuation", model_name=JUDGE_MODEL_ID) + assert result.severity == BenchmarkSeverity.SKIPPED + assert result.message.startswith("P4 skipped:") + + def test_chat_judged_with_prompt_context(self): + """A fluent off-topic stub scores ~98 when chat output is judged + standalone; conditioning on the user prompt is the relevance signal.""" + from transformer_lens.benchmarks.text_quality_profiles import ( + JUDGE_CONTEXT_KINDS, + ) + + assert "chat" in JUDGE_CONTEXT_KINDS + assert "task:instruction" in JUDGE_CONTEXT_KINDS + # Unconditioned summarization scored hallucinated summaries 100 (the + # judge never saw the article); unconditioned denoise rated broken + # restorations more fluent than the reference. + assert "task:summarization" in JUDGE_CONTEXT_KINDS + assert "task:denoise" in JUDGE_CONTEXT_KINDS + bridge = FakeBridge() + bridge.tokenizer.chat_template = "{{messages}}" + _run(bridge, profile="chat") + contexts = [c for c, _t in bridge.judge_calls] + assert any(c for c in contexts), "judge never saw the user prompt as context" + + def test_p1_only_note_labels_skip_as_coverage_gap(self): + """A skipped P4 is a coverage gap; the stale score must not be + relabeled 'text quality poor'.""" + from transformer_lens.tools.model_registry.verify_models import ( + _p1_only_core_note, + ) + + skipped = BenchmarkResult( + name="text_quality", + severity=BenchmarkSeverity.SKIPPED, + message="P4 skipped: no prompts for profile 'continuation@xx' — file an issue", + ) + skipped.phase = 4 + note = _p1_only_core_note(None, [skipped]) + assert "P4 skipped" in note and "poor" not in note + assert "poor (P4=40.0)" in _p1_only_core_note(40.0, []) + assert "errored" in _p1_only_core_note(None, []) + + def test_arch_rule_keeps_stored_language(self): + """The arch rule fixes the kind; a scraped @fr of the same kind must + survive resolve->writeback or curation can never stick.""" + from transformer_lens.benchmarks.text_quality_profiles import resolve_profile + + spec = resolve_profile( + "some/pegasus-clone", + "PegasusForConditionalGeneration", + registry_profile="task:summarization@fr", + ) + assert str(spec) == "task:summarization@fr" + spec = resolve_profile( + "some/pegasus-clone", + "PegasusForConditionalGeneration", + registry_profile="continuation@fr", + ) + assert str(spec) == "task:summarization" + + +class TestForcedBosVocabCollision: + """Bare ISO codes collide with ordinary subwords (T5's 'de' id 221, + Marian's 'en' id 39) and were injected as forced decoder tokens, + corrupting every translator without a real lang-code system.""" + + def test_plain_vocab_word_is_not_a_lang_code(self): + from transformer_lens.benchmarks.text_quality import _forced_bos_for_target + + class PlainSeq2SeqTokenizer: + unk_token_id = 3 + + def convert_tokens_to_ids(self, tok): + return {"de": 221, "en": 39}.get(tok, 3) + + assert _forced_bos_for_target(PlainSeq2SeqTokenizer(), "de") is None + + def test_nllb_style_code_still_resolves(self): + from transformer_lens.benchmarks.text_quality import _forced_bos_for_target + + class NllbLikeTokenizer: + unk_token_id = 3 + + def convert_tokens_to_ids(self, tok): + return {"deu_Latn": 256042}.get(tok, 3) + + assert _forced_bos_for_target(NllbLikeTokenizer(), "de") == 256042 + + +class TestAllGenerationsCaptured: + def test_details_carry_every_prompts_generation(self): + """Only the first prompt's output was stored; the registry-wide + review needs every generation inspectable.""" + bridge = FakeBridge() + result = _run(bridge, profile="continuation") + texts = result.details["generated_texts"] + assert len(texts) == result.details["num_prompts"] + assert all(isinstance(t, str) and t for t in texts) diff --git a/tests/unit/benchmarks/test_text_quality_seq2seq.py b/tests/unit/benchmarks/test_text_quality_seq2seq.py deleted file mode 100644 index 161fa0de46..0000000000 --- a/tests/unit/benchmarks/test_text_quality_seq2seq.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Encoder-decoder text-quality scoring must score the full decoder output. - -Seq2seq models (Marian/T5/BART) emit a standalone output, not a continuation -of the prompt. Scoring it as a continuation subtracts the prompt length and -trips the "continuation too short (< 2 tokens)" guard for every prompt when the -output is ~ the prompt length (Marian nl->en on an English prompt), scoring 0. -The fix scores the whole generated sequence for encoder-decoder models. -""" - -import pytest - -pytest.importorskip("transformers") - - -def test_marian_text_quality_scores_full_output(): - from transformer_lens.benchmarks.text_quality import benchmark_text_quality - from transformer_lens.model_bridge import TransformerBridge - - try: - bridge = TransformerBridge.boot_transformers("Helsinki-NLP/opus-mt-nl-en", device="cpu") - except (OSError, ConnectionError, TimeoutError) as exc: - pytest.skip(f"marian unavailable offline: {exc}") - - assert bridge.original_model.config.is_encoder_decoder # precondition - - result = benchmark_text_quality( - bridge, "Natural language processing is", max_new_tokens=20, device="cpu" - ) - # Pre-fix this returned "Scoring failed for all prompts" (score absent). - assert result.details is not None, result.message - assert "score" in result.details, result.message - assert result.details["score"] > 0 diff --git a/tests/unit/model_bridge/compatibility/test_use_attn_result.py b/tests/unit/model_bridge/compatibility/test_use_attn_result.py index 521a5a7bfc..86de8e0168 100644 --- a/tests/unit/model_bridge/compatibility/test_use_attn_result.py +++ b/tests/unit/model_bridge/compatibility/test_use_attn_result.py @@ -70,21 +70,19 @@ def _hook(tensor, hook): def test_hook_result_does_not_fire_when_flag_off(gpt2_bridge): - """When `use_attn_result=False` the per-head einsum path is skipped, so - `hook_result` must NOT fire (no activation captured).""" + """When `use_attn_result=False`, explicitly adding hook_result should fail + with a clear error explaining how to enable the gated hook.""" x = torch.arange(1, 9).unsqueeze(0) assert gpt2_bridge.cfg.use_attn_result is False - fired = {"result": False} def _hook(tensor, hook): - fired["result"] = True return tensor - gpt2_bridge.run_with_hooks(x, fwd_hooks=[("blocks.0.attn.hook_result", _hook)]) - assert fired["result"] is False, ( - "hook_result fired when use_attn_result was False; the flag is " - "supposed to skip the per-head computation." - ) + with pytest.raises(ValueError, match="set_use_attn_result"): + gpt2_bridge.run_with_hooks( + x, + fwd_hooks=[("blocks.0.attn.hook_result", _hook)], + ) def test_use_attn_result_applicability_raises_on_unsupported(monkeypatch, gpt2_bridge): diff --git a/tests/unit/model_bridge/generalized_components/test_base.py b/tests/unit/model_bridge/generalized_components/test_base.py index df3c9a1529..23754c5dd8 100644 --- a/tests/unit/model_bridge/generalized_components/test_base.py +++ b/tests/unit/model_bridge/generalized_components/test_base.py @@ -5,9 +5,15 @@ import torch.nn as nn from transformer_lens.hook_points import HookPoint +from transformer_lens.model_bridge.generalized_components.attention import ( + AttentionBridge, +) from transformer_lens.model_bridge.generalized_components.base import ( GeneralizedComponent, ) +from transformer_lens.model_bridge.generalized_components.unembedding import ( + UnembeddingBridge, +) class MockOriginalComponent(nn.Module): @@ -377,5 +383,80 @@ def test_complex_object_attributes(self): assert component.complex_attr["nested"]["deep"] == [1, 2, 3] +@pytest.mark.parametrize("use_keyword", [False, True]) +def test_forward_preserves_input_dtype(use_keyword: bool): + """Parameter storage dtype must not determine the component compute dtype.""" + + class MixedPrecisionComponent(nn.Module): + def __init__(self): + super().__init__() + self.auxiliary = nn.Parameter(torch.ones((), dtype=torch.float32)) + self.received_dtype = None + + def forward(self, hidden_states): + self.received_dtype = hidden_states.dtype + return hidden_states.clone() + + original = MixedPrecisionComponent() + component = MockGeneralizedComponent("mixed_precision") + component.set_original_component(original) + inputs = torch.ones(2, 3, dtype=torch.bfloat16) + + output = component(hidden_states=inputs) if use_keyword else component(inputs) + + assert original.received_dtype == torch.bfloat16 + assert output.dtype == torch.bfloat16 + torch.testing.assert_close(output, inputs) + + +@pytest.mark.parametrize("input_name", ["positional", "hidden_states", "query_input"]) +def test_attention_forward_preserves_input_dtype(input_name: str): + class MixedPrecisionAttention(nn.Module): + def __init__(self): + super().__init__() + self.auxiliary = nn.Parameter(torch.ones((), dtype=torch.float32)) + self.received_dtype = None + + def forward(self, hidden_states=None, query_input=None): + value = query_input if query_input is not None else hidden_states + self.received_dtype = value.dtype + return value + + original = MixedPrecisionAttention() + bridge = AttentionBridge(name="attention", config=None) + bridge.set_original_component(original) + inputs = torch.ones(2, 3, dtype=torch.bfloat16) + + if input_name == "positional": + output = bridge(inputs) + else: + output = bridge(**{input_name: inputs}) + + assert original.received_dtype == torch.bfloat16 + assert output.dtype == torch.bfloat16 + + +def test_unembedding_forward_preserves_input_dtype(): + class MixedPrecisionUnembedding(nn.Module): + def __init__(self): + super().__init__() + self.auxiliary = nn.Parameter(torch.ones((), dtype=torch.float32)) + self.received_dtype = None + + def forward(self, hidden_states): + self.received_dtype = hidden_states.dtype + return hidden_states + + original = MixedPrecisionUnembedding() + bridge = UnembeddingBridge(name="unembed") + bridge.set_original_component(original) + inputs = torch.ones(2, 3, dtype=torch.bfloat16) + + output = bridge(inputs) + + assert original.received_dtype == torch.bfloat16 + assert output.dtype == torch.bfloat16 + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/unit/model_bridge/generalized_components/test_moe_bridge_tuple_output.py b/tests/unit/model_bridge/generalized_components/test_moe_bridge_tuple_output.py index 3b644497bc..c321c9ec4a 100644 --- a/tests/unit/model_bridge/generalized_components/test_moe_bridge_tuple_output.py +++ b/tests/unit/model_bridge/generalized_components/test_moe_bridge_tuple_output.py @@ -39,6 +39,23 @@ def _bridge_with_stub(fake_forward) -> MoEBridge: class TestMoEBridgeTupleOutput: + @pytest.mark.parametrize("use_keyword", [False, True]) + def test_preserves_input_dtype(self, use_keyword: bool) -> None: + received_dtype = None + + def fake_forward(hidden_states): + nonlocal received_dtype + received_dtype = hidden_states.dtype + return hidden_states + + bridge = _bridge_with_stub(fake_forward) + hidden_states = torch.ones(1, 3, 4, dtype=torch.bfloat16) + + output = bridge(hidden_states=hidden_states) if use_keyword else bridge(hidden_states) + + assert received_dtype == torch.bfloat16 + assert output.dtype == torch.bfloat16 + def test_empty_tuple_raises_clear_type_error(self) -> None: bridge = _bridge_with_stub(lambda *a, **kw: ()) with pytest.raises(TypeError, match="torch.Tensor"): diff --git a/tests/unit/model_bridge/sources/__init__.py b/tests/unit/model_bridge/sources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/model_bridge/sources/test_native_init_semantics.py b/tests/unit/model_bridge/sources/test_native_init_semantics.py new file mode 100644 index 0000000000..69ca1581bf --- /dev/null +++ b/tests/unit/model_bridge/sources/test_native_init_semantics.py @@ -0,0 +1,76 @@ +"""Native init: seeded reproducibility across device/dtype, and initializer_range gain.""" + +import pytest +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.sources.native.init import initialize_native_model + + +def _cfg(**overrides): + base = dict( + n_layers=2, + d_model=64, + d_head=16, + n_heads=4, + d_mlp=128, + d_vocab=100, + n_ctx=16, + act_fn="gelu", + seed=0, + ) + base.update(overrides) + return TransformerBridgeConfig(**base) + + +def test_seeded_reinit_after_dtype_cast_reproduces_boot_weights(): + """Same seed must give the same weights whether init runs before or after + the .to(dtype) cast (boot inits first; init_weights() runs after).""" + cfg = _cfg() + boot_then_cast = TransformerBridge.boot_native(cfg).to(torch.float16) + + recast = TransformerBridge.boot_native(cfg).to(torch.float16) + native = recast.original_model + initialize_native_model(native, cfg) # re-init AFTER the cast + + for (name, a), (_, b) in zip( + boot_then_cast.original_model.named_parameters(), native.named_parameters() + ): + assert torch.equal(a, b), f"{name} diverged between init-before and init-after cast" + + +def test_seeded_init_is_deterministic(): + cfg = _cfg() + a = TransformerBridge.boot_native(cfg) + b = TransformerBridge.boot_native(cfg) + for (name, pa), (_, pb) in zip( + a.original_model.named_parameters(), b.original_model.named_parameters() + ): + assert torch.equal(pa, pb), name + + +@pytest.mark.parametrize("mode", ["xavier_normal", "kaiming_normal"]) +def test_initializer_range_scales_non_gpt2_modes(mode): + """An explicit initializer_range acts as the gain (legacy semantics).""" + plain = TransformerBridge.boot_native(_cfg(init_mode=mode)) + scaled = TransformerBridge.boot_native(_cfg(init_mode=mode, initializer_range=0.5)) + + w_plain = plain.original_model.layers[0].attn.q.weight + w_scaled = scaled.original_model.layers[0].attn.q.weight + ratio = (w_scaled.std() / w_plain.std()).item() + assert ratio == pytest.approx(0.5, rel=0.05), f"gain not applied: ratio {ratio:.4f}" + + +def test_gpt2_mode_default_std_matches_legacy_formula(): + """Unset initializer_range must give N(0, 0.64/d_model) — std 0.8/sqrt(d_model). + + The legacy scheme, not GPT-2's paper 0.02: toy-model training dynamics + (the grokking demo memorizes vs. stalls) depend on this scale. + """ + import math + + cfg = _cfg(d_model=128, d_head=32, d_mlp=512, d_vocab=114) + bridge = TransformerBridge.boot_native(cfg) + std = bridge.original_model.layers[0].attn.q.weight.std().item() + assert std == pytest.approx(0.8 / math.sqrt(128), rel=0.05) diff --git a/tests/unit/model_bridge/supported_architectures/test_adapter_capability_guards.py b/tests/unit/model_bridge/supported_architectures/test_adapter_capability_guards.py new file mode 100644 index 0000000000..8c44ae39b1 --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_adapter_capability_guards.py @@ -0,0 +1,76 @@ +"""Capability flags must reach their consuming bridge code paths. + +Exercises the real forward/loss guard and generation-caching resolver with the +real adapter classes on a bare bridge (full construction needs remote-code +models these unit tests cannot load). +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch.nn as nn + +from tests.unit.model_bridge.supported_architectures.helpers import make_bridge_cfg +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.supported_architectures.dream import ( + DreamArchitectureAdapter, +) +from transformer_lens.model_bridge.supported_architectures.gidd import ( + GiddArchitectureAdapter, +) + + +class _StubModel(nn.Module): + """original_model is beartype-hinted nn.Module; carry only .config.""" + + def __init__(self) -> None: + super().__init__() + self.config = SimpleNamespace(is_encoder_decoder=False) + + +def _bare_bridge(adapter) -> TransformerBridge: + bridge = object.__new__(TransformerBridge) + nn.Module.__init__(bridge) + bridge.adapter = adapter + bridge.cfg = adapter.cfg + bridge.__dict__["original_model"] = _StubModel() + return bridge + + +class TestDiffusionLossGuard: + """Diffusion LMs must refuse the shifted causal loss instead of silently + computing it (bridge.forward's supports_causal_loss guard).""" + + def test_dream_loss_raises(self) -> None: + cfg = make_bridge_cfg("DreamModel", n_key_value_heads=4) + bridge = _bare_bridge(DreamArchitectureAdapter(cfg)) + with pytest.raises(NotImplementedError, match="shifted causal"): + bridge.forward("hi", return_type="loss") + + def test_gidd_both_raises(self) -> None: + cfg = make_bridge_cfg("GiddForDiffusionLM") + bridge = _bare_bridge(GiddArchitectureAdapter(cfg)) + with pytest.raises(NotImplementedError, match="shifted causal"): + bridge.forward("hi", return_type="both") + + +class TestRavenGenerationCaching: + """Huginn's depth recurrence cannot use HF past_key_values stepping; the + resolver must refuse the cache and reject batched generation.""" + + def _bridge(self) -> TransformerBridge: + from transformer_lens.model_bridge.supported_architectures.raven import ( + RavenArchitectureAdapter, + ) + + cfg = make_bridge_cfg("RavenForCausalLM") + return _bare_bridge(RavenArchitectureAdapter(cfg)) + + def test_kv_cache_refused(self) -> None: + assert self._bridge()._resolve_generation_caching(True, batched=False) is False + + def test_batched_generation_rejected(self) -> None: + with pytest.raises(NotImplementedError): + self._bridge()._resolve_generation_caching(True, batched=True) diff --git a/tests/unit/model_bridge/supported_architectures/test_ast_adapter.py b/tests/unit/model_bridge/supported_architectures/test_ast_adapter.py index 2bb395ea95..a2fe7f910e 100644 --- a/tests/unit/model_bridge/supported_architectures/test_ast_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_ast_adapter.py @@ -68,6 +68,7 @@ def test_classification_prefix_rebinding(self, hf_config, tl_config): adapter.prepare_model(model) assert adapter.component_mapping["blocks"].name == "audio_spectrogram_transformer.layers" + assert adapter.component_mapping["classifier_ln"].name == "classifier.layernorm" assert adapter.component_mapping["unembed"].name == "classifier.dense" assert adapter.cfg.d_vocab_out == 2 diff --git a/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py b/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py index a5d8e5ae31..60a966aec2 100644 --- a/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py @@ -8,9 +8,10 @@ - Anti-drift config flags """ +from types import SimpleNamespace + import pytest -from transformer_lens.config import TransformerBridgeConfig from transformer_lens.config.transformer_bridge_config import TransformerBridgeConfig from transformer_lens.conversion_utils.conversion_steps import RearrangeTensorConversion from transformer_lens.conversion_utils.param_processing_conversion import ( @@ -74,8 +75,11 @@ class TestBertComponentMapping: def test_top_level_keys(self, adapter: BertArchitectureAdapter) -> None: assert set(adapter.component_mapping.keys()) == { "embed", + "token_type_embed", "pos_embed", + "embed_ln", "blocks", + "mlm_head", "ln_final", "unembed", } @@ -83,19 +87,55 @@ def test_top_level_keys(self, adapter: BertArchitectureAdapter) -> None: def test_bridge_types(self, adapter: BertArchitectureAdapter) -> None: mapping = adapter.component_mapping assert isinstance(mapping["embed"], EmbeddingBridge) + assert isinstance(mapping["token_type_embed"], EmbeddingBridge) assert isinstance(mapping["pos_embed"], PosEmbedBridge) + assert isinstance(mapping["embed_ln"], NormalizationBridge) assert isinstance(mapping["blocks"], BlockBridge) + assert isinstance(mapping["mlm_head"], LinearBridge) assert isinstance(mapping["ln_final"], NormalizationBridge) assert isinstance(mapping["unembed"], UnembeddingBridge) def test_top_level_hf_paths(self, adapter: BertArchitectureAdapter) -> None: mapping = adapter.component_mapping assert mapping["embed"].name == "bert.embeddings.word_embeddings" + assert mapping["token_type_embed"].name == "bert.embeddings.token_type_embeddings" assert mapping["pos_embed"].name == "bert.embeddings.position_embeddings" + assert mapping["embed_ln"].name == "bert.embeddings.LayerNorm" assert mapping["blocks"].name == "bert.encoder.layer" + assert mapping["mlm_head"].name == "cls.predictions.transform.dense" assert mapping["ln_final"].name == "cls.predictions.transform.LayerNorm" assert mapping["unembed"].name == "cls.predictions.decoder" + def test_token_type_embedding_is_cached_with_hf_output(self) -> None: + import torch + from transformers import BertForMaskedLM + + from transformer_lens.model_bridge.sources import build_bridge_from_module + + hf_model = BertForMaskedLM.from_pretrained("bert-base-cased").eval() + input_ids = torch.tensor([[101, 7592, 102, 2088, 102]]) + token_type_ids = torch.tensor([[0, 0, 0, 1, 1]]) + with torch.no_grad(): + expected = hf_model.bert.embeddings.token_type_embeddings(token_type_ids).clone() + + bridge = build_bridge_from_module( + hf_model, + "BertForMaskedLM", + hf_config=hf_model.config, + dtype=torch.float32, + device="cpu", + model_name="bert-base-cased", + ) + with torch.no_grad(): + _, cache = bridge.run_with_cache( + input_ids, + token_type_ids=token_type_ids, + names_filter=["token_type_embed.hook_out"], + ) + + assert "token_type_embed.hook_out" in cache + torch.testing.assert_close(cache["token_type_embed.hook_out"], expected) + def test_block_submodule_keys(self, adapter: BertArchitectureAdapter) -> None: assert set(adapter.component_mapping["blocks"].submodules.keys()) == { "ln1", @@ -140,6 +180,36 @@ def test_mlp_submodule_hf_paths(self, adapter: BertArchitectureAdapter) -> None: assert mlp.submodules["out"].name == "output.dense" +class TestBertTaskHeadMappings: + def test_nsp_only_model_uses_hooked_encoder_names(self) -> None: + adapter = BertArchitectureAdapter(_make_cfg()) + hf_model = SimpleNamespace( + bert=SimpleNamespace(pooler=object()), + cls=SimpleNamespace(seq_relationship=object()), + ) + + adapter.prepare_model(hf_model) + + assert adapter.components["pooler"].name == "bert.pooler.dense" + assert adapter.components["unembed"].name == "cls.seq_relationship" + assert "mlm_head" not in adapter.components + assert "ln_final" not in adapter.components + + def test_combined_mlm_nsp_model_registers_both_heads(self) -> None: + adapter = BertArchitectureAdapter(_make_cfg()) + hf_model = SimpleNamespace( + bert=SimpleNamespace(pooler=object()), + cls=SimpleNamespace(predictions=object(), seq_relationship=object()), + ) + + adapter.prepare_model(hf_model) + + assert adapter.components["pooler"].name == "bert.pooler.dense" + assert adapter.components["mlm_head"].name == "cls.predictions.transform.dense" + assert adapter.components["nsp_head"].name == "cls.seq_relationship" + assert adapter.components["unembed"].name == "cls.predictions.decoder" + + # --------------------------------------------------------------------------- # Anti-drift config flags # --------------------------------------------------------------------------- diff --git a/tests/unit/model_bridge/supported_architectures/test_olmo_adapter.py b/tests/unit/model_bridge/supported_architectures/test_olmo_adapter.py index ab0f5d04d7..201d2381d7 100644 --- a/tests/unit/model_bridge/supported_architectures/test_olmo_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_olmo_adapter.py @@ -100,6 +100,14 @@ def __init__(self, d_model: int, n_heads: int, n_kv_heads: int) -> None: self.o_proj = nn.Linear(n_heads * head_dim, d_model, bias=False) +class _RecordingLinear(nn.Linear): + """Record the bridge input dtype while handling conversion internally.""" + + def forward(self, input: torch.Tensor) -> torch.Tensor: + self.received_dtype = input.dtype + return super().forward(input.to(self.weight.dtype)) + + def _wire_attention_bridge( adapter: OlmoArchitectureAdapter, cfg: TransformerBridgeConfig, @@ -297,6 +305,32 @@ def _hook(tensor: torch.Tensor, hook: Any) -> None: assert seen["v"] == torch.Size([batch, seq_len, cfg.n_key_value_heads, cfg.d_head]) assert seen["z"] == torch.Size([batch, seq_len, cfg.n_heads, cfg.d_head]) + def test_forward_preserves_input_dtype_at_projection_boundary( + self, adapter: OlmoArchitectureAdapter, cfg: TransformerBridgeConfig + ) -> None: + attn_bridge = _wire_attention_bridge(adapter, cfg) + recordings = [] + for name in ("q", "k", "v"): + projection = getattr(attn_bridge, name) + original = projection.original_component + assert isinstance(original, nn.Linear) + recording = _RecordingLinear(original.in_features, original.out_features, bias=False) + recording.load_state_dict(original.state_dict()) + projection.set_original_component(recording) + recordings.append(recording) + + hidden_states = torch.randn(1, 3, cfg.d_model, dtype=torch.bfloat16) + position_embeddings = identity_rope(3, cfg.d_head) + + attn_bridge( + hidden_states=hidden_states, + position_embeddings=position_embeddings, + attention_mask=None, + ) + + for projection in recordings: + assert projection.received_dtype == torch.bfloat16 + class TestOlmoClipQkv: """The reconstructed forward must clamp Q/K/V when config.clip_qkv is set.""" diff --git a/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py b/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py index cdf99ae1a3..fdf7541279 100644 --- a/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py @@ -279,6 +279,9 @@ def _bare_model(self) -> object: """No 'vit'/'deit'/'classifier' attribute — mimics bare ViTModel/DeiTModel.""" return SimpleNamespace() + def _bare_model_with_pooler(self) -> object: + return SimpleNamespace(pooler=SimpleNamespace(dense=SimpleNamespace())) + def _vit_for_classification(self) -> object: return SimpleNamespace(vit=SimpleNamespace(), classifier=SimpleNamespace()) @@ -305,6 +308,12 @@ def test_bare_model_has_no_unembed(self, adapter: ViTArchitectureAdapter) -> Non adapter.prepare_model(self._bare_model()) assert "unembed" not in adapter.component_mapping + def test_bare_model_maps_pooler_without_root_name_collision( + self, adapter: ViTArchitectureAdapter + ) -> None: + adapter.prepare_model(self._bare_model_with_pooler()) + assert adapter.component_mapping["pooler"].name == "pooler.dense" + def test_bare_model_does_not_require_encoder_attribute( self, adapter: ViTArchitectureAdapter ) -> None: diff --git a/tests/unit/model_bridge/test_batched_string_padding.py b/tests/unit/model_bridge/test_batched_string_padding.py new file mode 100644 index 0000000000..4345d18717 --- /dev/null +++ b/tests/unit/model_bridge/test_batched_string_padding.py @@ -0,0 +1,131 @@ +"""Padding behavior for ragged string lists passed to TransformerBridge.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pytest +import torch + +from transformer_lens.utilities import get_attention_mask + +PROMPTS = ["The quick brown fox", "Hello there, world! This is a longer sentence."] +RESID_PRE = "blocks.0.hook_resid_pre" + + +@pytest.fixture(scope="module") +def compatibility_bridge(): + """A cached model used for token-layout and real-token numerical checks.""" + from transformer_lens.model_bridge import TransformerBridge + + bridge = TransformerBridge.boot_transformers("distilgpt2", device="cpu", dtype=torch.float32) + bridge.enable_compatibility_mode() + bridge.eval() + return bridge + + +def test_to_tokens_honors_explicit_padding_side(compatibility_bridge) -> None: + """A per-call padding override must win without mutating the shared tokenizer.""" + tokenizer = compatibility_bridge.tokenizer + original_side = tokenizer.padding_side + tokenizer.padding_side = "right" + try: + left = compatibility_bridge.to_tokens(PROMPTS, padding_side="left") + right = compatibility_bridge.to_tokens(PROMPTS, padding_side="right") + + assert not torch.equal(left, right) + assert left[0, 0].item() == tokenizer.pad_token_id + assert left[0, -1].item() != tokenizer.pad_token_id + assert right[0, -1].item() == tokenizer.pad_token_id + assert tokenizer.padding_side == "right" + finally: + tokenizer.padding_side = original_side + + +def test_default_right_padded_text_matches_pretokenized_logits(compatibility_bridge) -> None: + """Default text forwarding must match the public right-padded token layout.""" + tokenizer = compatibility_bridge.tokenizer + original_side = tokenizer.padding_side + tokenizer.padding_side = "right" + try: + tokens = compatibility_bridge.to_tokens(PROMPTS) + with torch.no_grad(): + text_logits = compatibility_bridge(PROMPTS) + token_logits = compatibility_bridge(tokens) + + torch.testing.assert_close(text_logits, token_logits, rtol=0, atol=0) + finally: + tokenizer.padding_side = original_side + + +@pytest.mark.parametrize("padding_side", ["left", "right"]) +def test_ragged_forward_matches_public_layout_and_single_sequences( + compatibility_bridge, monkeypatch, padding_side: str +) -> None: + """String-list forward uses public token layout and preserves real-token values.""" + tokenizer = compatibility_bridge.tokenizer + original_side = tokenizer.padding_side + tokenizer.padding_side = padding_side + try: + expected_tokens = compatibility_bridge.to_tokens(PROMPTS) + attention_mask = get_attention_mask(tokenizer, expected_tokens, prepend_bos=True).bool() + tokenization_calls: list[torch.Tensor] = [] + original_to_tokens: Callable[..., torch.Tensor] = compatibility_bridge.to_tokens + + def recording_to_tokens(input: Any, *args: Any, **kwargs: Any) -> torch.Tensor: + tokens = original_to_tokens(input, *args, **kwargs) + if isinstance(input, list): + tokenization_calls.append(tokens.detach().clone()) + return tokens + + monkeypatch.setattr(compatibility_bridge, "to_tokens", recording_to_tokens) + + with torch.no_grad(): + batch_logits, batch_cache = compatibility_bridge.run_with_cache( + PROMPTS, names_filter=[RESID_PRE] + ) + single_results = [ + compatibility_bridge.run_with_cache(prompt, names_filter=[RESID_PRE]) + for prompt in PROMPTS + ] + + assert len(tokenization_calls) == 1 + assert torch.equal(tokenization_calls[0], expected_tokens) + + for index, (single_logits, single_cache) in enumerate(single_results): + real_tokens = attention_mask[index] + torch.testing.assert_close( + batch_logits[index, real_tokens], single_logits[0], rtol=1e-5, atol=1e-4 + ) + torch.testing.assert_close( + batch_cache[RESID_PRE][index, real_tokens], + single_cache[RESID_PRE][0], + rtol=1e-5, + atol=1e-4, + ) + finally: + tokenizer.padding_side = original_side + + +def test_generate_keeps_left_padding_for_ragged_strings(compatibility_bridge) -> None: + """Generation keeps real tokens flush-right even when the tokenizer defaults right.""" + tokenizer = compatibility_bridge.tokenizer + original_side = tokenizer.padding_side + try: + tokenizer.padding_side = "left" + expected_tokens = compatibility_bridge.to_tokens(PROMPTS) + tokenizer.padding_side = "right" + + _, input_tokens = compatibility_bridge.generate( + PROMPTS, + max_new_tokens=1, + do_sample=False, + verbose=False, + return_input_tokens=True, + ) + + assert torch.equal(input_tokens, expected_tokens) + assert tokenizer.padding_side == "right" + finally: + tokenizer.padding_side = original_side diff --git a/tests/unit/model_bridge/test_boot_native.py b/tests/unit/model_bridge/test_boot_native.py index 1e11891f64..2f0069069f 100644 --- a/tests/unit/model_bridge/test_boot_native.py +++ b/tests/unit/model_bridge/test_boot_native.py @@ -1,12 +1,17 @@ """Tests for ``TransformerBridge.boot_native`` classmethod.""" + from __future__ import annotations import sys +import pytest import torch +import torch.nn as nn from transformer_lens.config import TransformerBridgeConfig from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.generalized_components import LinearBridge from transformer_lens.model_bridge.sources.native import NativeModel @@ -71,6 +76,51 @@ def test_boot_native_returns_bridge_over_native_model(): assert isinstance(bridge.original_model, NativeModel) +@pytest.mark.parametrize("stop_at_layer", [0, 2, -1]) +def test_boot_native_direct_stop_matches_cached_stop(stop_at_layer: int): + bridge = TransformerBridge.boot_native(_cfg(n_layers=3)) + bridge.eval() + tokens = torch.tensor([[1, 2, 3]]) + + with torch.no_grad(): + expected, _ = bridge.run_with_cache(tokens, stop_at_layer=stop_at_layer) + actual = bridge(tokens, stop_at_layer=stop_at_layer) + + assert actual.shape == (1, 3, bridge.cfg.d_model) + torch.testing.assert_close(actual, expected) + + +def test_native_state_dict_round_trip_restores_parameters(): + bridge = TransformerBridge.boot_native(_cfg()) + + saved_state_dict = {key: value.detach().clone() for key, value in bridge.state_dict().items()} + original_parameters = { + name: parameter.detach().clone() for name, parameter in bridge.named_parameters() + } + + with torch.no_grad(): + for parameter in bridge.parameters(): + parameter.zero_() + + result = bridge.load_state_dict(saved_state_dict, strict=True) + + assert result.missing_keys == [] + assert result.unexpected_keys == [] + + for name, parameter in bridge.named_parameters(): + torch.testing.assert_close(parameter, original_parameters[name]) + + +def test_native_state_dict_strict_rejects_unexpected_keys(): + bridge = TransformerBridge.boot_native(_cfg()) + + with pytest.raises(RuntimeError, match="Unexpected key"): + bridge.load_state_dict( + {"not.a.real.weight": torch.zeros(1)}, + strict=True, + ) + + def test_boot_native_accepts_dict_config(): cfg_dict = dict( d_model=32, @@ -88,6 +138,30 @@ def test_boot_native_accepts_dict_config(): assert bridge.cfg.architecture == "TransformerLensNative" +def test_boot_native_rejects_legacy_config_with_actionable_error(): + import pytest + + from transformer_lens import HookedTransformerConfig + + legacy_config = HookedTransformerConfig( + n_layers=1, + d_model=32, + n_ctx=8, + d_head=16, + n_heads=2, + d_vocab=16, + act_fn="gelu", + ) + + with pytest.raises( + TypeError, + match=( + "boot_native expected a TransformerBridgeConfig or dict, " "got HookedTransformerConfig" + ), + ): + TransformerBridge.boot_native(legacy_config) + + def test_boot_native_does_not_perturb_global_rng(): """``boot_native(seed=...)`` must use a scoped torch.Generator instead of ``torch.manual_seed``. Otherwise a user calling boot_native then @@ -127,6 +201,70 @@ def test_boot_native_distinct_seeds_diverge(): assert any(diffs), "Two different seeds produced identical params" +def test_boot_native_skips_custom_init_when_disabled(monkeypatch): + def fail_if_called(*_args, **_kwargs): + pytest.fail("initialize_native_model was called with init_weights=False") + + def fail_if_forked(*_args, **_kwargs): + pytest.fail("fork_rng was called with init_weights=False") + + monkeypatch.setattr( + "transformer_lens.model_bridge.sources.native.initialize_native_model", + fail_if_called, + ) + monkeypatch.setattr(torch.random, "fork_rng", fail_if_forked) + bridge = TransformerBridge.boot_native(_cfg(init_weights=False)) + + assert isinstance(bridge.original_model, NativeModel) + assert torch.count_nonzero(bridge.original_model.layers[0].attn.q.bias) > 0 + + +def test_native_bridge_init_weights_reinitializes_in_place_and_honors_seed(): + bridge = TransformerBridge.boot_native(_cfg(seed=123)) + model = bridge.original_model + expected = {name: param.detach().clone() for name, param in model.named_parameters()} + + with torch.no_grad(): + for param in model.parameters(): + param.fill_(42) + + bridge.init_weights() + + assert bridge.original_model is model + for name, param in model.named_parameters(): + assert torch.equal(param, expected[name]), f"Seed mismatch on {name}" + + +def test_native_bridge_init_weights_does_not_perturb_global_rng(): + bridge = TransformerBridge.boot_native(_cfg(seed=42)) + torch.manual_seed(0) + expected_after = torch.randn(5) + + torch.manual_seed(0) + bridge.init_weights() + actual_after = torch.randn(5) + + assert torch.equal(actual_after, expected_after) + + +def test_init_weights_rejects_non_native_bridge(): + class StubModel(nn.Module): + def __init__(self): + super().__init__() + self.proj = nn.Linear(4, 4) + + class StubAdapter(ArchitectureAdapter): + def __init__(self, cfg): + super().__init__(cfg) + self.component_mapping = {"stub_proj": LinearBridge(name="proj")} + + cfg = _cfg(architecture="StubForTest") + bridge = TransformerBridge(StubModel(), StubAdapter(cfg), tokenizer=None) + + with pytest.raises(RuntimeError, match=r"boot_native.*StubModel"): + bridge.init_weights() + + def test_boot_native_forward_and_cache(): cfg = _cfg() bridge = TransformerBridge.boot_native(cfg) @@ -137,6 +275,35 @@ def test_boot_native_forward_and_cache(): assert "blocks.0.attn.hook_pattern" in cache +@pytest.mark.parametrize("prepend_bos", [True, False]) +def test_run_with_cache_forwards_prepend_bos_for_string_input(monkeypatch, prepend_bos): + cfg = _cfg() + bridge = TransformerBridge.boot_native(cfg) + bridge._tokenizer = object() + tokenization_calls = [] + + def to_tokens(input, prepend_bos=None, padding_side=None): + tokenization_calls.append((input, prepend_bos, padding_side)) + if prepend_bos is None: + prepend_bos = bridge.cfg.default_prepend_bos + tokens = [0, 7] if prepend_bos else [7] + return torch.tensor([tokens]) + + monkeypatch.setattr(bridge, "to_tokens", to_tokens) + bridge.eval() + + with torch.no_grad(): + direct_logits = bridge("hello", prepend_bos=prepend_bos) + cached_logits, cache = bridge.run_with_cache("hello", prepend_bos=prepend_bos) + + assert tokenization_calls == [ + ("hello", prepend_bos, None), + ("hello", prepend_bos, None), + ] + torch.testing.assert_close(cached_logits, direct_logits) + assert cache["hook_embed"].shape[1] == direct_logits.shape[1] + + def test_boot_native_does_not_load_transformers_runtime(): # Sanity that the native path doesn't depend on HuggingFace's `transformers` # for the runtime work — we check that calling boot_native doesn't trigger @@ -323,3 +490,40 @@ def test_boot_native_supports_training_step(): ), "No non-zero gradients after backward" optimizer.step() optimizer.zero_grad() + + +def test_boot_native_resolves_initializer_range_sentinel(): + """Regression for #1568 — the resolved initializer range must be used by boot_native.""" + import math + + cfg = _cfg(init_mode="gpt2") + expected = 0.8 / math.sqrt(cfg.d_model) + + assert cfg.initializer_range == pytest.approx(expected) + + bridge = TransformerBridge.boot_native(cfg) + assert bridge.W_E.std().item() == pytest.approx(expected, rel=0.15) + + +def test_boot_native_resolves_non_gpt2_initializer_range_sentinel(): + cfg = _cfg(init_mode="kaiming_normal") + + assert cfg.initializer_range == pytest.approx(1.0) + + +def test_boot_native_kaiming_gain_scales_weights(): + """Regression for #1568 — xavier/kaiming init must use initializer_range + as a multiplicative gain. Without this, the config value is silently + ignored and every kaiming/xavier model gets the same fixed scale + regardless of what the caller asked for.""" + cfg_gain_1 = _cfg(init_mode="kaiming_normal", initializer_range=1.0, seed=0) + cfg_gain_2 = _cfg(init_mode="kaiming_normal", initializer_range=2.0, seed=0) + + bridge_1 = TransformerBridge.boot_native(cfg_gain_1) + bridge_2 = TransformerBridge.boot_native(cfg_gain_2) + + std_1 = bridge_1.W_E.std().item() + std_2 = bridge_2.W_E.std().item() + + # Same seed, only gain differs -> std should scale ~proportionally. + assert std_2 / std_1 == pytest.approx(2.0) diff --git a/tests/unit/model_bridge/test_bridge_generate_no_tokenizer.py b/tests/unit/model_bridge/test_bridge_generate_no_tokenizer.py index 9819fec1bc..cdeeedcfed 100644 --- a/tests/unit/model_bridge/test_bridge_generate_no_tokenizer.py +++ b/tests/unit/model_bridge/test_bridge_generate_no_tokenizer.py @@ -8,7 +8,6 @@ generation path (algorithmic/custom-tokenized use cases). """ -import platform import pytest import torch @@ -17,8 +16,6 @@ _PROMPT_TOKENS = torch.tensor([[15496, 11, 314, 1101, 257]], dtype=torch.long) -_MACOS_ARM64 = platform.system() == "Darwin" and platform.machine() == "arm64" - @pytest.fixture(scope="module") def tokenizer_free_bridge(): @@ -27,7 +24,6 @@ def tokenizer_free_bridge(): return bridge -@pytest.mark.skipif(_MACOS_ARM64, reason="Upstream macOS-arm64 KV-cache NaN; see linked issue.") def test_generate_without_tokenizer_stop_at_eos_false_kv_cache(tokenizer_free_bridge): """generate() with no tokenizer, stop_at_eos=False, use_past_kv_cache=True.""" bridge = tokenizer_free_bridge @@ -164,7 +160,6 @@ def test_generate_string_input_without_tokenizer_errors(tokenizer_free_bridge): bridge.generate("hello", max_new_tokens=3, verbose=False) -@pytest.mark.skipif(_MACOS_ARM64, reason="Upstream macOS-arm64 KV-cache NaN; see linked issue.") def test_generate_return_type_str_without_tokenizer_errors(tokenizer_free_bridge): """generate(return_type='str') must error when no tokenizer is set. diff --git a/tests/unit/model_bridge/test_bridge_vs_hooked_transformer_mlp_in_patching.py b/tests/unit/model_bridge/test_bridge_vs_hooked_transformer_mlp_in_patching.py index e643905a13..917ac098ab 100644 --- a/tests/unit/model_bridge/test_bridge_vs_hooked_transformer_mlp_in_patching.py +++ b/tests/unit/model_bridge/test_bridge_vs_hooked_transformer_mlp_in_patching.py @@ -3,6 +3,7 @@ Parameterized over Pythia (native autograd LN) and GPT-2 (manual LN), and over ``no_processing`` so both folded and unfolded compat-mode setups are covered. """ + from __future__ import annotations import pytest @@ -105,20 +106,19 @@ def _inner(tensor: torch.Tensor, hook: object) -> torch.Tensor: @pytest.mark.slow def test_mlp_in_gated_off_does_not_fire() -> None: - """When ``use_hook_mlp_in`` is False, the bridge pre-ln2 closure must skip firing.""" + """When ``use_hook_mlp_in`` is False, explicitly adding hook_mlp_in + should fail with a clear error explaining how to enable the gated hook.""" bridge = TransformerBridge.boot_transformers("gpt2", device="cpu") bridge.enable_compatibility_mode(no_processing=True) bridge.set_use_hook_mlp_in(False) - fire_count = {"n": 0} - def _counter(tensor: torch.Tensor, hook: object) -> torch.Tensor: - fire_count["n"] += 1 return tensor prompt = torch.arange(1, 9).unsqueeze(0) - bridge.run_with_hooks(prompt, fwd_hooks=[("blocks.0.hook_mlp_in", _counter)]) - assert fire_count["n"] == 0, ( - f"hook_mlp_in fired {fire_count['n']} times with use_hook_mlp_in=False; " - "should not fire when the flag is off" - ) + + with pytest.raises(ValueError, match="set_use_hook_mlp_in"): + bridge.run_with_hooks( + prompt, + fwd_hooks=[("blocks.0.hook_mlp_in", _counter)], + ) diff --git a/tests/unit/model_bridge/test_config_flag_assignment.py b/tests/unit/model_bridge/test_config_flag_assignment.py new file mode 100644 index 0000000000..d613312a05 --- /dev/null +++ b/tests/unit/model_bridge/test_config_flag_assignment.py @@ -0,0 +1,223 @@ +"""Tests for direct assignment of Bridge-managed hook flags (#1689).""" + +from __future__ import annotations + +import copy +import gc + +import pytest +import torch +from torch import nn +from transformers import GPT2Config, GPT2LMHeadModel, LlamaConfig, LlamaForCausalLM + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.factories.architecture_adapter_factory import ( + ArchitectureAdapterFactory, +) +from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, +) +from transformer_lens.model_bridge.sources.native import NativeModel + + +def _cfg() -> TransformerBridgeConfig: + return TransformerBridgeConfig( + d_model=32, + d_head=16, + n_heads=2, + n_layers=1, + n_ctx=8, + d_vocab=16, + d_mlp=64, + act_fn="gelu", + normalization_type="LN", + seed=0, + ) + + +def _tiny_gpt2_bridge() -> TransformerBridge: + hf_config = GPT2Config( + n_layer=1, + n_head=2, + n_embd=32, + n_positions=8, + n_ctx=8, + vocab_size=16, + ) + hf_model = GPT2LMHeadModel(hf_config).eval() + return build_bridge_from_module( + hf_model, + "GPT2LMHeadModel", + hf_config=hf_config, + tokenizer=None, + device="cpu", + ) + + +def _tiny_llama_bridge() -> TransformerBridge: + hf_config = LlamaConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=1, + num_attention_heads=2, + num_key_value_heads=2, + vocab_size=16, + max_position_embeddings=8, + ) + hf_model = LlamaForCausalLM(hf_config).eval() + return build_bridge_from_module( + hf_model, + "LlamaForCausalLM", + hf_config=hf_config, + tokenizer=None, + device="cpu", + ) + + +@pytest.fixture(params=["gpt2", "llama"], ids=["shared-config", "cloned-config"]) +def bridge_with_config_mode(request: pytest.FixtureRequest) -> TransformerBridge: + bridge = _tiny_gpt2_bridge() if request.param == "gpt2" else _tiny_llama_bridge() + attn_config_is_shared = bridge.blocks[0].attn.config is bridge.cfg + assert attn_config_is_shared is (request.param == "gpt2") + return bridge + + +@pytest.mark.parametrize( + ("flag_name", "hook_name"), + [ + ("use_attn_result", "blocks.0.attn.hook_result"), + ("use_attn_in", "blocks.0.attn.hook_attn_in"), + ("use_hook_mlp_in", "blocks.0.hook_mlp_in"), + ("use_split_qkv_input", "blocks.0.attn.hook_q_input"), + ], +) +def test_direct_assignment_matches_setter_hook_behavior( + bridge_with_config_mode: TransformerBridge, flag_name: str, hook_name: str +) -> None: + bridge = bridge_with_config_mode + tokens = torch.randint(0, bridge.cfg.d_vocab, (1, 8)) + + setattr(bridge.cfg, flag_name, True) + _, direct_cache = bridge.run_with_cache(tokens, names_filter=[hook_name]) + + setattr(bridge.cfg, flag_name, False) + getattr(bridge, f"set_{flag_name}")(True) + _, setter_cache = bridge.run_with_cache(tokens, names_filter=[hook_name]) + + assert list(direct_cache) == [hook_name] + assert list(setter_cache) == [hook_name] + assert direct_cache[hook_name].shape == setter_cache[hook_name].shape + + +def test_direct_assignment_preserves_mutual_exclusivity() -> None: + bridge = _tiny_gpt2_bridge() + + bridge.cfg.use_split_qkv_input = True + with pytest.raises(ValueError, match="mutually exclusive"): + bridge.cfg.use_attn_in = True + assert bridge.cfg.use_attn_in is False + + bridge.cfg.use_split_qkv_input = False + bridge.cfg.use_attn_in = True + with pytest.raises(ValueError, match="mutually exclusive"): + bridge.cfg.use_split_qkv_input = True + assert bridge.cfg.use_split_qkv_input is False + + +@pytest.mark.parametrize("flag_name", ["use_attn_result", "use_attn_in", "use_split_qkv_input"]) +def test_direct_assignment_preserves_unsupported_architecture_errors( + monkeypatch: pytest.MonkeyPatch, flag_name: str +) -> None: + bridge = TransformerBridge.boot_native(_cfg()) + + class _FakeBlock(nn.Module): + def __init__(self) -> None: + super().__init__() + self.attn = nn.Identity() + + monkeypatch.setattr(bridge, "blocks", nn.ModuleList([_FakeBlock()]), raising=True) + + with pytest.raises(NotImplementedError, match=flag_name): + setattr(bridge.cfg, flag_name, True) + assert getattr(bridge.cfg, flag_name) is False + + +def test_deepcopied_live_config_is_not_bound_to_original_bridge() -> None: + bridge = TransformerBridge.boot_native(_cfg()) + copied_cfg = copy.deepcopy(bridge.cfg) + + copied_cfg.use_hook_mlp_in = True + + assert copied_cfg.use_hook_mlp_in is True + assert bridge.cfg.use_hook_mlp_in is False + + +def test_deepcopied_bridge_rebinds_its_config() -> None: + bridge = TransformerBridge.boot_native(_cfg()) + copied_bridge = copy.deepcopy(bridge) + + copied_bridge.cfg.use_hook_mlp_in = True + + assert copied_bridge.cfg.use_hook_mlp_in is True + assert copied_bridge.blocks[0].config.use_hook_mlp_in is True + assert bridge.cfg.use_hook_mlp_in is False + + +def test_shallow_copied_bridge_does_not_replace_live_config_binding() -> None: + bridge = TransformerBridge.boot_native(_cfg()) + with pytest.warns(UserWarning, match="already bound to another live"): + copied_bridge = copy.copy(bridge) + + assert copied_bridge.cfg is bridge.cfg + assert bridge.cfg._bridge_ref() is bridge + + del copied_bridge + gc.collect() + bridge.cfg.use_hook_mlp_in = True + + assert bridge.blocks[0].config.use_hook_mlp_in is True + + +def test_constructor_warns_when_live_bridge_already_owns_config() -> None: + cfg = _cfg() + cfg.architecture = "TransformerLensNative" + first_model = NativeModel(cfg) + second_model = NativeModel(cfg) + first_adapter = ArchitectureAdapterFactory.select_architecture_adapter(cfg) + second_adapter = ArchitectureAdapterFactory.select_architecture_adapter(cfg) + first_adapter.prepare_model(first_model) + second_adapter.prepare_model(second_model) + first_bridge = TransformerBridge(first_model, first_adapter, tokenizer=None) + + with pytest.warns(UserWarning, match="already bound to another live"): + second_bridge = TransformerBridge(second_model, second_adapter, tokenizer=None) + + assert second_bridge.cfg is first_bridge.cfg + assert cfg._bridge_ref() is first_bridge + + +def test_attention_flag_propagation_does_not_dispatch_bound_cloned_config() -> None: + bridge = _tiny_llama_bridge() + cloned_cfg = bridge.blocks[0].attn.config + other_bridge = TransformerBridge.boot_native(_cfg()) + assert cloned_cfg is not bridge.cfg + cloned_cfg._bind_bridge(other_bridge) + + bridge.set_use_attn_in(True) + + assert cloned_cfg.use_attn_in is True + assert other_bridge.cfg.use_attn_in is False + + +def test_mlp_flag_propagation_does_not_dispatch_bound_cloned_config() -> None: + bridge = _tiny_gpt2_bridge() + cloned_cfg = bridge.blocks[0].config + other_bridge = TransformerBridge.boot_native(_cfg()) + assert cloned_cfg is not bridge.cfg + cloned_cfg._bind_bridge(other_bridge) + + bridge.set_use_hook_mlp_in(True) + + assert cloned_cfg.use_hook_mlp_in is True + assert other_bridge.cfg.use_hook_mlp_in is False diff --git a/tests/unit/model_bridge/test_expand_kv_heads.py b/tests/unit/model_bridge/test_expand_kv_heads.py new file mode 100644 index 0000000000..35b60b78db --- /dev/null +++ b/tests/unit/model_bridge/test_expand_kv_heads.py @@ -0,0 +1,56 @@ +"""Unit tests for TransformerBridge._expand_kv_heads. + +Regression for https://github.com/TransformerLensOrg/TransformerLens/issues/1553: +weight circuits on GQA models must expand the grouped K/V head axis to n_heads +(repeat_interleave, matching HookedTransformer's GroupedQueryAttention layout) +before factoring, while MHA weights pass through untouched. +""" + +import pytest +import torch + +from transformer_lens.config.transformer_bridge_config import TransformerBridgeConfig +from transformer_lens.model_bridge.bridge import TransformerBridge + + +def _bridge_stub(n_heads: int) -> TransformerBridge: + """Uninitialized bridge carrying only the cfg that _expand_kv_heads reads.""" + bridge = TransformerBridge.__new__(TransformerBridge) + bridge.cfg = TransformerBridgeConfig( + d_model=8, + d_head=2, + n_heads=n_heads, + n_layers=2, + n_ctx=8, + d_vocab=16, + ) + return bridge + + +class TestExpandKvHeads: + def test_grouped_kv_expands_by_repeat_interleave(self): + bridge = _bridge_stub(n_heads=4) + grouped = torch.arange(2 * 2 * 3 * 2, dtype=torch.float32).reshape(2, 2, 3, 2) + + expanded = bridge._expand_kv_heads(grouped) + + assert expanded.shape == (2, 4, 3, 2) + # Query head h must read kv head h // (n_heads // n_kv_heads). + for h in range(4): + assert torch.equal(expanded[:, h], grouped[:, h // 2]) + + def test_mha_weights_pass_through_untouched(self): + bridge = _bridge_stub(n_heads=4) + mha = torch.randn(2, 4, 3, 2) + assert bridge._expand_kv_heads(mha) is mha + + def test_non_4d_input_passes_through_untouched(self): + bridge = _bridge_stub(n_heads=4) + bias_stack = torch.randn(2, 2, 2) + assert bridge._expand_kv_heads(bias_stack) is bias_stack + + def test_indivisible_head_counts_raise(self): + bridge = _bridge_stub(n_heads=4) + grouped = torch.randn(2, 3, 3, 2) + with pytest.raises(ValueError, match="multiple of n_kv_heads"): + bridge._expand_kv_heads(grouped) diff --git a/tests/unit/model_bridge/test_gated_hooks.py b/tests/unit/model_bridge/test_gated_hooks.py new file mode 100644 index 0000000000..4e212eb0af --- /dev/null +++ b/tests/unit/model_bridge/test_gated_hooks.py @@ -0,0 +1,90 @@ +"""Tests for gated hook validation (issue #1688). + +Adding a hook to a gated-off hook point (hook_result, hook_mlp_in, hook_attn_in, +hook_{q,k,v}_input) should fail loudly, not silently accept the hook and never fire it. +""" + +from __future__ import annotations + +import warnings + +import pytest +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge + + +def _cfg(**overrides) -> TransformerBridgeConfig: + base = dict( + d_model=32, + d_head=16, + n_heads=2, + n_layers=1, + n_ctx=8, + d_vocab=16, + d_mlp=64, + act_fn="gelu", + normalization_type="LN", + seed=0, + ) + base.update(overrides) + return TransformerBridgeConfig(**base) + + +def test_add_hook_rejects_gated_attn_result(): + """add_hook on hook_result with use_attn_result=False raises a clear ValueError.""" + bridge = TransformerBridge.boot_native(_cfg()) + with pytest.raises(ValueError, match="use_attn_result"): + bridge.add_hook("blocks.0.attn.hook_result", lambda t, hook=None: t) + + +def test_add_hook_rejects_gated_split_qkv_input(): + """add_hook on hook_q_input with use_split_qkv_input=False raises a clear ValueError.""" + bridge = TransformerBridge.boot_native(_cfg()) + with pytest.raises(ValueError, match="use_split_qkv_input"): + bridge.add_hook("blocks.0.attn.hook_q_input", lambda t, hook=None: t) + + +def test_add_hook_rejects_gated_mlp_in(): + """add_hook on hook_mlp_in with use_hook_mlp_in=False raises a clear ValueError.""" + bridge = TransformerBridge.boot_native(_cfg()) + with pytest.raises(ValueError, match="use_hook_mlp_in"): + bridge.add_hook("blocks.0.hook_mlp_in", lambda t, hook=None: t) + + +def test_add_hook_rejects_gated_attn_in(): + """add_hook on hook_attn_in with use_attn_in=False raises a clear ValueError.""" + bridge = TransformerBridge.boot_native(_cfg()) + with pytest.raises(ValueError, match="use_attn_in"): + bridge.add_hook("blocks.0.hook_attn_in", lambda t, hook=None: t) + + +def test_add_hook_succeeds_after_enabling_setter(): + """Regression guard: enabling the flag via the setter still lets the hook fire.""" + bridge = TransformerBridge.boot_native(_cfg()) + bridge.set_use_hook_mlp_in(True) + + fired = [] + bridge.add_hook("blocks.0.hook_mlp_in", lambda t, hook=None: fired.append(1) or t) + + tokens = torch.randint(0, 16, (1, 8)) + bridge(tokens, return_type="logits") + + assert len(fired) > 0, "Hook did not fire after enabling use_hook_mlp_in via the setter" + + +def test_run_with_cache_warns_on_fully_gated_names_filter(): + """run_with_cache with a filter matching only gated-off names warns instead of + silently returning an empty cache.""" + bridge = TransformerBridge.boot_native(_cfg()) + tokens = torch.randint(0, 16, (1, 8)) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _, cache = bridge.run_with_cache(tokens, names_filter=["blocks.0.hook_mlp_in"]) + + assert len(cache) == 0 + assert any("gated-off" in str(w.message) for w in caught), ( + "Expected a warning naming the gated-off hook, got: " f"{[str(w.message) for w in caught]}" + ) diff --git a/tests/unit/model_bridge/test_get_params_util.py b/tests/unit/model_bridge/test_get_params_util.py index e6ff03b7ba..4284988d8c 100644 --- a/tests/unit/model_bridge/test_get_params_util.py +++ b/tests/unit/model_bridge/test_get_params_util.py @@ -73,15 +73,15 @@ def test_get_bridge_params_attention_reshaping(self): w_v = params[f"blocks.{layer_idx}.attn.W_V"] w_o = params[f"blocks.{layer_idx}.attn.W_O"] - # Shape alone cannot catch a reshape that scrambles the elements, - # nor Q/K/V being read from the wrong projection: pin the VALUES - # against the source weights the mock block exposes. + # Shape alone cannot catch Q/K/V being read from the wrong + # projection: pin the VALUES against the TL-layout properties the + # mock block exposes. block = mock_bridge.blocks[layer_idx] n_heads, d_model, d_head = 12, 768, 64 assert w_q.shape == (n_heads, d_model, d_head) assert w_o.shape == (n_heads, d_head, d_model) - torch.testing.assert_close(w_q, block.attn.q.weight.reshape(n_heads, d_model, d_head)) - torch.testing.assert_close(w_o, block.attn.o.weight.reshape(n_heads, d_head, d_model)) + torch.testing.assert_close(w_q, block.attn.W_Q) + torch.testing.assert_close(w_o, block.attn.W_O) # Negative control: Q and K must be distinguishable in this fixture, # or reading either one would satisfy the assertions above. assert not torch.equal(w_q, w_k) @@ -197,13 +197,12 @@ def _create_mock_bridge_with_none_biases(self): # Set all biases to None for block in mock_bridge.blocks: - block.attn.q.bias = None - block.attn.k.bias = None - block.attn.v.bias = None - block.attn.o.bias = None - setattr(block.mlp, "in", Mock()) - getattr(block.mlp, "in").bias = None - block.mlp.out.bias = None + block.attn.b_Q = None + block.attn.b_K = None + block.attn.b_V = None + block.attn.b_O = None + block.mlp.b_in = None + block.mlp.b_out = None return mock_bridge @@ -231,45 +230,99 @@ def _create_mock_bridge_with_gate_weights(self): # Add gate weights to MLP for block in mock_bridge.blocks: - block.mlp.gate = Mock() - block.mlp.gate.weight = torch.randn(3072, 768) - block.mlp.gate.bias = torch.randn(3072) - block.mlp.W_gate = torch.randn(3072, 768) + block.mlp.W_gate = torch.randn(768, 3072) + block.mlp.b_gate = torch.randn(3072) return mock_bridge def _create_mock_block(self): - """Create a mock transformer block.""" + """Create a mock transformer block exposing TL-layout weight properties.""" block = Mock() - # Mock attention + # Mock attention (TL-layout properties, as the component bridges expose) block.attn = Mock() - block.attn.q = Mock() - block.attn.q.weight = torch.randn(768, 768) - block.attn.q.bias = torch.randn(768) + block.attn.W_Q = torch.randn(12, 768, 64) + block.attn.W_K = torch.randn(12, 768, 64) + block.attn.W_V = torch.randn(12, 768, 64) + block.attn.W_O = torch.randn(12, 64, 768) + block.attn.b_Q = torch.randn(12, 64) + block.attn.b_K = torch.randn(12, 64) + block.attn.b_V = torch.randn(12, 64) + block.attn.b_O = torch.randn(768) + + # Mock MLP + block.mlp = Mock() + block.mlp.W_in = torch.randn(768, 3072) + block.mlp.W_out = torch.randn(3072, 768) + block.mlp.b_in = torch.randn(3072) + block.mlp.b_out = torch.randn(768) - block.attn.k = Mock() - block.attn.k.weight = torch.randn(768, 768) - block.attn.k.bias = torch.randn(768) + return block - block.attn.v = Mock() - block.attn.v.weight = torch.randn(768, 768) - block.attn.v.bias = torch.randn(768) - block.attn.o = Mock() - block.attn.o.weight = torch.randn(768, 768) - block.attn.o.bias = torch.randn(768) +class TestGQAExpansion: + """Grouped K/V must be expanded to n_heads (legacy HT convention).""" - # Mock MLP (mirrors MLPBridge's normalized accessor API) - block.mlp = Mock() - block.mlp.W_in = torch.randn(768, 3072) - block.mlp.W_out = torch.randn(3072, 768) - setattr(block.mlp, "in", Mock()) - getattr(block.mlp, "in").weight = torch.randn(768, 3072) - getattr(block.mlp, "in").bias = torch.randn(3072) + def _make_gqa_bridge(self): + mock_bridge = Mock() + mock_bridge.cfg = Mock() + mock_bridge.cfg.n_layers = 1 + mock_bridge.cfg.d_model = 64 + mock_bridge.cfg.n_heads = 4 + mock_bridge.cfg.d_head = 16 + mock_bridge.cfg.d_vocab = 100 + mock_bridge.cfg.n_ctx = 32 + mock_bridge.cfg.d_mlp = 128 + mock_bridge.cfg.device = torch.device("cpu") - block.mlp.out = Mock() - block.mlp.out.weight = torch.randn(3072, 768) - block.mlp.out.bias = torch.randn(768) + mock_bridge.embed = Mock() + mock_bridge.embed.weight = torch.randn(100, 64) + mock_bridge.pos_embed = Mock() + mock_bridge.pos_embed.weight = torch.randn(32, 64) + mock_bridge.unembed = Mock() + mock_bridge.unembed.weight = torch.randn(100, 64) - return block + block = Mock() + block.attn = Mock() + block.attn.W_Q = torch.randn(4, 64, 16) + block.attn.W_K = torch.randn(2, 64, 16) # grouped: n_kv_heads=2 + block.attn.W_V = torch.randn(2, 64, 16) + block.attn.W_O = torch.randn(4, 16, 64) + block.attn.b_Q = torch.randn(4, 16) + block.attn.b_K = torch.randn(2, 16) + block.attn.b_V = torch.randn(2, 16) + block.attn.b_O = torch.randn(64) + block.mlp = Mock() + block.mlp.W_in = torch.randn(64, 128) + block.mlp.W_out = torch.randn(128, 64) + block.mlp.b_in = torch.randn(128) + block.mlp.b_out = torch.randn(64) + mock_bridge.blocks = [block] + return mock_bridge + + def test_grouped_kv_expanded_to_n_heads(self): + bridge = self._make_gqa_bridge() + params = get_bridge_params(bridge) + + w_k = params["blocks.0.attn.W_K"] + w_v = params["blocks.0.attn.W_V"] + assert w_k.shape == (4, 64, 16) + assert w_v.shape == (4, 64, 16) + # repeat_interleave semantics: heads 0,1 share kv head 0; heads 2,3 share kv head 1 + assert torch.equal(w_k[0], w_k[1]) + assert torch.equal(w_k[0], bridge.blocks[0].attn.W_K[0]) + assert torch.equal(w_k[2], bridge.blocks[0].attn.W_K[1]) + + def test_grouped_biases_expanded(self): + bridge = self._make_gqa_bridge() + params = get_bridge_params(bridge) + b_k = params["blocks.0.attn.b_K"] + assert b_k.shape == (4, 16) + assert params["blocks.0.attn.b_V"].shape == (4, 16) + # Pairing must be repeat_interleave (blocked), not tiling: heads 0,1 + # share kv head 0 and heads 2,3 share kv head 1. + grouped = bridge.blocks[0].attn.b_K + assert torch.equal(b_k[0], b_k[1]) + assert torch.equal(b_k[0], grouped[0]) + assert torch.equal(b_k[2], grouped[1]) + assert torch.equal(params["blocks.0.attn.b_Q"], bridge.blocks[0].attn.b_Q) diff --git a/tests/unit/model_bridge/test_get_params_util_helpers.py b/tests/unit/model_bridge/test_get_params_util_helpers.py index de764400e1..daa1ba4177 100644 --- a/tests/unit/model_bridge/test_get_params_util_helpers.py +++ b/tests/unit/model_bridge/test_get_params_util_helpers.py @@ -1,91 +1,40 @@ """Tests for get_params_util helper functions.""" -import torch - -from transformer_lens.model_bridge.get_params_util import ( - _get_n_kv_heads, - _get_or_create_bias, - _reshape_kv_weight, -) - - -class _FakeCfg: - """Minimal config stub for testing.""" +from unittest.mock import Mock - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - - -class TestGetNKVHeads: - def test_prefers_kv_heads_over_n_heads(self): - cfg = _FakeCfg(n_heads=12, n_key_value_heads=4) - assert _get_n_kv_heads(cfg) == 4 - assert _get_n_kv_heads(cfg) != cfg.n_heads - - def test_fallback_to_n_heads_when_missing(self): - cfg = _FakeCfg(n_heads=12) - assert _get_n_kv_heads(cfg) == 12 - assert not hasattr(cfg, "n_key_value_heads") +import torch - def test_none_kv_heads_falls_back(self): - # n_key_value_heads exists but is None — should fall back - cfg = _FakeCfg(n_heads=12, n_key_value_heads=None) - assert _get_n_kv_heads(cfg) == 12 +from transformer_lens.model_bridge.get_params_util import _tensor_attr -class TestReshapeKVWeight: - def test_full_size_preserves_data(self): - cfg = _FakeCfg(d_model=64, n_heads=4, d_head=16) - weight = torch.randn(64, 64) - result = _reshape_kv_weight(weight, cfg, "cpu", torch.float32) - assert result.shape == (4, 64, 16) - # Total elements must be preserved - assert result.numel() == weight.numel() - # Data must be the same (just reshaped) - assert torch.equal(result.reshape(-1), weight.reshape(-1)) +class TestTensorAttr: + def test_returns_first_tensor_among_names(self): + obj = Mock() + obj.w = torch.ones(3) + obj.weight = torch.zeros(3) + assert torch.equal(_tensor_attr(obj, "w", "weight"), torch.ones(3)) - def test_mqa_weight_expands_heads(self): - cfg = _FakeCfg(d_model=64, n_heads=4, d_head=16) - # MQA: single head (d_head, d_model) - weight = torch.randn(16, 64) - result = _reshape_kv_weight(weight, cfg, "cpu", torch.float32) - assert result.shape == (4, 64, 16) - # All 4 heads should be identical copies of the single head - for i in range(1, 4): - assert torch.equal(result[i], result[0]) + def test_falls_through_non_tensor_values(self): + # Mock auto-attributes return Mocks, which must not be mistaken for weights. + obj = Mock() + obj.weight = torch.full((2,), 5.0) + result = _tensor_attr(obj, "w", "weight") + assert torch.equal(result, torch.full((2,), 5.0)) - def test_numel_match_uses_view(self): - cfg = _FakeCfg(d_model=64, n_heads=4, d_head=16) - # Non-standard shape but total elements match - weight = torch.randn(4 * 64 * 16).reshape(32, 128) - result = _reshape_kv_weight(weight, cfg, "cpu", torch.float32) - assert result.shape == (4, 64, 16) - assert result.numel() == weight.numel() + def test_none_object_returns_none(self): + assert _tensor_attr(None, "weight") is None - def test_incompatible_shape_returns_zeros(self): - cfg = _FakeCfg(d_model=64, n_heads=4, d_head=16) - weight = torch.randn(7, 13) # impossible to reshape - result = _reshape_kv_weight(weight, cfg, "cpu", torch.float32) - assert result.shape == (4, 64, 16) - assert torch.all(result == 0) - # Verify it's actually zeros, not just small values - assert result.sum().item() == 0.0 + def test_missing_and_none_attrs_return_none(self): + class Holder: + bias = None + assert _tensor_attr(Holder(), "nonexistent", "bias") is None -class TestGetOrCreateBias: - def test_reshapes_existing_bias(self): - bias = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]) - result = _get_or_create_bias(bias, n_heads=2, d_head=4, device="cpu", dtype=torch.float32) - assert result.shape == (2, 4) - # Verify the reshape is correct — first head gets [1,2,3,4] - assert torch.equal(result[0], torch.tensor([1.0, 2.0, 3.0, 4.0])) - assert torch.equal(result[1], torch.tensor([5.0, 6.0, 7.0, 8.0])) + def test_property_raising_is_skipped(self): + class Flaky: + @property + def w(self): + raise AttributeError("not materialized") - def test_none_creates_zeros(self): - result = _get_or_create_bias(None, n_heads=4, d_head=16, device="cpu", dtype=torch.float32) - assert result.shape == (4, 16) - assert result.sum().item() == 0.0 + weight = torch.ones(2) - def test_none_respects_dtype(self): - result = _get_or_create_bias(None, n_heads=2, d_head=8, device="cpu", dtype=torch.float16) - assert result.dtype == torch.float16 + assert torch.equal(_tensor_attr(Flaky(), "w", "weight"), torch.ones(2)) diff --git a/tests/unit/model_bridge/test_loss_attention_mask.py b/tests/unit/model_bridge/test_loss_attention_mask.py new file mode 100644 index 0000000000..de9151f629 --- /dev/null +++ b/tests/unit/model_bridge/test_loss_attention_mask.py @@ -0,0 +1,200 @@ +"""Regression tests for padding-aware TransformerBridge causal loss.""" + +from __future__ import annotations + +import pytest +import torch +import torch.nn.functional as F + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge + + +def _bridge() -> TransformerBridge: + cfg = TransformerBridgeConfig( + d_model=32, + d_head=8, + n_heads=4, + n_layers=2, + n_ctx=6, + d_vocab=32, + d_mlp=64, + act_fn="gelu", + normalization_type="LN", + seed=7, + initializer_range=0.2, + ) + return TransformerBridge.boot_native(cfg) + + +def _extract_loss(output: torch.Tensor | tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: + return output[1] if isinstance(output, tuple) else output + + +def _manual_masked_loss( + logits: torch.Tensor, tokens: torch.Tensor, attention_mask: torch.Tensor +) -> torch.Tensor: + transition_mask = attention_mask[:, :-1].bool() & attention_mask[:, 1:].bool() + return F.cross_entropy( + logits[:, :-1][transition_mask], + tokens[:, 1:][transition_mask], + ) + + +@pytest.mark.parametrize("return_type", ["loss", "both"]) +def test_forward_loss_ignores_masked_padding_tokens(return_type: str) -> None: + bridge = _bridge() + attention_mask = torch.tensor( + [ + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 1, 1, 1], + ] + ) + token_batches = ( + torch.tensor( + [ + [1, 2, 3, 0, 0, 0], + [4, 5, 6, 7, 8, 9], + ] + ), + torch.tensor( + [ + [1, 2, 3, 31, 30, 29], + [4, 5, 6, 7, 8, 9], + ] + ), + ) + + losses = [] + for tokens in token_batches: + output = bridge(tokens, attention_mask=attention_mask, return_type=return_type) + loss = _extract_loss(output) + logits = bridge(tokens, attention_mask=attention_mask, return_type="logits") + expected = _manual_masked_loss(logits, tokens, attention_mask) + + torch.testing.assert_close(loss, expected) + losses.append(loss) + + torch.testing.assert_close(losses[0], losses[1]) + + +def test_forward_loss_per_token_zeros_masked_transitions() -> None: + bridge = _bridge() + tokens = torch.tensor( + [ + [1, 2, 3, 0, 0, 0], + [4, 5, 6, 7, 8, 9], + ] + ) + attention_mask = torch.tensor( + [ + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 1, 1, 1], + ] + ) + + loss = bridge( + tokens, + attention_mask=attention_mask, + return_type="loss", + loss_per_token=True, + ) + next_token_mask = torch.logical_and(attention_mask[:, :-1], attention_mask[:, 1:]) + + assert torch.count_nonzero(loss[~next_token_mask]) == 0 + + +def test_forward_loss_is_finite_with_left_padding() -> None: + bridge = _bridge() + tokens = torch.tensor( + [ + [0, 0, 0, 1, 2, 3], + [4, 5, 6, 7, 8, 9], + ] + ) + attention_mask = torch.tensor( + [ + [0, 0, 0, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + ] + ) + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + + logits = bridge( + tokens, + attention_mask=attention_mask, + position_ids=position_ids, + return_type="logits", + ) + loss = bridge( + tokens, + attention_mask=attention_mask, + position_ids=position_ids, + return_type="loss", + ) + + assert torch.isfinite(logits).all() + assert torch.isfinite(loss) + + +@pytest.mark.parametrize("mask_kind", ["bool", "additive"]) +@pytest.mark.parametrize("mask_layout", ["key_only", "causal"]) +def test_forward_loss_accepts_equivalent_4d_attention_mask( + mask_kind: str, mask_layout: str +) -> None: + bridge = _bridge() + tokens = torch.tensor( + [ + [1, 2, 3, 0, 0, 0], + [4, 5, 6, 7, 8, 9], + ] + ) + attention_mask = torch.tensor( + [ + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 1, 1, 1], + ] + ) + blocked = ~attention_mask.bool()[:, None, None, :] + if mask_layout == "causal": + blocked = blocked | torch.ones(6, 6, dtype=torch.bool).triu(1)[None, None] + attention_mask_4d = blocked if mask_kind == "bool" else blocked.float() * -10_000.0 + + logits_2d, loss_2d = bridge( + tokens, + attention_mask=attention_mask, + return_type="both", + ) + logits_4d, loss_4d = bridge( + tokens, + attention_mask=attention_mask_4d, + return_type="both", + ) + + torch.testing.assert_close(logits_4d, logits_2d, rtol=0, atol=0) + torch.testing.assert_close(loss_4d, loss_2d) + torch.testing.assert_close(loss_4d, _manual_masked_loss(logits_4d, tokens, attention_mask)) + + +def test_loss_fn_reduces_rectangular_cached_4d_attention_mask() -> None: + bridge = _bridge() + tokens = torch.tensor([[4, 5]]) + logits = torch.zeros(1, 2, 32) + logits[0, 0, 5] = 2.0 + cache_and_new_mask = torch.tensor([[0, 1, 1, 1, 1, 1]]) + key_blocked = ~cache_and_new_mask.bool()[:, None, None, :] + query_positions = torch.tensor([4, 5]) + causal = torch.arange(6)[None, None, None, :] > query_positions[None, None, :, None] + attention_mask_4d = key_blocked | causal + + loss = bridge.loss_fn( + logits, + tokens, + attention_mask=attention_mask_4d, + per_token=True, + ) + + expected = F.cross_entropy(logits[:, 0], tokens[:, 1]) + torch.testing.assert_close(loss, expected.reshape(1, 1)) + assert loss.shape == (1, 1) diff --git a/tests/unit/model_bridge/test_n_params_total.py b/tests/unit/model_bridge/test_n_params_total.py new file mode 100644 index 0000000000..d34142efda --- /dev/null +++ b/tests/unit/model_bridge/test_n_params_total.py @@ -0,0 +1,27 @@ +"""Tests for ``TransformerBridge.n_params_total`` on real model layouts.""" + +import pytest +import torch +from transformers import AutoModelForCausalLM + +from transformer_lens.model_bridge import TransformerBridge + + +@pytest.mark.parametrize("model_name", ["gpt2", "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5"]) +def test_n_params_total_matches_uninstrumented_model(model_name: str) -> None: + hf_model = AutoModelForCausalLM.from_pretrained( + model_name, dtype=torch.float32, attn_implementation="eager" + ) + expected = sum(parameter.numel() for parameter in hf_model.parameters()) + bridge = TransformerBridge.boot_transformers(model_name, hf_model=hf_model) + bridge.enable_compatibility_mode() + assert bridge.n_params_total == expected + + if "Qwen2" in model_name: + tl_parameters = bridge.tl_parameters() + for name in ("W_K", "W_V"): + weight = tl_parameters[f"blocks.0.attn.{name}"] + assert weight.shape[0] == bridge.cfg.n_heads + assert torch.count_nonzero(weight) + assert not torch.count_nonzero(tl_parameters["pos_embed.W_pos"]) + assert bridge.n_params_total < sum(p.numel() for p in tl_parameters.values()) diff --git a/tests/unit/model_bridge/test_position_ids_injection_gate.py b/tests/unit/model_bridge/test_position_ids_injection_gate.py new file mode 100644 index 0000000000..68d5a8f8b0 --- /dev/null +++ b/tests/unit/model_bridge/test_position_ids_injection_gate.py @@ -0,0 +1,160 @@ +"""Unit tests for the target gate on mask-derived ``position_ids`` injection. + +``TransformerBridge.forward`` derives ``position_ids`` from ``attention_mask`` +so left-padded input gets the right absolute positions (see #1609). That kwarg +is only safe for models that both accept it and do not derive positions +themselves, so the injection is gated the same way ``output_attentions`` is in +``run_with_cache``. The gate is exercised directly here with stand-in modules: +the models it exists to protect (fixed-signature remote code, mRoPE) are either +never loaded in CI or belong to the integration tier. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any, Optional + +import torch +import torch.nn as nn + +from transformer_lens.model_bridge import TransformerBridge + +# Unbound so it can run against a stand-in that owns only ``original_model``; +# the gate reads nothing else off the bridge. +gate = TransformerBridge._accepts_derived_position_ids + + +def _bridge_over(model: Optional[nn.Module]) -> Any: + return SimpleNamespace(original_model=model) + + +class _FixedSignature(nn.Module): + """Mirrors ``LLaDAModelLM.forward``: no ``position_ids``, no ``**kwargs``.""" + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + use_cache: bool = False, + ) -> torch.Tensor: + return input_ids + + +class _AcceptsPositionIds(nn.Module): + def forward( + self, + input_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return input_ids + + +class _AcceptsKwargs(nn.Module): + def forward(self, input_ids: torch.Tensor, **kwargs: Any) -> torch.Tensor: + return input_ids + + +class _MaskConsumingEmbedding(nn.Embedding): + """Mirrors ``OPTLearnedPositionalEmbedding``: positions come from the mask.""" + + def forward( # type: ignore[override] + self, + attention_mask: torch.Tensor, + past_key_values_length: int = 0, + position_ids: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if position_ids is None: + position_ids = (attention_mask.cumsum(1) * attention_mask - 1).long() + return super().forward(position_ids) + + +class _OwnsPositions(_AcceptsPositionIds): + """mRoPE models compute a 3-D index here, but only while position_ids is None.""" + + def get_rope_index(self, *args: Any, **kwargs: Any) -> None: + return None + + +class TestSignatureGate: + def test_refuses_model_that_cannot_take_position_ids(self) -> None: + """Injecting into a fixed-signature remote-code forward raises TypeError + where the model previously returned logits.""" + assert gate(_bridge_over(_FixedSignature())) is False + + def test_allows_explicit_position_ids_parameter(self) -> None: + assert gate(_bridge_over(_AcceptsPositionIds())) is True + + def test_allows_var_keyword_forward(self) -> None: + """**kwargs forwards pass the kwarg through to the inner model.""" + assert gate(_bridge_over(_AcceptsKwargs())) is True + + +class TestOwnsPositionsGate: + def test_refuses_model_defining_get_rope_index(self) -> None: + assert gate(_bridge_over(_OwnsPositions())) is False + + def test_refuses_wrapper_whose_inner_model_owns_positions(self) -> None: + """get_rope_index lives on the inner text model, while original_model is + usually the ForConditionalGeneration wrapper around it.""" + wrapper = _AcceptsPositionIds() + wrapper.model = _OwnsPositions() + assert gate(_bridge_over(wrapper)) is False + + def test_refuses_wrapper_whose_language_model_owns_positions(self) -> None: + wrapper = _AcceptsPositionIds() + wrapper.language_model = _OwnsPositions() + assert gate(_bridge_over(wrapper)) is False + + def test_refuses_mrope_section_in_config(self) -> None: + """Config-level backstop: the section list is what makes positions 3-D.""" + model = _AcceptsPositionIds() + model.config = SimpleNamespace( # type: ignore[assignment] + rope_scaling={"mrope_section": [1, 1, 2], "rope_type": "default"} + ) + assert gate(_bridge_over(model)) is False + + def test_refuses_mrope_section_in_text_config(self) -> None: + """Multimodal configs nest the text model's rope_scaling one level down.""" + model = _AcceptsPositionIds() + model.config = SimpleNamespace( # type: ignore[assignment] + rope_scaling=None, + text_config=SimpleNamespace(rope_scaling={"mrope_section": [1, 1, 2]}), + ) + assert gate(_bridge_over(model)) is False + + def test_refuses_mask_consuming_positional_embedding(self) -> None: + """OPT's OPTLearnedPositionalEmbedding takes the mask and derives its own + positions, including its own convention for the padded slots.""" + model = _AcceptsPositionIds() + model.embed_positions = _MaskConsumingEmbedding(8, 4) + assert gate(_bridge_over(model)) is False + + def test_ordinary_embeddings_do_not_trip_the_scan(self) -> None: + """nn.Embedding takes only indices, so the common case stays injectable.""" + model = _AcceptsPositionIds() + model.wte = nn.Embedding(8, 4) + model.wpe = nn.Embedding(8, 4) + assert gate(_bridge_over(model)) is True + + def test_plain_rope_scaling_is_not_treated_as_mrope(self) -> None: + """Only mrope_section means a multi-stream index; yarn/linear do not.""" + model = _AcceptsPositionIds() + model.config = SimpleNamespace( # type: ignore[assignment] + rope_scaling={"rope_type": "yarn", "factor": 8.0} + ) + assert gate(_bridge_over(model)) is True + + +class TestDriverAndCaching: + def test_refuses_bridge_without_a_local_module(self) -> None: + """A bridge with no wrapped module exposes nothing to introspect.""" + assert gate(_bridge_over(None)) is False + + def test_recomputes_when_the_underlying_model_is_swapped(self) -> None: + """Weight processing replaces original_model, so a cached verdict keyed on + the old module must not survive.""" + bridge = _bridge_over(_AcceptsPositionIds()) + assert gate(bridge) is True + bridge.original_model = _FixedSignature() + assert gate(bridge) is False diff --git a/tests/unit/model_bridge/test_reset_hooks_registry.py b/tests/unit/model_bridge/test_reset_hooks_registry.py new file mode 100644 index 0000000000..ff15660c3f --- /dev/null +++ b/tests/unit/model_bridge/test_reset_hooks_registry.py @@ -0,0 +1,63 @@ +"""reset_hooks must clear every registered hook point, not just component-owned ones.""" + +from __future__ import annotations + +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge + + +def _bridge() -> TransformerBridge: + cfg = TransformerBridgeConfig( + d_model=32, + d_head=16, + n_heads=2, + n_layers=1, + n_ctx=8, + d_vocab=16, + d_mlp=64, + act_fn="gelu", + normalization_type="LN", + seed=0, + ) + return TransformerBridge.boot_native(cfg) + + +def _points_with_hooks(bridge: TransformerBridge) -> list[str]: + return [name for name, hp in bridge._hook_registry.items() if hp.fwd_hooks or hp.bwd_hooks] + + +def test_reset_hooks_clears_every_registry_point() -> None: + bridge = _bridge() + for hp in bridge._hook_registry.values(): + hp.add_hook(lambda tensor, hook: tensor) + assert _points_with_hooks(bridge), "sanity: hooks were added" + + bridge.reset_hooks() + + leaked = _points_with_hooks(bridge) + assert leaked == [], f"reset_hooks leaked hooks on registry points: {leaked[:5]}" + + +def test_reset_hooks_permanent_semantics() -> None: + bridge = _bridge() + name, hp = next(iter(bridge._hook_registry.items())) + hp.add_hook(lambda tensor, hook: tensor, is_permanent=True) + + bridge.reset_hooks() + assert hp.fwd_hooks, "permanent hook must survive a default reset" + + bridge.reset_hooks(including_permanent=True) + assert not hp.fwd_hooks, "including_permanent=True must clear permanent hooks" + + +def test_reset_hooks_still_functional_after_forward() -> None: + bridge = _bridge() + seen: list[str] = [] + for name, hp in bridge._hook_registry.items(): + hp.add_hook(lambda tensor, hook: seen.append(hook.name)) + bridge.reset_hooks() + with torch.no_grad(): + bridge(torch.randint(0, bridge.cfg.d_vocab, (1, 4))) + assert seen == [], "cleared hooks must not fire on forward" diff --git a/tests/unit/model_bridge/test_state_dict_composition.py b/tests/unit/model_bridge/test_state_dict_composition.py new file mode 100644 index 0000000000..a8f5a7401b --- /dev/null +++ b/tests/unit/model_bridge/test_state_dict_composition.py @@ -0,0 +1,171 @@ +"""Regression tests for recursive TransformerBridge checkpoint composition (#1655).""" + +from __future__ import annotations + +from collections import OrderedDict +from types import SimpleNamespace + +import pytest +import torch +from transformers import GPT2Config, GPT2LMHeadModel + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.generalized_components import ( + JointGateUpMLPBridge, + JointQKVAttentionBridge, + LinearBridge, +) +from transformer_lens.model_bridge.sources import build_bridge_from_module + + +def _native_bridge() -> TransformerBridge: + cfg = TransformerBridgeConfig( + d_model=32, + d_head=16, + n_heads=2, + n_layers=2, + n_ctx=8, + d_vocab=16, + d_mlp=64, + act_fn="gelu", + normalization_type="LN", + seed=0, + ) + return TransformerBridge.boot_native(cfg) + + +def _parent_with_bridge(bridge: TransformerBridge) -> torch.nn.Module: + parent = torch.nn.Module() + parent.add_module("bridge", bridge) + return parent + + +def test_state_dict_with_destination_and_prefix_uses_recursive_semantics() -> None: + bridge = _native_bridge() + sentinel = torch.tensor(1) + destination: OrderedDict[str, torch.Tensor] = OrderedDict({"sentinel": sentinel}) + + returned = bridge.state_dict(destination=destination, prefix="nested.bridge.") + + assert returned is destination + assert destination["sentinel"] is sentinel + recursive_keys = set(destination) - {"sentinel"} + assert recursive_keys + assert all(key.startswith("nested.bridge.") for key in recursive_keys) + + +def test_parent_state_dict_strict_round_trip() -> None: + parent = _parent_with_bridge(_native_bridge()) + checkpoint = {key: value.clone() for key, value in parent.state_dict().items()} + + with torch.no_grad(): + for parameter in parent.parameters(): + parameter.zero_() + + result = parent.load_state_dict(checkpoint, strict=True) + + assert result.missing_keys == [] + assert result.unexpected_keys == [] + reloaded = parent.state_dict() + for key, value in checkpoint.items(): + assert torch.equal(reloaded[key], value), f"{key} did not round-trip" + + +def test_parent_registration_is_stable_across_first_forward() -> None: + bridge = _native_bridge() + parent = _parent_with_bridge(bridge) + + for block in bridge.blocks: + assert block.attn._ln1_module is block.ln1.original_component + + keys_before = tuple(parent.state_dict()) + assert not any("._ln1_module." in key for key in keys_before) + with torch.no_grad(): + bridge(torch.randint(0, bridge.cfg.d_vocab, (1, 4))) + keys_after = tuple(parent.state_dict()) + + assert keys_after == keys_before + assert not any("._ln1_module." in key for key in keys_after) + + +def test_nested_joint_qkv_bridge_strict_round_trip() -> None: + cfg = GPT2Config( + vocab_size=32, + n_positions=16, + n_embd=16, + n_layer=1, + n_head=2, + n_inner=32, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + ) + bridge = build_bridge_from_module( + GPT2LMHeadModel(cfg), + architecture="GPT2LMHeadModel", + hf_config=cfg, + ) + parent = _parent_with_bridge(bridge) + + checkpoint = {key: value.clone() for key, value in parent.state_dict().items()} + assert not any(".qkv." in key for key in checkpoint) + with torch.no_grad(): + for parameter in parent.parameters(): + parameter.zero_() + + result = parent.load_state_dict(checkpoint, strict=True) + + assert result.missing_keys == [] + assert result.unexpected_keys == [] + reloaded = parent.state_dict() + for key, value in checkpoint.items(): + assert torch.equal(reloaded[key], value), f"{key} did not round-trip" + + +def _filtered_joint_component(kind: str) -> torch.nn.Module: + filtered_child = LinearBridge(name=kind) + filtered_child.set_original_component(torch.nn.Linear(4, 8)) + cfg = SimpleNamespace(n_heads=2, d_head=4) + + if kind == "qkv": + qkv_component = JointQKVAttentionBridge( + name="attn", + config=cfg, + submodules={"qkv": filtered_child}, + ) + for child_name in ("q", "k", "v"): + getattr(qkv_component, child_name).set_original_component(torch.nn.Linear(4, 4)) + return qkv_component + gate_up_component = JointGateUpMLPBridge( + name="mlp", + config=cfg, + submodules={"gate_up": filtered_child}, + ) + gate_up_component.add_module("gate_up", filtered_child) + gate_up_component.gate.set_original_component(torch.nn.Linear(4, 4)) + getattr(gate_up_component, "in").set_original_component(torch.nn.Linear(4, 4)) + return gate_up_component + + +@pytest.mark.parametrize("filtered_child_name", ["qkv", "gate_up"]) +def test_filtered_joint_component_strict_round_trip(filtered_child_name: str) -> None: + component = _filtered_joint_component(filtered_child_name) + filtered_child = component.get_submodule(filtered_child_name) + checkpoint = {key: value.clone() for key, value in component.state_dict().items()} + + assert checkpoint + assert not any(key.startswith(f"{filtered_child_name}.") for key in checkpoint) + with torch.no_grad(): + for parameter in component.parameters(): + parameter.zero_() + + result = component.load_state_dict(checkpoint, strict=True) + + assert result.missing_keys == [] + assert result.unexpected_keys == [] + reloaded = component.state_dict() + for key, value in checkpoint.items(): + assert torch.equal(reloaded[key], value), f"{key} did not round-trip" + for parameter in filtered_child.parameters(): + assert torch.count_nonzero(parameter) == 0 diff --git a/tests/unit/model_bridge/test_state_dict_round_trip.py b/tests/unit/model_bridge/test_state_dict_round_trip.py new file mode 100644 index 0000000000..b0d707d89f --- /dev/null +++ b/tests/unit/model_bridge/test_state_dict_round_trip.py @@ -0,0 +1,196 @@ +"""Regression tests for TransformerBridge.state_dict()/load_state_dict() round-tripping (#1587). + +state_dict() emits TL-renamed keys (e.g. "blocks.0.attn.q.weight"), but +load_state_dict() only matched raw native parameter names, so a +state_dict() -> load_state_dict() round trip silently loaded nothing and +strict=True was silently downgraded to strict=False. +""" +from __future__ import annotations + +import pytest +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge + + +def _native_cfg(**overrides) -> TransformerBridgeConfig: + base = dict( + d_model=32, + d_head=16, + n_heads=2, + n_layers=2, + n_ctx=8, + d_vocab=16, + d_mlp=64, + act_fn="gelu", + normalization_type="LN", + seed=0, + ) + base.update(overrides) + return TransformerBridgeConfig(**base) + + +def test_native_round_trip_overwrites_params_not_a_noop(): + bridge = TransformerBridge.boot_native(_native_cfg()) + + sd = {k: v.clone() for k, v in bridge.state_dict().items()} + assert sd, "state_dict() returned no TL-format keys" + + with torch.no_grad(): + for p in bridge.parameters(): + p.zero_() + assert all((p == 0).all() for p in bridge.parameters()) + + bridge.load_state_dict(sd, strict=True) + + # Compare against the snapshot directly rather than asserting "not all + # zero" - LayerNorm bias legitimately initializes to all-zero, so that + # check would pass even for a param that never got reloaded. + reloaded = bridge.state_dict() + for key, value in sd.items(): + assert torch.equal(reloaded[key], value), f"{key} did not round-trip" + + +def test_native_strict_true_raises_on_missing_key(): + bridge = TransformerBridge.boot_native(_native_cfg()) + sd = bridge.state_dict() + incomplete = dict(sd) + incomplete.pop(next(iter(incomplete))) + + with pytest.raises(RuntimeError, match="Missing key"): + bridge.load_state_dict(incomplete, strict=True) + + +def test_native_strict_true_raises_on_unexpected_key(): + bridge = TransformerBridge.boot_native(_native_cfg()) + sd = dict(bridge.state_dict()) + sd["totally.bogus.key"] = torch.zeros(1) + + with pytest.raises(RuntimeError, match="Unexpected key"): + bridge.load_state_dict(sd, strict=True) + + +def test_native_strict_false_does_not_raise_on_partial_dict(): + bridge = TransformerBridge.boot_native(_native_cfg()) + sd = bridge.state_dict() + first_key = next(iter(sd)) + partial = {first_key: sd[first_key]} + + result = bridge.load_state_dict(partial, strict=False) + assert result.unexpected_keys == [] + assert len(result.missing_keys) > 0 + + +def test_native_raw_keys_still_load_tracr_style(): + """boot_native's own raw parameter names must keep loading directly, + mirroring tracr's make_tracr_transformer_bridge_state_dict compatibility + contract (utilities/tracr.py).""" + bridge = TransformerBridge.boot_native(_native_cfg()) + raw_sd = {k: v.clone() for k, v in bridge.original_model.state_dict().items()} + + with torch.no_grad(): + for p in bridge.parameters(): + p.zero_() + + bridge.load_state_dict(raw_sd, strict=True) + + reloaded_raw = bridge.original_model.state_dict() + for key, value in raw_sd.items(): + assert torch.equal(reloaded_raw[key], value), f"{key} did not round-trip" + + +def test_native_clean_key_dict_with_partial_aliases_does_not_raise_strict(): + """A complete raw-HF-format-style state dict (clean keys, _original_component + stripped) writes only one alias per shared-storage TL key -- boot_native's own + wrapping produces this aliasing internally (e.g. "layers.0.ln1.weight" is + reachable via two different _original_component paths onto the same + Parameter), not just gpt2's c_attn split. Since aliases of the same tensor + share storage, writing any one of them is sufficient; strict=True must not + report the other, unwritten aliases as missing.""" + bridge = TransformerBridge.boot_native(_native_cfg()) + + raw_sd = bridge.original_model.state_dict() + clean_to_actuals: dict[str, list[str]] = {} + for actual_key in raw_sd: + if actual_key != "_original_component": + clean_to_actuals.setdefault(actual_key.replace("._original_component", ""), []).append( + actual_key + ) + assert any(len(keys) > 1 for keys in clean_to_actuals.values()), ( + "fixture assumption broken: expected boot_native to have some " + "clean key reachable through more than one actual path" + ) + + # One representative actual key's value per clean key, same shape as a + # real raw-HF-format checkpoint (no duplicate paths for the same param). + clean_sd = {clean_key: raw_sd[keys[0]].clone() for clean_key, keys in clean_to_actuals.items()} + + with torch.no_grad(): + for p in bridge.parameters(): + p.zero_() + + result = bridge.load_state_dict(clean_sd, strict=True) + assert result.missing_keys == [] + assert result.unexpected_keys == [] + + reloaded_raw = bridge.original_model.state_dict() + for clean_key, actual_keys in clean_to_actuals.items(): + value = clean_sd[clean_key] + for actual_key in actual_keys: + assert torch.equal( + reloaded_raw[actual_key], value + ), f"{actual_key} (alias of {clean_key}) did not round-trip" + + +@pytest.mark.slow +def test_boot_transformers_round_trip_matches_forward_pass(): + """GPT-2's Conv1D-combined attention makes the bridge's q/k/v components + storage-sharing VIEWS into c_attn, not independent parameters - so this is + the case that actually exercises convert_hf_key_to_tl_key's HF-name + renaming, not just identity passthrough like boot_native does.""" + bridge = TransformerBridge.boot_transformers("gpt2", device="cpu") + bridge.eval() + + torch.manual_seed(0) + tokens = torch.randint(0, 1000, (1, 8)) + with torch.no_grad(): + logits_before = bridge(tokens).clone() + + sd = {k: v.clone() for k, v in bridge.state_dict().items()} + + with torch.no_grad(): + for p in bridge.parameters(): + p.zero_() + + bridge.load_state_dict(sd, strict=True) + + with torch.no_grad(): + logits_after = bridge(tokens).clone() + + max_diff = (logits_before - logits_after).abs().max().item() + assert torch.allclose( + logits_before, logits_after, atol=1e-5 + ), f"round trip did not restore forward-pass output: max diff={max_diff:.3e}" + + +@pytest.mark.slow +def test_boot_transformers_clean_key_dict_does_not_raise_strict(): + """Reported review case on real gpt2: a complete raw-HF-format-style state + dict (clean keys) writes only one alias per shared-storage TL key, since + gpt2's split q/k/v are views into c_attn reachable via multiple actual + paths. strict=True previously raised ~337 false "missing key" errors even + though the load fully restores the forward pass.""" + bridge = TransformerBridge.boot_transformers("gpt2", device="cpu") + + raw_sd = bridge.original_model.state_dict() + clean_sd = { + actual_key.replace("._original_component", ""): value.clone() + for actual_key, value in raw_sd.items() + if actual_key != "_original_component" + } + assert len(clean_sd) < len(raw_sd), "fixture assumption broken: expected some aliasing on gpt2" + + result = bridge.load_state_dict(clean_sd, strict=True) + assert result.missing_keys == [] + assert result.unexpected_keys == [] diff --git a/tests/unit/model_bridge/test_tokenizer_reassignment.py b/tests/unit/model_bridge/test_tokenizer_reassignment.py new file mode 100644 index 0000000000..b32d9f6140 --- /dev/null +++ b/tests/unit/model_bridge/test_tokenizer_reassignment.py @@ -0,0 +1,128 @@ +"""Tests for tokenizer reassignment wiring.""" + +import pytest +from transformers import AutoTokenizer + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.bridge import TransformerBridge + + +class MockAdapter(ArchitectureAdapter): + """Minimal adapter for testing.""" + + def __init__(self, cfg: TransformerBridgeConfig): + super().__init__(cfg) + self.component_mapping = {"embed": None} + + +def _bare_bridge(adapter, tokenizer): + """Bridge stand-in replaying __init__'s tokenizer sequence without a model. + + Full construction needs a wrapped model; the wiring under test only reads + adapter/cfg, so build a bare instance the way + test_generation_benchmark_mechanics does. + """ + import torch.nn as nn + + bridge = object.__new__(TransformerBridge) + nn.Module.__init__(bridge) + bridge.adapter = adapter + bridge.cfg = adapter.cfg + bridge._tokenizer = None + if tokenizer is not None: + bridge.tokenizer = tokenizer + if bridge.cfg.d_vocab_out == -1: + bridge.cfg.d_vocab_out = bridge.cfg.d_vocab + return bridge + + +class TestTokenizerReassignment: + """Test that tokenizer reassignment re-runs wiring logic.""" + + @pytest.fixture + def base_cfg(self) -> TransformerBridgeConfig: + return TransformerBridgeConfig( + d_model=768, + d_head=64, + n_layers=12, + n_ctx=1024, + d_vocab=-1, # Will be inferred from tokenizer + d_mlp=3072, + n_heads=12, + ) + + @pytest.fixture + def gpt2_tokenizer(self): + """GPT-2 tokenizer (does not prepend BOS by default).""" + return AutoTokenizer.from_pretrained("gpt2") + + @pytest.fixture + def llama_style_tokenizer(self): + """A tokenizer that prepends BOS (using gpt-neox as example).""" + tok = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b") + return tok + + def test_initial_tokenizer_sets_d_vocab(self, base_cfg, gpt2_tokenizer): + """Test that initial tokenizer assignment sets d_vocab.""" + adapter = MockAdapter(base_cfg) + bridge = _bare_bridge(adapter, gpt2_tokenizer) + + # GPT-2 vocab size is 50257 + assert bridge.cfg.d_vocab == 50257 + assert bridge.cfg.d_vocab_out == 50257 + + def test_reassignment_updates_d_vocab(self, base_cfg, gpt2_tokenizer, llama_style_tokenizer): + """Test that reassigning tokenizer updates d_vocab.""" + adapter = MockAdapter(base_cfg) + bridge = _bare_bridge(adapter, gpt2_tokenizer) + + old_d_vocab = bridge.cfg.d_vocab + + bridge.tokenizer = llama_style_tokenizer + + # GPT-NeoX has a different vocab size than GPT-2 + assert bridge.cfg.d_vocab != old_d_vocab + assert bridge.cfg.d_vocab_out == bridge.cfg.d_vocab + + def test_reassignment_updates_bos_flag(self, base_cfg, gpt2_tokenizer, llama_style_tokenizer): + """Test that reassigning tokenizer updates tokenizer_prepends_bos.""" + adapter = MockAdapter(base_cfg) + bridge = _bare_bridge(adapter, gpt2_tokenizer) + + gpt2_bos = bridge.cfg.tokenizer_prepends_bos + + bridge.tokenizer = llama_style_tokenizer + + neox_bos = bridge.cfg.tokenizer_prepends_bos + # The flags should be properly detected (actual values depend on tokenizer behavior) + assert isinstance(neox_bos, bool) + + def test_reassignment_to_none_preserves_config(self, base_cfg, gpt2_tokenizer): + """Test that setting tokenizer to None doesn't crash.""" + adapter = MockAdapter(base_cfg) + bridge = _bare_bridge(adapter, gpt2_tokenizer) + + old_d_vocab = bridge.cfg.d_vocab + + bridge.tokenizer = None + + assert bridge.tokenizer is None + assert bridge.cfg.d_vocab == old_d_vocab # Preserved from previous tokenizer + + def test_tokenizer_property_returns_tokenizer(self, base_cfg, gpt2_tokenizer): + """Test that the tokenizer property returns the stored tokenizer.""" + adapter = MockAdapter(base_cfg) + bridge = _bare_bridge(adapter, gpt2_tokenizer) + + assert bridge.tokenizer is not None + assert hasattr(bridge.tokenizer, "encode") + + def test_no_tokenizer_at_init(self, base_cfg): + """Test that bridge can be created without tokenizer.""" + base_cfg.d_vocab = 50257 # Set explicitly since no tokenizer + adapter = MockAdapter(base_cfg) + bridge = _bare_bridge(adapter, None) + + assert bridge.tokenizer is None + assert bridge.cfg.d_vocab == 50257 diff --git a/tests/unit/test_activation_cache.py b/tests/unit/test_activation_cache.py new file mode 100644 index 0000000000..9debaf4a30 --- /dev/null +++ b/tests/unit/test_activation_cache.py @@ -0,0 +1,63 @@ +import pytest +import torch + +from transformer_lens import ActivationCache +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge + + +@pytest.fixture(scope="module", params=["LN", "RMS"]) +def activation_cache(request: pytest.FixtureRequest) -> ActivationCache: + cfg = TransformerBridgeConfig( + n_layers=2, + d_model=16, + n_ctx=8, + d_head=4, + n_heads=4, + d_vocab=32, + act_fn="gelu", + normalization_type=request.param, + ) + with torch.random.fork_rng(devices=[]): + torch.manual_seed(0) + model = TransformerBridge.boot_native(cfg) + tokens = torch.tensor( + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + ] + ) + _, cache = model.run_with_cache(tokens) + return cache + + +@pytest.mark.parametrize("layer", [1, -1], ids=["cached-scale", "recomputed-final-ln"]) +@pytest.mark.parametrize( + "pos_slice", [None, (1, 3), -1], ids=["all-positions", "position-slice", "scalar-position"] +) +@pytest.mark.parametrize("apply_ln", [False, True], ids=["raw", "normalized"]) +def test_batchless_accumulated_resid_matches_batched_row( + activation_cache: ActivationCache, + layer: int, + pos_slice: tuple[int, int] | int | None, + apply_ln: bool, +) -> None: + batch_index = 1 + batched = activation_cache.accumulated_resid( + layer=layer, + pos_slice=pos_slice, + apply_ln=apply_ln, + ) + + batchless_cache = activation_cache.apply_slice_to_batch_dim(batch_index) + assert not batchless_cache.has_batch_dim + batchless = batchless_cache.accumulated_resid( + layer=layer, + pos_slice=pos_slice, + apply_ln=apply_ln, + ) + + expected = batched[:, batch_index] + assert batchless.shape == expected.shape + torch.testing.assert_close(batchless, expected) diff --git a/tests/unit/test_activation_cache_batch_dim.py b/tests/unit/test_activation_cache_batch_dim.py new file mode 100644 index 0000000000..d5521761f9 --- /dev/null +++ b/tests/unit/test_activation_cache_batch_dim.py @@ -0,0 +1,69 @@ +"""Batch-dim handling for caches holding broadcast / position-indexed entries. + +T5-style caches mix genuinely batched activations with entries whose leading dim +is not the batch: broadcast entries (leading 1) and position-indexed entries +(leading dim = seq len, e.g. relative position bias). remove_batch_dim and +apply_slice_to_batch_dim must not corrupt those. +""" + +from __future__ import annotations + +import pytest +import torch + +from transformer_lens.ActivationCache import ActivationCache + + +def _mixed_batch1_cache() -> ActivationCache: + return ActivationCache( + { + "blocks.0.hook_resid_pre": torch.randn(1, 5, 4), + "blocks.0.hook_pattern": torch.randn(1, 2, 5, 5), + # Position-indexed entry: leading dim is seq len, not batch. + "blocks.0.attn.hook_rel_pos_bias": torch.randn(5, 5), + }, + model=None, + has_batch_dim=True, + ) + + +def test_remove_batch_dim_leaves_position_indexed_entries_alone() -> None: + cache = _mixed_batch1_cache() + bias_before = cache["blocks.0.attn.hook_rel_pos_bias"].clone() + + cache.remove_batch_dim() + + assert cache["blocks.0.hook_resid_pre"].shape == (5, 4) + assert cache["blocks.0.hook_pattern"].shape == (2, 5, 5) + assert torch.equal(cache["blocks.0.attn.hook_rel_pos_bias"], bias_before) + + +def test_remove_batch_dim_refuses_true_batch_gt_1_despite_broadcast_entry() -> None: + cache = ActivationCache( + { + "blocks.0.hook_resid_pre": torch.randn(2, 5, 4), + "blocks.0.hook_mlp_out": torch.randn(2, 5, 4), + "hook_pos_indices": torch.randn(1, 5), + }, + model=None, + has_batch_dim=True, + ) + with pytest.raises(AssertionError, match="batch size 2"): + cache.remove_batch_dim() + + +def test_apply_slice_to_batch_dim_skips_broadcast_entries() -> None: + cache = ActivationCache( + { + "blocks.0.hook_resid_pre": torch.randn(3, 5, 4), + "blocks.0.hook_mlp_out": torch.randn(3, 5, 4), + "hook_pos_indices": torch.randn(1, 5), + }, + model=None, + has_batch_dim=True, + ) + sliced = cache.apply_slice_to_batch_dim((1, 3)) + + assert sliced["blocks.0.hook_resid_pre"].shape == (2, 5, 4) + assert sliced["blocks.0.hook_mlp_out"].shape == (2, 5, 4) + assert sliced["hook_pos_indices"].shape == (1, 5) diff --git a/tests/unit/test_gradient_mismatch_grading.py b/tests/unit/test_gradient_mismatch_grading.py new file mode 100644 index 0000000000..c7cabae231 --- /dev/null +++ b/tests/unit/test_gradient_mismatch_grading.py @@ -0,0 +1,130 @@ +"""Gradient mismatches are graded on scale-aware statistics, not worst-case elements. + +Registering backward hooks forces normalization off HF's native autograd onto the +python-norm path, which shifts results at float-rounding scale. Elementwise +`allclose` graded that shift as failure: one element of 55,296 crossing the +tolerance scored the same as a real divergence. The band accepts a mismatch only +when it is both diffuse (rel_l2) and localized to a handful of elements (COUNT, +not fraction — detection guarantees count >= 1, so a fractional guard was +unsatisfiable below 10,000 elements and re-created the false failure on +gemma-3-270m's 6,912-element MQA hook_rot_k). +""" + +import pytest + +from transformer_lens.benchmarks.backward_gradients import ( + OVER_TOLERANCE_MAX_ELEMENTS, + REL_L2_TOLERANCE, + gradient_mismatch_is_numerical_noise, +) + +# Measured bridge-vs-HT fallback noise. Only mismatches the detection gate records +# can reach the classifier, so every fixture has count >= 1 (frac-zero rows from +# the original study never reach it and prove nothing). +NOISE = [ + ("Qwen3-0.6B rot_q [1,27,16,128]", 1.714e-05, 1), + ("gemma-3-270m rot_k [1,27,1,256] (MQA, 6912 elements)", 5.0e-05, 1), +] +# Injected bugs of known severity on the Qwen3 rot_q tensor (55,296 elements). +BUGS = [ + ("one head scaled 1%", 2.97e-03, 758), + ("uniform scale 0.1%", 1.00e-03, 432), + ("60 elements +50", 1.09e-02, 60), +] + + +@pytest.mark.parametrize("label,rel_l2,count", NOISE) +def test_fallback_noise_is_accepted(label: str, rel_l2: float, count: int) -> None: + assert gradient_mismatch_is_numerical_noise(rel_l2, count), label + + +@pytest.mark.parametrize("label,rel_l2,count", BUGS) +def test_real_divergence_is_rejected(label: str, rel_l2: float, count: int) -> None: + assert not gradient_mismatch_is_numerical_noise(rel_l2, count), label + + +def test_thresholds_keep_a_margin_on_both_dimensions() -> None: + """Both guards must sit clear of both populations, not graze either.""" + worst_noise_rel = max(rel for _, rel, _ in NOISE) + best_bug_rel = min(rel for _, rel, _ in BUGS) + assert worst_noise_rel * 2 <= REL_L2_TOLERANCE, worst_noise_rel + assert best_bug_rel >= REL_L2_TOLERANCE * 5, best_bug_rel + worst_noise_count = max(count for _, _, count in NOISE) + best_bug_count = min(count for _, _, count in BUGS) + assert worst_noise_count * 3 <= OVER_TOLERANCE_MAX_ELEMENTS + 1, worst_noise_count + assert best_bug_count >= OVER_TOLERANCE_MAX_ELEMENTS * 10, best_bug_count + + +def test_localized_divergence_is_rejected_even_when_diffuse_error_is_small() -> None: + """The count is the guard: a concentrated error rel_l2 would dilute still fails.""" + assert not gradient_mismatch_is_numerical_noise(REL_L2_TOLERANCE / 10, 60) + assert not gradient_mismatch_is_numerical_noise(1e-2, 1) + + +def test_zero_reference_divergence_is_rejected() -> None: + """A zero reference gradient with a nonzero bridge gradient records + rel_l2=inf in the benchmark body — maximal divergence, never noise.""" + assert not gradient_mismatch_is_numerical_noise(float("inf"), 1) + + +def test_boundaries_are_inclusive() -> None: + """Pin the <= contract on both dimensions.""" + assert gradient_mismatch_is_numerical_noise(REL_L2_TOLERANCE, OVER_TOLERANCE_MAX_ELEMENTS) + assert not gradient_mismatch_is_numerical_noise( + REL_L2_TOLERANCE * 1.01, OVER_TOLERANCE_MAX_ELEMENTS + ) + assert not gradient_mismatch_is_numerical_noise( + REL_L2_TOLERANCE, OVER_TOLERANCE_MAX_ELEMENTS + 1 + ) + + +class TestMismatchStatsHelper: + """The stats the classifier consumes, driven with synthetic tensors.""" + + def test_zero_reference_nonzero_bridge_is_infinite(self) -> None: + import torch + + from transformer_lens.benchmarks.backward_gradients import ( + gradient_mismatch_stats, + ) + + stats = gradient_mismatch_stats(torch.ones(100), torch.zeros(100), 0.2, 3e-4) + assert stats["rel_l2"] == float("inf") + assert not gradient_mismatch_is_numerical_noise(stats["rel_l2"], stats["over_count"]) + + def test_matching_zeros_agree(self) -> None: + import torch + + from transformer_lens.benchmarks.backward_gradients import ( + gradient_mismatch_stats, + ) + + stats = gradient_mismatch_stats(torch.zeros(100), torch.zeros(100), 0.2, 3e-4) + assert stats["rel_l2"] == 0.0 and stats["over_count"] == 0 + + def test_over_count_matches_detection_predicate(self) -> None: + import torch + + from transformer_lens.benchmarks.backward_gradients import ( + gradient_mismatch_stats, + ) + + ref = torch.full((6912,), 258.0) # gemma-3-270m rot_k scale + bridge = ref.clone() + bridge[0] += 0.5 # one element past atol + rtol*|ref| + stats = gradient_mismatch_stats(bridge, ref, 0.2, 3e-4) + assert stats["over_count"] == 1 + assert gradient_mismatch_is_numerical_noise(stats["rel_l2"], stats["over_count"]), stats + + +class TestFp32GradientPredicate: + def test_reduced_precision_needs_upcast(self) -> None: + import torch + + from transformer_lens.benchmarks.backward_gradients import needs_fp32_gradients + + assert needs_fp32_gradients(torch.bfloat16) + assert needs_fp32_gradients(torch.float16) + assert not needs_fp32_gradients(torch.float32) + assert not needs_fp32_gradients(torch.float64) + assert not needs_fp32_gradients(None) diff --git a/tests/unit/test_lm_utils.py b/tests/unit/test_lm_utils.py new file mode 100644 index 0000000000..ca3e6a8975 --- /dev/null +++ b/tests/unit/test_lm_utils.py @@ -0,0 +1,54 @@ +"""Unit tests for language-model loss and accuracy helpers.""" + +from __future__ import annotations + +import pytest +import torch +from beartype.roar import BeartypeCallHintParamViolation + +from transformer_lens.utilities.lm_utils import lm_accuracy, lm_cross_entropy_loss + + +def test_lm_cross_entropy_loss_rejects_mismatched_attention_mask() -> None: + logits = torch.zeros(1, 2, 3) + tokens = torch.tensor([[0, 1]]) + attention_mask = torch.ones(1, 5, dtype=torch.long) + + with pytest.raises( + (AssertionError, BeartypeCallHintParamViolation), + match="attention_mask|axis 'pos'", + ): + lm_cross_entropy_loss(logits, tokens, attention_mask) + + +def test_lm_cross_entropy_loss_masks_nan_transition() -> None: + logits = torch.tensor( + [ + [ + [torch.nan, torch.nan], + [0.0, 0.0], + [0.0, 0.0], + ] + ] + ) + tokens = torch.tensor([[0, 1, 0]]) + attention_mask = torch.tensor([[0, 1, 1]]) + + per_token = lm_cross_entropy_loss(logits, tokens, attention_mask, per_token=True) + scalar = lm_cross_entropy_loss(logits, tokens, attention_mask) + expected = torch.log(torch.tensor(2.0)) + + torch.testing.assert_close(per_token, torch.stack((expected.new_zeros(()), expected))[None]) + torch.testing.assert_close(scalar, expected) + assert torch.isfinite(per_token).all() + assert torch.isfinite(scalar) + + +def test_lm_accuracy_per_token_returns_bool_pos_minus_one() -> None: + logits = torch.zeros(2, 4, 3) + tokens = torch.tensor([[0, 1, 2, 0], [2, 1, 0, 2]]) + + accuracy = lm_accuracy(logits, tokens, per_token=True) + + assert accuracy.dtype is torch.bool + assert accuracy.shape == (2, 3) diff --git a/tests/unit/test_moe_fold_guard.py b/tests/unit/test_moe_fold_guard.py new file mode 100644 index 0000000000..f6afddf5c5 --- /dev/null +++ b/tests/unit/test_moe_fold_guard.py @@ -0,0 +1,104 @@ +"""MoE models fold their norms like dense models do. + +History: HT once switched MoE models to the gain-less *Pre norm while its +process step refused to fold the experts, silently dropping the gains entirely +(OLMoE sat 20.5 off HF in log-softmax, 0% argmax). A guard then refused folding +outright, which diverged from the bridge (which folds) at unembed.hook_in. The +shared ProcessWeights fold handles the router and every expert's W_in/W_gate, +and HT-with-MoE-fold measures bit-exact against HF (0.0000 log-softmax, +100% argmax on OLMoE-1B-7B), so folding is simply enabled. +""" + +from types import SimpleNamespace +from unittest import mock + +import pytest + +from transformer_lens.loading_from_pretrained import get_pretrained_model_config + + +def _moe_config(architecture: str, num_experts: int) -> SimpleNamespace: + return SimpleNamespace( + architectures=[architecture], + hidden_size=64, + num_attention_heads=4, + num_key_value_heads=4, + intermediate_size=128, + num_hidden_layers=2, + max_position_embeddings=512, + rms_norm_eps=1e-6, + vocab_size=100, + hidden_act="silu", + rope_theta=500000.0, + sliding_window=None, + num_experts=num_experts, + num_local_experts=num_experts, + num_experts_per_tok=2, + norm_topk_prob=False, + tie_word_embeddings=False, + initializer_range=0.02, + ) + + +@pytest.mark.parametrize( + "model_name,architecture,n_experts", + [ + ("allenai/OLMoE-1B-7B-0924", "OlmoeForCausalLM", 64), + ("mistralai/Mixtral-8x7B-v0.1", "MixtralForCausalLM", 8), + ], +) +@mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") +def test_moe_folds_like_dense(mock_auto_config, caplog, model_name, architecture, n_experts): + """fold_ln=True switches MoE to the folded *Pre norm, same as dense, without + a refusal warning — the gains are folded into router and experts, not dropped.""" + mock_auto_config.from_pretrained.return_value = _moe_config(architecture, n_experts) + with caplog.at_level("WARNING"): + cfg = get_pretrained_model_config(model_name, fold_ln=True) + assert cfg.num_experts == n_experts + assert cfg.normalization_type == "RMSPre", cfg.normalization_type + assert not any("MoE" in r.getMessage() for r in caplog.records), [ + r.getMessage() for r in caplog.records + ] + + +def test_process_weights_folds_moe_and_preserves_outputs() -> None: + """The fold must engage (norms swap to *Pre) and be equivalence-preserving on + a MoE model — a skipped fold leaves RMS modules; a broken fold moves logits.""" + import torch + + from transformer_lens import HookedTransformer, HookedTransformerConfig + from transformer_lens.components import RMSNormPre + + torch.manual_seed(0) + cfg = HookedTransformerConfig( + n_layers=2, + d_model=32, + d_head=8, + n_heads=4, + d_mlp=64, + d_vocab=50, + n_ctx=16, + act_fn="silu", + normalization_type="RMS", + gated_mlp=True, + num_experts=4, + experts_per_token=2, + ) + model = HookedTransformer(cfg) + with torch.no_grad(): + for name, param in model.named_parameters(): + torch.nn.init.normal_(param, std=0.2) + # Non-trivial gains, or folding is a vacuous multiply-by-one. + for block in model.blocks: + block.ln1.w.copy_(torch.rand_like(block.ln1.w) + 0.5) + block.ln2.w.copy_(torch.rand_like(block.ln2.w) + 0.5) + model.ln_final.w.copy_(torch.rand_like(model.ln_final.w) + 0.5) + model.eval() + tokens = torch.randint(0, 50, (1, 8)) + with torch.no_grad(): + before = model(tokens) + model.process_weights_(fold_ln=True, center_writing_weights=False, center_unembed=False) + assert isinstance(model.blocks[0].ln2, RMSNormPre), type(model.blocks[0].ln2).__name__ + with torch.no_grad(): + after = model(tokens) + torch.testing.assert_close(after, before, atol=1e-4, rtol=1e-4) diff --git a/tests/unit/test_post_norm_processing_guards.py b/tests/unit/test_post_norm_processing_guards.py new file mode 100644 index 0000000000..deb7684a04 --- /dev/null +++ b/tests/unit/test_post_norm_processing_guards.py @@ -0,0 +1,119 @@ +"""LN folding and writing-weight centering must stay off for post-norm decoders. + +Both transforms assume the norm gain sits on a sublayer's INPUT. OLMo 2/3 apply +ln1/ln2 to the sublayer OUTPUT, so folding is the wrong algebra: on the real +allenai/Olmo-3-1025-7B it moved log-softmax by 19.73 and dropped argmax agreement +with HF to 0%. The guard existed but named only OLMo 2, so OLMo 3 folded silently. +""" + +from types import SimpleNamespace +from unittest import mock + +import pytest +import torch + +from transformer_lens.loading_from_pretrained import get_pretrained_model_config +from transformer_lens.utilities.architectures import POST_NORM_ARCHITECTURES +from transformer_lens.weight_processing import ProcessWeights + + +def _tl_config(architecture: str): + from transformer_lens import HookedTransformerConfig + + return HookedTransformerConfig( + n_layers=0, + d_model=8, + n_ctx=16, + d_head=4, + n_heads=2, + d_vocab=10, + act_fn="silu", + normalization_type="RMS", + original_architecture=architecture, + positional_embedding_type="rotary", + ) + + +POST_NORM_MODELS = [ + ("allenai/Olmo-3-1025-7B", "Olmo3ForCausalLM"), + ("allenai/OLMo-2-0425-1B", "Olmo2ForCausalLM"), +] + + +def _olmo_hf_config(architecture: str) -> SimpleNamespace: + return SimpleNamespace( + architectures=[architecture], + hidden_size=64, + num_attention_heads=4, + num_key_value_heads=4, + intermediate_size=128, + num_hidden_layers=4, + max_position_embeddings=512, + rms_norm_eps=1e-6, + vocab_size=100, + hidden_act="silu", + rope_theta=500000.0, + layer_types=["sliding_attention"] * 3 + ["full_attention"], + sliding_window=4096, + initializer_range=0.02, + tie_word_embeddings=False, + rope_parameters={ + "sliding_attention": {"rope_type": "default", "rope_theta": 500000.0}, + "full_attention": {"rope_type": "default", "rope_theta": 500000.0}, + }, + ) + + +@pytest.mark.parametrize("model_name,architecture", POST_NORM_MODELS) +@mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") +def test_fold_ln_is_refused(mock_auto_config, caplog, model_name, architecture) -> None: + mock_auto_config.from_pretrained.return_value = _olmo_hf_config(architecture) + with caplog.at_level("WARNING"): + get_pretrained_model_config(model_name, fold_ln=True) + assert any( + "fold_ln=True is incompatible" in record.getMessage() for record in caplog.records + ), [r.getMessage() for r in caplog.records] + + +@mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") +def test_pre_norm_architecture_still_folds(mock_auto_config, caplog) -> None: + """Negative control: the guard must not disable folding for everyone.""" + mock_auto_config.from_pretrained.return_value = SimpleNamespace( + architectures=["LlamaForCausalLM"], + hidden_size=64, + num_attention_heads=4, + num_key_value_heads=4, + intermediate_size=128, + num_hidden_layers=2, + max_position_embeddings=512, + rms_norm_eps=1e-6, + vocab_size=100, + hidden_act="silu", + rope_theta=10000.0, + ) + with caplog.at_level("WARNING"): + get_pretrained_model_config("01-ai/Yi-6B", fold_ln=True) + assert not any("fold_ln=True is incompatible" in r.getMessage() for r in caplog.records) + + +@pytest.mark.parametrize("architecture", sorted(POST_NORM_ARCHITECTURES)) +def test_embeddings_are_not_centered(architecture) -> None: + """The first attention's input is un-normed, so centering W_E shifts a residual + stream nothing re-normalizes.""" + torch.manual_seed(0) + embedding = torch.randn(10, 8) + state = {"embed.W_E": embedding.clone()} + cfg = _tl_config(architecture) + out = ProcessWeights.center_writing_weights(state, cfg) + torch.testing.assert_close(out["embed.W_E"], embedding) + + +def test_embeddings_are_centered_for_pre_norm() -> None: + """Engagement check: centering is a no-op above only because of the guard.""" + torch.manual_seed(0) + embedding = torch.randn(10, 8) + state = {"embed.W_E": embedding.clone()} + cfg = _tl_config("LlamaForCausalLM") + out = ProcessWeights.center_writing_weights(state, cfg) + assert not torch.allclose(out["embed.W_E"], embedding) + torch.testing.assert_close(out["embed.W_E"].mean(-1), torch.zeros(10), atol=1e-6, rtol=0) diff --git a/tests/unit/test_train_config_isolation.py b/tests/unit/test_train_config_isolation.py new file mode 100644 index 0000000000..0107d35694 --- /dev/null +++ b/tests/unit/test_train_config_isolation.py @@ -0,0 +1,37 @@ +"""train() must not mutate the caller's config (device/wandb defaults).""" + +from __future__ import annotations + +import torch +from torch.utils.data import Dataset + +from transformer_lens import HookedTransformer, HookedTransformerConfig +from transformer_lens.train import HookedTransformerTrainConfig, train + + +def test_train_leaves_caller_config_untouched() -> None: + model = HookedTransformer( + HookedTransformerConfig( + n_layers=1, d_model=16, d_head=8, n_heads=2, n_ctx=8, d_vocab=16, act_fn="gelu" + ) + ) + + class _TokensDataset(Dataset): + def __len__(self) -> int: + return 1 + + def __getitem__(self, idx: int) -> dict: + return {"tokens": torch.tensor([1, 2, 3, 4])} + + dataset = _TokensDataset() + config = HookedTransformerTrainConfig( + num_epochs=1, + batch_size=1, + lr=1e-3, + seed=0, + device=None, + ) + + train(model, config, dataset) + + assert config.device is None, "train() wrote its resolved device onto the caller's config" diff --git a/tests/unit/tools/model_registry/test_clear_hf_cache.py b/tests/unit/tools/model_registry/test_clear_hf_cache.py new file mode 100644 index 0000000000..b9047a736d --- /dev/null +++ b/tests/unit/tools/model_registry/test_clear_hf_cache.py @@ -0,0 +1,30 @@ +"""_clear_hf_cache must never delete the pinned Phase-4 judge: the sweep clears +the HF cache after every model family, and re-downloading the judge each time +defeats the batch preload.""" + +import pytest + +pytest.importorskip("transformers") + + +def test_clear_hf_cache_preserves_judge_snapshot(tmp_path, monkeypatch): + from pathlib import Path + + from transformer_lens.benchmarks.text_quality import JUDGE_MODEL_ID + from transformer_lens.tools.model_registry import verify_models + + hub = tmp_path / ".cache" / "huggingface" / "hub" + judge_dir = hub / ("models--" + JUDGE_MODEL_ID.replace("/", "--")) / "blobs" + other_dir = hub / "models--someone--other-model" / "blobs" + judge_dir.mkdir(parents=True) + other_dir.mkdir(parents=True) + judge_blob = judge_dir / "aaaa" + other_blob = other_dir / "bbbb" + judge_blob.write_bytes(b"judge-weights") + other_blob.write_bytes(b"other-weights") + + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + verify_models._clear_hf_cache(quiet=True) + + assert judge_blob.exists(), "judge blob must survive the per-family cache clear" + assert not other_blob.exists(), "non-judge blobs must still be cleared" diff --git a/tests/unit/tools/model_registry/test_prompt_profiles.py b/tests/unit/tools/model_registry/test_prompt_profiles.py new file mode 100644 index 0000000000..00c4a4dcd7 --- /dev/null +++ b/tests/unit/tools/model_registry/test_prompt_profiles.py @@ -0,0 +1,231 @@ +"""Profile resolution: curation must beat unreliable Hub metadata (observed +mis-tags: mt0-base as text-generation, conversational on base models, +unordered Helsinki-NLP language tags), and gaps must fall through safely.""" + +import pytest + +pytest.importorskip("transformers") + +from transformer_lens.benchmarks.text_quality_profiles import ( + DEFAULT_PROFILE, + HFSignals, + ProfileSpec, + extract_languages, + profile_from_hf_signals, + resolve_profile, +) + + +class TestPrecedence: + def test_override_beats_architecture_rule(self, monkeypatch): + """A real override-vs-arch clash: Pegasus's arch rule says + summarization; a per-model override must still win.""" + from transformer_lens.benchmarks import text_quality_profiles as tp + + monkeypatch.setitem(tp.MODEL_PROFILE_OVERRIDES, "google/pegasus-xsum", "continuation") + spec = resolve_profile("google/pegasus-xsum", "PegasusForConditionalGeneration") + assert spec == ProfileSpec("continuation") + + def test_override_beats_signals(self): + """long-t5 override must win even when signals disagree.""" + spec = resolve_profile( + "google/long-t5-tglobal-base", + "LongT5ForConditionalGeneration", + signals=HFSignals(pipeline_tag="summarization"), + ) + assert spec == ProfileSpec("task:denoise") + + def test_architecture_rule_beats_fetched_tag(self): + """Pegasus is summarization by architecture even if the Hub tag lies.""" + spec = resolve_profile( + "google/pegasus-xsum", + "PegasusForConditionalGeneration", + signals=HFSignals(pipeline_tag="text-generation"), + ) + assert spec.kind == "task:summarization" + + def test_mt0_mistag_resolves_to_instruction(self): + """Hub tags mt0-base text-generation; the override must correct it.""" + spec = resolve_profile( + "bigscience/mt0-base", + "MT5ForConditionalGeneration", + signals=HFSignals(pipeline_tag="text-generation"), + ) + assert spec.kind == "task:instruction" + + def test_fetched_tag_fills_gap(self): + """BART has no arch rule (checkpoint-dependent); the Hub tag decides.""" + spec = resolve_profile( + "facebook/bart-large-cnn", + "BartForConditionalGeneration", + signals=HFSignals(pipeline_tag="summarization", languages=("en",)), + ) + assert spec.kind == "task:summarization" + + def test_null_pipeline_tag_falls_through_to_arch_rule(self): + """m2m100 has pipeline_tag=None on the Hub; the arch rule must hold.""" + spec = resolve_profile( + "facebook/m2m100_418M", + "M2M100ForConditionalGeneration", + signals=HFSignals(pipeline_tag=None), + ) + assert spec.kind == "task:translation" + + def test_stored_registry_value_used_when_no_signals(self): + spec = resolve_profile("some/model", "GPT2LMHeadModel", "continuation@fr") + assert spec == ProfileSpec("continuation", "fr") + + def test_unknown_seq2seq_defaults_to_denoise_not_continuation(self): + """An unlabelled seq2seq cannot continue text; denoising is its only prompt.""" + spec = resolve_profile("someone/random-t5", "T5GemmaForConditionalGeneration") + assert spec.kind == "task:denoise" + + def test_unknown_causal_lm_defaults_to_continuation(self): + assert resolve_profile("someone/random-lm", "LlamaForCausalLM") == DEFAULT_PROFILE + + +class TestHubSignals: + def test_conversational_tag_alone_is_not_chat(self): + """HF adds `conversational` to ANY repo shipping a chat template, base + models included (observed on Qwen/Qwen2.5-0.5B).""" + spec = profile_from_hf_signals( + "Qwen/Qwen2.5-0.5B", + "Qwen2ForCausalLM", + HFSignals(pipeline_tag="text-generation", tags=("conversational",)), + ) + assert spec is not None and spec.kind == "continuation" + + def test_code_tag_maps_to_code_continuation(self): + spec = profile_from_hf_signals( + "bigcode/some-model", "GPTBigCodeForCausalLM", HFSignals(tags=("code",)) + ) + assert spec == ProfileSpec("continuation", "code") + + def test_marian_direction_from_model_id_not_tag_order(self): + """Helsinki-NLP language tags are unordered; only opus-mt-{src}-{tgt} + carries the direction.""" + spec = resolve_profile( + "Helsinki-NLP/opus-mt-nl-en", + "MarianMTModel", + signals=HFSignals(languages=("en", "nl")), # tag order is wrong on purpose + ) + assert (spec.src, spec.lang) == ("nl", "en") + + def test_translation_tag_without_direction_returns_none(self): + """Tag lists are unordered: guessing a pair risks a reversed or + identity direction, so signals alone must abstain (resolution then + falls through to overrides/arch rules — t5-small still lands on + en-de via its override).""" + spec = profile_from_hf_signals( + "google-t5/t5-small", + "T5ForConditionalGeneration", + HFSignals(pipeline_tag="translation"), + ) + assert spec is None + resolved = resolve_profile( + "google-t5/t5-small", + "T5ForConditionalGeneration", + signals=HFSignals(pipeline_tag="translation"), + ) + assert (resolved.src, resolved.lang) == ("en", "de") + + def test_translation_tag_with_en_and_target_infers_pair(self): + spec = profile_from_hf_signals( + "someone/en-fr-translator", + "BartForConditionalGeneration", + HFSignals(pipeline_tag="translation", languages=("en", "fr")), + ) + assert spec is not None and (spec.src, spec.lang) == ("en", "fr") + + +class TestLanguageExtraction: + def test_handles_str_and_list(self): + assert extract_languages("fr", []) == ("fr",) + assert extract_languages(["de", "en"], []) == ("de", "en") + + def test_merges_iso_tags_and_drops_noise(self): + langs = extract_languages( + None, ["pytorch", "transformers", "nl", "marian", "safetensors", "en"] + ) + assert langs == ("nl", "en") + + def test_caps_at_eight(self): + many = ["fr", "es", "de", "it", "nl", "pt", "ru", "ja", "ar", "hi"] + assert len(extract_languages(many, [])) == 8 + + +class TestProfileSpecGrammar: + def test_round_trip(self): + for text in ("continuation", "continuation@code", "chat@fr", "task:translation@en-de"): + assert str(ProfileSpec.parse(text)) == text + + def test_rejects_unknown_kind(self): + with pytest.raises(ValueError): + ProfileSpec.parse("poetry@en") + + def test_rejects_translation_without_pair(self): + with pytest.raises(ValueError): + ProfileSpec.parse("task:translation@de") + + +class TestChatIdHeuristic: + """Instruct/chat/-it ids resolve to the chat profile — nothing else can + (the conversational tag covers base models; no arch distinguishes tuned + from base). Runtime downgrades template-less models back to continuation.""" + + def test_instruct_id_resolves_chat(self): + spec = resolve_profile("Qwen/Qwen2.5-0.5B-Instruct", "Qwen2ForCausalLM") + assert spec.kind == "chat" + + def test_it_suffix_resolves_chat(self): + spec = resolve_profile("google/gemma-2-2b-it", "Gemma2ForCausalLM") + assert spec.kind == "chat" + + def test_base_id_stays_continuation(self): + assert resolve_profile("Qwen/Qwen2.5-0.5B", "Qwen2ForCausalLM").kind == "continuation" + assert resolve_profile("google/gemma-2-2b", "Gemma2ForCausalLM").kind == "continuation" + + def test_override_still_beats_chat_heuristic(self, monkeypatch): + from transformer_lens.benchmarks import text_quality_profiles as tp + + monkeypatch.setitem(tp.MODEL_PROFILE_OVERRIDES, "someone/model-instruct", "continuation@fr") + spec = resolve_profile("someone/model-instruct", "LlamaForCausalLM") + assert spec == ProfileSpec("continuation", "fr") + + def test_arch_rule_beats_chat_heuristic(self): + """A Blenderbot-style arch keeps its rule even with a chatty id.""" + spec = resolve_profile("someone/blenderbot-chat", "BlenderbotForConditionalGeneration") + assert spec.kind == "chat" # via arch rule, not id — same outcome + spec2 = resolve_profile("someone/opus-mt-nl-en-chat", "MarianMTModel") + assert spec2.kind == "task:translation" + + +def test_chat_heuristic_keeps_stored_language(): + """The id heuristic fixes the kind; a stored chat@fr must survive + resolve->writeback (it was flattened to chat@en and clobbered).""" + from transformer_lens.benchmarks.text_quality_profiles import resolve_profile + + spec = resolve_profile("org/model-7b-instruct", "LlamaForCausalLM", registry_profile="chat@fr") + assert str(spec) == "chat@fr" + # A stored non-chat profile does not hijack the heuristic. + spec = resolve_profile( + "org/model-7b-instruct", "LlamaForCausalLM", registry_profile="continuation@fr" + ) + assert str(spec) == "chat" + + +def test_non_english_denoise_is_a_coverage_gap(): + """IndicBART's stale score was measured under a broken MBart profile; + until Indic denoise prompts exist, a non-en denoise profile must SKIP + (coverage gap), never score against English sentences.""" + from transformer_lens.benchmarks.text_quality_profiles import ( + ProfileSpec, + prompts_for, + resolve_profile, + ) + + assert str(resolve_profile("ai4bharat/IndicBART", "MBartForConditionalGeneration")) == ( + "task:denoise@hi" + ) + assert prompts_for(ProfileSpec("task:denoise", lang="hi")) is None + assert prompts_for(ProfileSpec("task:denoise", lang="en")) is not None diff --git a/tests/unit/tools/model_registry/test_update_model_registry.py b/tests/unit/tools/model_registry/test_update_model_registry.py new file mode 100644 index 0000000000..663457c2fe --- /dev/null +++ b/tests/unit/tools/model_registry/test_update_model_registry.py @@ -0,0 +1,331 @@ +"""Regression tests for main_benchmark.update_model_registry. + +This path was a drifted mirror of verify_models' registry-writing logic: its +phase dict stopped at phase 3 and it wrote STATUS_VERIFIED unconditionally, +bypassing the provisional gate for --no-hf-reference runs. It now shares +registry_io's extract_phase_scores / pass_status, so these tests pin the +registry outcomes, not the internals. +""" +import json +from types import SimpleNamespace + +import pytest + +from transformer_lens.benchmarks.main_benchmark import update_model_registry +from transformer_lens.benchmarks.utils import BenchmarkResult, BenchmarkSeverity +from transformer_lens.tools.model_registry.registry_io import ( + STATUS_FAILED, + STATUS_PROVISIONAL, + STATUS_VERIFIED, +) + +ARCH = "GPT2LMHeadModel" + + +def _result(phase, passed, name="forward_pass", severity=None, details=None): + if severity is None: + severity = BenchmarkSeverity.INFO if passed else BenchmarkSeverity.DANGER + return BenchmarkResult( + name=name, + severity=severity, + message="ok" if passed else "mismatch", + details=details, + passed=passed, + phase=phase, + ) + + +@pytest.fixture +def registry_paths(tmp_path, monkeypatch): + """Point registry_io at temp files and stub the AutoConfig network call.""" + from transformer_lens.tools.model_registry import registry_io + + supported = { + "total_architectures": 1, + "total_models": 1, + "total_verified": 0, + "models": [ + { + "architecture_id": ARCH, + "model_id": "seeded/model", + "status": 0, + "verified_date": None, + "metadata": None, + "note": None, + }, + ], + } + supported_path = tmp_path / "supported_models.json" + supported_path.write_text(json.dumps(supported, indent=2)) + history_path = tmp_path / "verification_history.json" + + monkeypatch.setattr(registry_io, "_SUPPORTED_MODELS_PATH", supported_path) + monkeypatch.setattr(registry_io, "_VERIFICATION_HISTORY_PATH", history_path) + monkeypatch.setattr( + "transformers.AutoConfig.from_pretrained", + lambda *args, **kwargs: SimpleNamespace(architectures=[ARCH]), + ) + return supported_path, history_path + + +def _entry(supported_path, model_id): + data = json.loads(supported_path.read_text()) + return next(m for m in data["models"] if m["model_id"] == model_id), data + + +class TestProvisionalGate: + def test_no_hf_reference_writes_provisional(self, registry_paths): + supported_path, history_path = registry_paths + results = [_result(1, True)] + + assert update_model_registry("new/model", results, use_hf_reference=False) + + entry, data = _entry(supported_path, "new/model") + assert entry["status"] == STATUS_PROVISIONAL + assert entry["note"].startswith("Structural only (no HF reference)") + assert data["total_verified"] == 0 + assert data["total_provisional"] == 1 + # No history record: VerificationHistory.is_verified() treats any + # record as verified — the second "counts as verified" path. + assert not history_path.exists() + + def test_hf_reference_writes_verified(self, registry_paths): + supported_path, history_path = registry_paths + results = [_result(1, True)] + + assert update_model_registry("new/model", results, use_hf_reference=True) + + entry, data = _entry(supported_path, "new/model") + assert entry["status"] == STATUS_VERIFIED + assert data["total_verified"] == 1 + history = json.loads(history_path.read_text()) + assert history["records"][-1]["model_id"] == "new/model" + assert history["records"][-1]["verified_by"] == "main_benchmark" + + def test_default_is_conservative_provisional(self, registry_paths): + supported_path, _ = registry_paths + update_model_registry("new/model", [_result(1, True)]) + entry, _ = _entry(supported_path, "new/model") + assert entry["status"] == STATUS_PROVISIONAL + + +class TestPhaseCoverage: + def test_phase9_contributes_phase9_score(self, registry_paths): + # The drifted mirror's {1: [], 2: [], 3: []} dict silently dropped P9. + supported_path, _ = registry_paths + results = [_result(1, True), _result(9, True, name="vision_forward")] + + update_model_registry("new/model", results, use_hf_reference=True) + + entry, _ = _entry(supported_path, "new/model") + assert entry["phase9_score"] == 100.0 + assert entry["status"] == STATUS_VERIFIED + + def test_unrun_phases_preserve_existing_scores(self, registry_paths): + # The old path wrote None for unrun phases, clobbering prior scores. + supported_path, _ = registry_paths + update_model_registry("seeded/model", [_result(2, True)], use_hf_reference=True) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["phase2_score"] == 100.0 + + update_model_registry("seeded/model", [_result(1, True)], use_hf_reference=True) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["phase1_score"] == 100.0 + assert entry["phase2_score"] == 100.0 + + +class TestThresholdGate: + def test_failing_scores_write_failed_not_verified(self, registry_paths): + # The drifted mirror wrote VERIFIED even for all-fail runs. + supported_path, _ = registry_paths + results = [_result(1, False, name="logits_equivalence")] + + update_model_registry("seeded/model", results, use_hf_reference=True) + + entry, data = _entry(supported_path, "seeded/model") + assert entry["status"] == STATUS_FAILED + assert "Below threshold" in entry["note"] + assert data["total_verified"] == 0 + + +class TestPromptProfileWriteback: + """The Phase-4 profile actually used must land in the registry sparsely: + non-default profiles are recorded, the default writes no key at all (the + registry JSON is served to the docs site; 15k default keys are dead weight).""" + + def _p4(self, profile): + return _result( + 4, + True, + name="text_quality", + details={"score": 91.0, "prompt_profile": profile}, + ) + + def test_prompt_profile_written_from_p4_details(self, registry_paths): + supported_path, _ = registry_paths + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("task:translation@en-de")], + use_hf_reference=True, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["prompt_profile"] == "task:translation@en-de" + # Key order: sparse key sits right after note, before phase scores. + keys = list(entry) + assert keys.index("prompt_profile") == keys.index("note") + 1 + + def test_default_profile_not_written(self, registry_paths): + supported_path, _ = registry_paths + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("continuation")], + use_hf_reference=True, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert "prompt_profile" not in entry + + def test_existing_profile_survives_profileless_rerun(self, registry_paths): + """A later run without a P4 result must not clobber the stored profile.""" + supported_path, _ = registry_paths + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("chat@fr")], + use_hf_reference=True, + ) + update_model_registry("seeded/model", [_result(1, True)], use_hf_reference=True) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["prompt_profile"] == "chat@fr" + + def test_default_profile_clears_stale_nondefault(self, registry_paths): + """A model re-resolved to the default must lose its old sparse key — + otherwise a stale 'chat@fr' misdescribes how the score was produced.""" + supported_path, _ = registry_paths + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("chat@fr")], + use_hf_reference=True, + ) + update_model_registry( + "seeded/model", + [_result(1, True), self._p4("continuation")], + use_hf_reference=True, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert "prompt_profile" not in entry + + def test_new_entry_append_carries_profile(self, registry_paths): + """The append (model-not-in-registry) branch must also write the sparse + key, positioned after note.""" + supported_path, _ = registry_paths + update_model_registry( + "unseeded/model", + [_result(1, True), self._p4("task:summarization")], + use_hf_reference=True, + ) + entry, _ = _entry(supported_path, "unseeded/model") + assert entry["prompt_profile"] == "task:summarization" + keys = list(entry) + assert keys.index("prompt_profile") == keys.index("note") + 1 + + +class TestP4ScoringVersionStamp: + """phase4_score is a mixed-scale column (old GPT-2 scale vs pinned-judge + ratio scale); every P4-bearing write must stamp the scale it measured on, + and writes without a P4 result must not touch an existing stamp.""" + + def test_p4_write_stamps_current_version(self, registry_paths): + from transformer_lens.benchmarks.text_quality_profiles import P4_SCORING_VERSION + from transformer_lens.tools.model_registry import registry_io + + supported_path, _ = registry_paths + registry_io.update_model_status( + "seeded/model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0, 4: 91.0}, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["p4_scoring_version"] == P4_SCORING_VERSION + + def test_no_p4_write_preserves_existing_stamp(self, registry_paths): + from transformer_lens.tools.model_registry import registry_io + + supported_path, _ = registry_paths + registry_io.update_model_status( + "seeded/model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0, 4: 91.0}, + ) + registry_io.update_model_status( + "seeded/model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0}, + ) + entry, _ = _entry(supported_path, "seeded/model") + assert entry["p4_scoring_version"] == 2 + assert entry["phase4_score"] == 91.0 + + def test_old_scale_entry_has_no_stamp(self, registry_paths): + supported_path, _ = registry_paths + entry, _ = _entry(supported_path, "seeded/model") + assert "p4_scoring_version" not in entry + + def test_new_entry_with_p4_is_stamped(self, registry_paths): + from transformer_lens.tools.model_registry import registry_io + + supported_path, _ = registry_paths + registry_io.update_model_status( + "brand/new-model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0, 4: 77.0}, + ) + entry, _ = _entry(supported_path, "brand/new-model") + assert entry["p4_scoring_version"] == 2 + + +class TestPreservedIssueSuffix: + """A phases-1-4 pass must not overwrite tracked residue from phases it + did not re-run (gemma-2-2b-it's P3=95.5 unembed_centering note was + clobbered by a bare 'Core verification completed').""" + + def test_sub100_score_from_unrun_phase_is_retained(self, registry_paths): + from transformer_lens.tools.model_registry import registry_io + from transformer_lens.tools.model_registry.verify_models import ( + _preserved_issue_suffix, + ) + + registry_io.update_model_status( + "seeded/model", + "GPT2LMHeadModel", + registry_io.STATUS_VERIFIED, + phase_scores={1: 100.0, 3: 95.5}, + ) + assert _preserved_issue_suffix("seeded/model", [1, 4]) == ( + " (prior issues retained: P3=95.5%)" + ) + # Re-running the phase drops it from the suffix (the fresh score speaks). + assert _preserved_issue_suffix("seeded/model", [1, 3, 4]) == "" + + def test_clean_entry_has_no_suffix(self, registry_paths): + from transformer_lens.tools.model_registry.verify_models import ( + _preserved_issue_suffix, + ) + + assert _preserved_issue_suffix("seeded/model", [1, 4]) == "" + + +def test_judge_overhead_not_charged_to_accelerator(): + """The judge is CPU-pinned; charging its 2.5 GB to a cuda budget caused + spurious VRAM skips.""" + from transformer_lens.tools.model_registry.verify_models import ( + estimate_benchmark_memory_gb, + ) + + # Small model so the phase-4 peak (model + judge) is the max across phases. + cpu = estimate_benchmark_memory_gb(int(1e6), phases=[1, 4], device="cpu") + cuda = estimate_benchmark_memory_gb(int(1e6), phases=[1, 4], device="cuda") + assert cpu > 2.5 + assert cuda < 0.1 diff --git a/tests/unit/tools/test_jacobian_lens.py b/tests/unit/tools/test_jacobian_lens.py index f01ebe302b..9b83793725 100644 --- a/tests/unit/tools/test_jacobian_lens.py +++ b/tests/unit/tools/test_jacobian_lens.py @@ -4,7 +4,7 @@ from enum import IntEnum from inspect import Parameter, signature from types import SimpleNamespace -from typing import Any +from typing import Any, Optional, Sequence import numpy as np import pytest @@ -21,6 +21,8 @@ from transformer_lens.tools.analysis import ( JacobianLens, JSpaceDecomposition, + JSpaceOccupancy, + JSpaceVarianceProfile, get_sparse_decomposition, ) from transformer_lens.utilities.activation_functions import apply_softcap @@ -1391,3 +1393,107 @@ def test_decompose_rejects_unfitted_layer(toy_model: _ToyBridge, fitted_lens: Ja fitted_lens.decompose( toy_model, torch.randn(toy_model.cfg.d_model), layer=N_LAYERS - 1, k=3 ) + + +def test_occupancy_on_toy_model_raw_activation( + toy_model: _ToyBridge, fitted_lens: JacobianLens +) -> None: + """occupancy on a raw activation returns a JSpaceOccupancy with a count in [1, max_atoms].""" + layer = fitted_lens.source_layers[0] + activation = torch.randn(toy_model.cfg.d_model) + result = fitted_lens.occupancy(toy_model, activation, layer, max_atoms=5, seed=0) + assert isinstance(result, JSpaceOccupancy) + assert 1 <= result.occupancy <= 5 + assert result.support.numel() == 5 + assert result.marginal_captured_variance.shape == result.control_captured_variance.shape + + +def test_occupancy_prompt_path_runs(toy_model: _ToyBridge, fitted_lens: JacobianLens) -> None: + """occupancy accepts a prompt plus position, running the model to fetch the activation.""" + layer = fitted_lens.source_layers[0] + result = fitted_lens.occupancy( + toy_model, "a toy prompt", layer, position=-1, max_atoms=5, seed=0 + ) + assert isinstance(result, JSpaceOccupancy) + assert 1 <= result.occupancy <= 5 + + +def test_occupancy_rejects_unfitted_layer(toy_model: _ToyBridge, fitted_lens: JacobianLens) -> None: + """occupancy shares decompose's validation: an unfitted layer raises.""" + with pytest.raises(ValueError): + fitted_lens.occupancy(toy_model, torch.randn(toy_model.cfg.d_model), layer=N_LAYERS - 1) + + +def test_fraction_of_variance_on_toy_model( + toy_model: _ToyBridge, fitted_lens: JacobianLens +) -> None: + """fraction_of_variance returns per-layer median and pooled ratios in [0, 1] over a corpus.""" + profile = fitted_lens.fraction_of_variance( + toy_model, ["a toy prompt here", "another toy prompt goes here"], k=3, skip_first=0 + ) + assert isinstance(profile, JSpaceVarianceProfile) + assert profile.layers == list(fitted_lens.source_layers) + for layer in profile.layers: + assert 0.0 <= profile.median[layer] <= 1.0 + assert 0.0 <= profile.pooled[layer] <= 1.0 + assert profile.per_position[layer].numel() > 0 + + +def test_fraction_of_variance_rejects_unfitted_layer( + toy_model: _ToyBridge, fitted_lens: JacobianLens +) -> None: + with pytest.raises(ValueError): + fitted_lens.fraction_of_variance(toy_model, "a toy prompt", layers=[N_LAYERS - 1]) + + +def test_fraction_of_variance_rejects_empty_corpus( + toy_model: _ToyBridge, fitted_lens: JacobianLens +) -> None: + with pytest.raises(ValueError): + fitted_lens.fraction_of_variance(toy_model, []) + + +@pytest.mark.parametrize( + "tokens", + [ + torch.zeros(4, dtype=torch.long), + torch.zeros((2, 4), dtype=torch.long), + torch.zeros((1, 1, 4), dtype=torch.long), + ], + ids=["missing-batch-dimension", "multiple-prompts", "extra-dimension"], +) +def test_fraction_of_variance_rejects_invalid_token_shape( + toy_model: _ToyBridge, fitted_lens: JacobianLens, tokens: torch.Tensor +) -> None: + with pytest.raises( + ValueError, + match=r"fraction_of_variance expects each tokenized prompt to have shape \[1, seq\]", + ): + fitted_lens.fraction_of_variance(toy_model, tokens, k=3, skip_first=0) + + +@pytest.mark.parametrize("positions", [None, [0]], ids=["default-sampling", "explicit-positions"]) +def test_fraction_of_variance_rejects_negative_skip_first( + toy_model: _ToyBridge, + fitted_lens: JacobianLens, + positions: Optional[Sequence[int]], +) -> None: + with pytest.raises(ValueError, match="skip_first must be non-negative"): + fitted_lens.fraction_of_variance( + toy_model, "a toy prompt", k=3, skip_first=-1, positions=positions + ) + + +def test_fraction_of_variance_yields_nan_when_no_positions_are_sampled( + toy_model: _ToyBridge, fitted_lens: JacobianLens +) -> None: + """When ``skip_first`` exceeds every prompt's length no position is sampled, so each layer's + ``median`` and ``pooled`` are NaN and ``per_position`` is empty (the documented contract).""" + import math + + profile = fitted_lens.fraction_of_variance(toy_model, "a toy prompt", k=3, skip_first=999) + assert profile.layers == list(fitted_lens.source_layers) + for layer in profile.layers: + assert math.isnan(profile.median[layer]) + assert math.isnan(profile.pooled[layer]) + assert profile.per_position[layer].numel() == 0 diff --git a/tests/unit/tools/test_jacobian_lens_decomposition.py b/tests/unit/tools/test_jacobian_lens_decomposition.py index c448e5d80b..3b822166e7 100644 --- a/tests/unit/tools/test_jacobian_lens_decomposition.py +++ b/tests/unit/tools/test_jacobian_lens_decomposition.py @@ -12,10 +12,12 @@ from transformer_lens.tools.analysis.jacobian_lens_decomposition import ( JSpaceDecomposition, + JSpaceOccupancy, _gradient_pursuit_step, _nnls_tolerances, _nonnegative_least_squares, _validate_nnls_kkt, + estimate_occupancy, get_sparse_decomposition, ) @@ -753,3 +755,121 @@ def test_rejects_non_2d_dictionary(): def test_rejects_non_1d_target(): with pytest.raises(ValueError): get_sparse_decomposition(torch.ones(2, 4), torch.eye(4), k=1) + + +# --------------------------------------------------------------------------- # +# Occupancy estimator (estimate_occupancy) +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("planted_atom_count", [1, 4, 7]) +def test_occupancy_recovers_planted_sparsity_on_orthonormal_dictionary(planted_atom_count): + """On an orthonormal dictionary a k-sparse planted target has occupancy exactly k: the real + greedy captures the planted atoms then saturates, so the point of maximum separation from the + random control lands at k.""" + torch.manual_seed(0) + d_model = 24 + dictionary = torch.linalg.qr(torch.randn(d_model, d_model)).Q # orthonormal atoms (rows) + target = torch.zeros(d_model) + for index in range(planted_atom_count): + target = target + (3.0 - 0.3 * index) * dictionary[2 * index + 1] + + max_atoms = min(2 * planted_atom_count + 4, d_model) + result = estimate_occupancy(target, dictionary, max_atoms=max_atoms) + + assert isinstance(result, JSpaceOccupancy) + assert result.occupancy == planted_atom_count + assert result.support.numel() == max_atoms + + +def test_occupancy_increases_with_planted_sparsity(): + """Occupancy tracks the planted sparsity: a 2-sparse target occupies fewer atoms than a + 9-sparse one on the same orthonormal dictionary (both recovered exactly).""" + torch.manual_seed(1) + d_model = 24 + dictionary = torch.linalg.qr(torch.randn(d_model, d_model)).Q + + def planted(count): + target = torch.zeros(d_model) + for index in range(count): + target = target + (3.0 - 0.2 * index) * dictionary[2 * index + 1] + return target + + assert ( + estimate_occupancy(planted(2), dictionary, max_atoms=20).occupancy + < estimate_occupancy(planted(9), dictionary, max_atoms=20).occupancy + ) + + +def test_occupancy_is_deterministic(): + """Identical inputs and seed give identical results (the random control is seeded).""" + torch.manual_seed(2) + dictionary = torch.randn(40, 8) + target = torch.randn(8) + first = estimate_occupancy(target, dictionary, seed=0) + second = estimate_occupancy(target, dictionary, seed=0) + assert first.occupancy == second.occupancy + assert torch.equal(first.support, second.support) + assert torch.allclose(first.control_captured_variance, second.control_captured_variance) + + +def test_occupancy_rejects_invalid_inputs(): + dictionary = torch.eye(6) + with pytest.raises(ValueError, match="max_atoms must be between"): + estimate_occupancy(torch.ones(6), dictionary, max_atoms=0) + with pytest.raises(ValueError, match="max_atoms must be between"): + estimate_occupancy(torch.ones(6), dictionary, max_atoms=99) + with pytest.raises(ValueError, match="num_control_dictionaries must be at least 1"): + estimate_occupancy(torch.ones(6), dictionary, max_atoms=6, num_control_dictionaries=0) + with pytest.raises(ValueError, match="x must be 1-D of length"): + estimate_occupancy(torch.ones(5), dictionary) # target length != d_model + with pytest.raises(ValueError, match="x must be 1-D of length"): + estimate_occupancy(torch.ones(2, 6), dictionary) # non-1-D target + zero_norm_dictionary = torch.eye(6) + zero_norm_dictionary[2] = 0.0 + with pytest.raises(ValueError, match="dictionary contains a non-finite or zero-norm atom"): + estimate_occupancy(torch.ones(6), zero_norm_dictionary, max_atoms=6) + + +@pytest.mark.parametrize("invalid_value", [float("nan"), float("inf"), float("-inf")]) +def test_occupancy_rejects_non_finite_target(invalid_value): + target = torch.ones(6) + target[0] = invalid_value + with pytest.raises(ValueError, match="x contains non-finite entries"): + estimate_occupancy(target, torch.eye(6), max_atoms=6) + + +def test_occupancy_rejects_zero_norm_target(): + with pytest.raises(ValueError, match="x must have non-zero norm"): + estimate_occupancy(torch.zeros(6), torch.eye(6), max_atoms=6) + + +@pytest.mark.parametrize("invalid_value", [float("nan"), float("inf"), float("-inf")]) +def test_occupancy_rejects_non_finite_dictionary(invalid_value): + dictionary = torch.eye(6) + dictionary[0, 0] = invalid_value + with pytest.raises(ValueError, match="dictionary contains non-finite entries"): + estimate_occupancy(torch.ones(6), dictionary, max_atoms=6) + + +def test_occupancy_rejects_non_finite_dictionary_norm(): + dictionary = torch.eye(6) + dictionary[0] = torch.finfo(torch.float32).max + with pytest.raises(ValueError, match="dictionary contains a non-finite or zero-norm atom"): + estimate_occupancy(torch.ones(6), dictionary, max_atoms=6) + + +@pytest.mark.parametrize("complex_input", ["x", "dictionary"]) +def test_occupancy_rejects_complex_inputs(complex_input): + target = torch.ones(6) + dictionary = torch.eye(6) + if complex_input == "x": + target = target.to(torch.complex64) + else: + dictionary = dictionary.to(torch.complex64) + with pytest.raises(ValueError, match="x and dictionary must be real-valued"): + estimate_occupancy(target, dictionary, max_atoms=6) + + +def test_occupancy_rejects_non_finite_target_norm(): + target = torch.full((6,), torch.finfo(torch.float32).max) + with pytest.raises(ValueError, match="x must have finite norm"): + estimate_occupancy(target, torch.eye(6), max_atoms=6) diff --git a/tests/unit/utilities/test_get_input_with_manually_prepended_bos.py b/tests/unit/utilities/test_get_input_with_manually_prepended_bos.py new file mode 100644 index 0000000000..f11a38bac1 --- /dev/null +++ b/tests/unit/utilities/test_get_input_with_manually_prepended_bos.py @@ -0,0 +1,70 @@ +"""Tests for get_input_with_manually_prepended_bos when the tokenizer has no BOS token. + +``to_tokens`` reaches this helper whenever the caller wants a BOS that the tokenizer +will not add on its own — ``prepend_bos and not cfg.tokenizer_prepends_bos``. For a +tokenizer with no BOS token that condition is *correctly* true rather than stale: +``detect_tokenizer_bos_eos`` requires a ``bos_token_id``, so it reports False for +BERT and T5, and ``prepend_bos`` defaults to True. The helper then evaluated +``None + input`` and raised ``TypeError: unsupported operand type(s) for +: +'NoneType' and 'str'``, naming neither the tokenizer nor the flag. + +This is the prepend-side counterpart of #1628, which covers the removal side. +""" + +from __future__ import annotations + +import pytest +from transformers import AutoTokenizer + +from transformer_lens.utilities.tokenize_utils import ( + get_input_with_manually_prepended_bos, +) + + +@pytest.fixture( + scope="module", + params=["google-bert/bert-base-cased", "google-t5/t5-small"], +) +def no_bos_tokenizer(request): + """BERT opens with [CLS] and T5 with nothing, so bos_token is None for both.""" + tokenizer = AutoTokenizer.from_pretrained(request.param) + assert tokenizer.bos_token is None + return tokenizer + + +@pytest.fixture(scope="module") +def bos_tokenizer(): + tokenizer = AutoTokenizer.from_pretrained("distilgpt2") + assert tokenizer.bos_token is not None + return tokenizer + + +def test_no_bos_token_returns_string_unchanged(no_bos_tokenizer) -> None: + """There is no BOS to prepend, so the string must come back untouched.""" + assert get_input_with_manually_prepended_bos(no_bos_tokenizer.bos_token, "hello world") == ( + "hello world" + ) + + +def test_no_bos_token_returns_list_unchanged(no_bos_tokenizer) -> None: + """Same for the batched form — and no partially-prepended list.""" + inputs = ["hello world", "second string"] + + result = get_input_with_manually_prepended_bos(no_bos_tokenizer.bos_token, inputs) + + assert result == ["hello world", "second string"] + + +def test_a_real_bos_is_still_prepended_to_a_string(bos_tokenizer) -> None: + """The guard must not disturb the case the helper exists for.""" + result = get_input_with_manually_prepended_bos(bos_tokenizer.bos_token, "hello world") + + assert result == bos_tokenizer.bos_token + "hello world" + + +def test_a_real_bos_is_still_prepended_to_a_list(bos_tokenizer) -> None: + bos = bos_tokenizer.bos_token + + result = get_input_with_manually_prepended_bos(bos, ["hello world", "second string"]) + + assert result == [bos + "hello world", bos + "second string"] diff --git a/tests/unit/utilities/test_get_tokens_with_bos_removed.py b/tests/unit/utilities/test_get_tokens_with_bos_removed.py new file mode 100644 index 0000000000..75e51ad741 --- /dev/null +++ b/tests/unit/utilities/test_get_tokens_with_bos_removed.py @@ -0,0 +1,82 @@ +"""Tests for get_tokens_with_bos_removed when the tokenizer has no BOS token. + +Callers gate this helper on ``cfg.tokenizer_prepends_bos``. That flag is set by +``detect_tokenizer_bos_eos``, which requires a ``bos_token_id`` — so a tokenizer +with none should never reach here. It does when the flag is stale: a bridge built +via ``build_bridge_from_module(tokenizer=None)`` keeps the config default of True, +and the tokenizer setter only re-runs detection on *re*-assignment. + +Trusting a stale flag is not harmless. Under right padding the helper drops the +first token unconditionally, which silently removes ``[CLS]`` from a BERT +tokenizer's output; under left padding it compares tokens against ``None`` and +raises an ``AttributeError`` naming neither the tokenizer nor the flag. +""" + +from __future__ import annotations + +import pytest +import torch +from transformers import AutoTokenizer + +from transformer_lens.utilities.tokenize_utils import get_tokens_with_bos_removed + + +@pytest.fixture(scope="module") +def no_bos_tokenizer(): + """BERT uses [CLS] rather than a BOS token, so bos_token_id is None.""" + tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") + assert tokenizer.bos_token_id is None + return tokenizer + + +@pytest.fixture(scope="module") +def bos_tokenizer(): + tokenizer = AutoTokenizer.from_pretrained("distilgpt2") + assert tokenizer.bos_token_id is not None + return tokenizer + + +@pytest.mark.parametrize("padding_side", ["left", "right"]) +def test_no_bos_token_returns_tokens_unchanged(no_bos_tokenizer, padding_side) -> None: + """There is no BOS to remove, so the tokens must come back untouched.""" + no_bos_tokenizer.padding_side = padding_side + tokens = torch.tensor([[101, 19082, 1362, 102]]) + + result = get_tokens_with_bos_removed(no_bos_tokenizer, tokens) + + torch.testing.assert_close(result, tokens) + + +def test_no_bos_token_does_not_drop_cls_under_right_padding(no_bos_tokenizer) -> None: + """The damaging case: [CLS] is not a BOS token, and dropping it changes what + the model is asked to encode.""" + no_bos_tokenizer.padding_side = "right" + tokens = no_bos_tokenizer("hello world", return_tensors="pt")["input_ids"] + + result = get_tokens_with_bos_removed(no_bos_tokenizer, tokens) + + assert result.shape == tokens.shape + assert result[0, 0].item() == no_bos_tokenizer.cls_token_id + + +def test_no_bos_token_does_not_raise_under_left_padding(no_bos_tokenizer) -> None: + """Previously `(tokens == None).int()` — a Python bool, not a tensor.""" + no_bos_tokenizer.padding_side = "left" + tokens = torch.tensor([[101, 19082, 1362, 102]]) + + result = get_tokens_with_bos_removed(no_bos_tokenizer, tokens) + + assert result.shape == tokens.shape + + +@pytest.mark.parametrize("padding_side", ["left", "right"]) +def test_a_real_bos_is_still_removed(bos_tokenizer, padding_side) -> None: + """The guard must not disturb the case the helper exists for.""" + bos_tokenizer.padding_side = padding_side + bos = bos_tokenizer.bos_token_id + tokens = torch.tensor([[bos, 15496, 995]]) + + result = get_tokens_with_bos_removed(bos_tokenizer, tokens) + + assert result.shape[-1] == tokens.shape[-1] - 1 + assert bos not in result[0].tolist() diff --git a/tests/unit/utilities/test_multi_gpu_unit.py b/tests/unit/utilities/test_multi_gpu_unit.py index 5924c74762..f6b180b879 100644 --- a/tests/unit/utilities/test_multi_gpu_unit.py +++ b/tests/unit/utilities/test_multi_gpu_unit.py @@ -5,13 +5,17 @@ import pytest import torch +import torch.nn as nn from transformer_lens.utilities import ( calculate_available_device_cuda_memory, determine_available_memory_for_available_devices, sort_devices_based_on_available_memory, ) -from transformer_lens.utilities.multi_gpu import get_device_for_block_index +from transformer_lens.utilities.multi_gpu import ( + cast_floating_params_to_dtype, + get_device_for_block_index, +) def mock_available_devices(memory_stats: list[tuple[int, int]]): @@ -129,3 +133,127 @@ def test_cpu_device_is_returned_unchanged(self): cfg = _cuda_cfg(n_layers=62, n_devices=8) result = get_device_for_block_index(30, cfg, device="cpu") assert result.type == "cpu" + + +class TestCastFloatingParamsToDtype: + """Regression tests for cast_floating_params_to_dtype. + + See: https://github.com/TransformerLensOrg/TransformerLens/issues/1713 + The function was casting quantizer-owned FP8 scale tensors (float8_e8m0fnu) + to bfloat16, which corrupts the weight/scale pair relationship and breaks + MXFP4 checkpoints. + """ + + def test_casts_standard_floats_to_target_dtype(self): + """Positive control: standard float dtypes should be cast.""" + model = nn.Linear(4, 4) + model.weight = nn.Parameter(torch.zeros(4, 4, dtype=torch.float32)) + cast_floating_params_to_dtype(model, torch.bfloat16) + assert model.weight.dtype == torch.bfloat16 + + def test_skips_params_already_at_target_dtype(self): + """Params already at target dtype are left untouched.""" + model = nn.Linear(4, 4) + original = torch.zeros(4, 4, dtype=torch.bfloat16) + model.weight = nn.Parameter(original) + cast_floating_params_to_dtype(model, torch.bfloat16) + assert model.weight.dtype == torch.bfloat16 + assert model.weight.data_ptr() == original.data_ptr() + + def test_skips_non_floating_point_params(self): + """Integer params (packed quantized weights) are left untouched.""" + model = nn.Linear(4, 4, bias=False) + model.weight = nn.Parameter(torch.zeros(4, 4, dtype=torch.int8), requires_grad=False) + cast_floating_params_to_dtype(model, torch.bfloat16) + assert model.weight.dtype == torch.int8 + + @pytest.mark.parametrize( + "fp8_dtype", + [ + torch.float8_e4m3fn, + torch.float8_e5m2, + pytest.param( + getattr(torch, "float8_e8m0fnu", None), + marks=pytest.mark.skipif( + not hasattr(torch, "float8_e8m0fnu"), reason="torch < 2.7" + ), + ), + ], + ) + def test_skips_one_byte_floats_fp8_scales(self, fp8_dtype): + """FP8 scale tensors must NOT be cast — they are quantizer-owned. + + This is the key regression test for the MXFP4 bug: casting float8_e8m0fnu + scales to bfloat16 breaks the weight/scale pair relationship. + """ + model = nn.Linear(4, 4, bias=False) + model.weight = nn.Parameter(torch.zeros(4, 4, dtype=fp8_dtype), requires_grad=False) + cast_floating_params_to_dtype(model, torch.bfloat16) + assert model.weight.dtype == fp8_dtype + + def test_mixed_module_casts_selectively(self): + """A module with both standard and FP8 params: only standard params cast.""" + + class MixedModule(nn.Module): + def __init__(self): + super().__init__() + self.standard_weight = nn.Parameter(torch.zeros(4, 4, dtype=torch.float32)) + self.fp8_scale = nn.Parameter( + torch.zeros(4, 4, dtype=torch.float8_e4m3fn), requires_grad=False + ) + self.packed_weight = nn.Parameter( + torch.zeros(4, 4, dtype=torch.int8), requires_grad=False + ) + + model = MixedModule() + cast_floating_params_to_dtype(model, torch.bfloat16) + + assert model.standard_weight.dtype == torch.bfloat16 + assert model.fp8_scale.dtype == torch.float8_e4m3fn + assert model.packed_weight.dtype == torch.int8 + + +class TestMaybeCastFloatingParams: + """Tests for maybe_cast_floating_params helper. + + See: https://github.com/TransformerLensOrg/TransformerLens/issues/1713 + The helper wraps cast_floating_params_to_dtype with a quantization check, + skipping the cast entirely when the model has an active quantization_config. + """ + + def test_casts_unquantized_model(self): + """Unquantized models should have their params cast.""" + from types import SimpleNamespace + + from transformer_lens.utilities.multi_gpu import maybe_cast_floating_params + + model = nn.Linear(4, 4) + model.weight = nn.Parameter(torch.zeros(4, 4, dtype=torch.float32)) + model.config = SimpleNamespace(quantization_config=None) + + maybe_cast_floating_params(model, torch.bfloat16) + assert model.weight.dtype == torch.bfloat16 + + def test_skips_quantized_model(self): + """Quantized models should NOT have their params cast.""" + from types import SimpleNamespace + + from transformer_lens.utilities.multi_gpu import maybe_cast_floating_params + + model = nn.Linear(4, 4) + model.weight = nn.Parameter(torch.zeros(4, 4, dtype=torch.float32)) + model.config = SimpleNamespace(quantization_config=SimpleNamespace(quant_method="mxfp4")) + + maybe_cast_floating_params(model, torch.bfloat16) + assert model.weight.dtype == torch.float32 # NOT cast + + def test_skips_model_without_config(self): + """Models without a config attribute should be cast (no quantization).""" + from transformer_lens.utilities.multi_gpu import maybe_cast_floating_params + + model = nn.Linear(4, 4) + model.weight = nn.Parameter(torch.zeros(4, 4, dtype=torch.float32)) + # No model.config attribute + + maybe_cast_floating_params(model, torch.bfloat16) + assert model.weight.dtype == torch.bfloat16 diff --git a/tests/unit/utilities/test_tokenize_utils.py b/tests/unit/utilities/test_tokenize_utils.py new file mode 100644 index 0000000000..0a5447f8e6 --- /dev/null +++ b/tests/unit/utilities/test_tokenize_utils.py @@ -0,0 +1,33 @@ +"""Tests for per-call padding-side overrides in tokenization utilities.""" + +from copy import deepcopy + +import torch + +from transformer_lens import utils + + +def test_attention_mask_uses_explicit_padding_side(gpt2_tokenizer) -> None: + tokenizer = deepcopy(gpt2_tokenizer) + tokenizer.pad_token = tokenizer.eos_token + tokenizer.padding_side = "right" + pad = tokenizer.pad_token_id + tokens = torch.tensor([[pad, pad, 10, 11], [pad, 20, 21, 22]]) + + mask = utils.get_attention_mask(tokenizer, tokens, prepend_bos=True, padding_side="left") + + assert torch.equal(mask, torch.tensor([[0, 1, 1, 1], [1, 1, 1, 1]])) + + +def test_bos_removal_uses_explicit_padding_side(gpt2_tokenizer) -> None: + tokenizer = deepcopy(gpt2_tokenizer) + tokenizer.pad_token = tokenizer.eos_token + tokenizer.bos_token = tokenizer.convert_ids_to_tokens(0) + tokenizer.padding_side = "right" + pad = tokenizer.pad_token_id + bos = tokenizer.bos_token_id + tokens = torch.tensor([[pad, bos, 10, 11], [bos, 20, 21, 22]]) + + result = utils.get_tokens_with_bos_removed(tokenizer, tokens, padding_side="left") + + assert torch.equal(result, torch.tensor([[pad, 10, 11], [20, 21, 22]])) diff --git a/transformer_lens/ActivationCache.py b/transformer_lens/ActivationCache.py index 76992cf779..6c2a61122f 100644 --- a/transformer_lens/ActivationCache.py +++ b/transformer_lens/ActivationCache.py @@ -14,6 +14,7 @@ class first, including the examples, and then skimming the available methods. Yo from __future__ import annotations import logging +from collections import Counter from typing import ( TYPE_CHECKING, Any, @@ -152,6 +153,19 @@ def __init__( # Note: model reference prevents garbage collection. Set cache.model = None if unneeded. + def _batch_size(self) -> int: + """The cache's batch size: the most common leading dim across entries. + + Caches may hold non-batch entries alongside genuinely batched + activations — broadcast entries with a leading dim of 1 (e.g. the + bridge's position-index inputs) or position-indexed entries whose + leading dim is the sequence length (e.g. T5's relative position bias). + The batched activations vastly outnumber both, so the mode is the + reliable signal where max/min are not. + """ + counts = Counter(v.size(0) for v in self.cache_dict.values() if v.ndim > 0) + return counts.most_common(1)[0][0] if counts else 1 + def remove_batch_dim(self) -> ActivationCache: """Remove the Batch Dimension (if a single batch item). @@ -159,16 +173,13 @@ def remove_batch_dim(self) -> ActivationCache: The ActivationCache with the batch dimension removed. """ if self.has_batch_dim: - # Skip tensors without a batch dimension - has_batch_1 = any(v.size(0) == 1 for v in self.cache_dict.values()) + batch_size = self._batch_size() + assert ( + batch_size == 1 + ), f"Cannot remove batch dimension from cache with batch size {batch_size}" for key in self.cache_dict: - if self.cache_dict[key].size(0) == 1: + if self.cache_dict[key].ndim > 0 and self.cache_dict[key].size(0) == 1: self.cache_dict[key] = self.cache_dict[key][0] - else: - assert has_batch_1, ( - f"Cannot remove batch dimension from cache with batch size > 1, " - f"for key {key} with shape {self.cache_dict[key].shape}" - ) self.has_batch_dim = False else: logging.warning("Tried removing batch dimension after already having removed it.") @@ -338,8 +349,16 @@ def apply_slice_to_batch_dim(self, batch_slice: Union[Slice, SliceInput]) -> Act self.has_batch_dim or batch_slice.mode == "empty" ), "Cannot index into a cache without a batch dim" still_has_batch_dim = (batch_slice.mode != "int") and self.has_batch_dim + batch_size = self._batch_size() + # Broadcast entries (leading dim 1 when the true batch is larger) are not + # batched — leave them untouched so slicing can't index out of bounds. new_cache_dict = { - name: batch_slice.apply(param, dim=0) for name, param in self.cache_dict.items() + name: ( + batch_slice.apply(param, dim=0) + if param.ndim > 0 and param.size(0) == batch_size + else param + ) + for name, param in self.cache_dict.items() } return ActivationCache(new_cache_dict, self.model, has_batch_dim=still_has_batch_dim) @@ -495,6 +514,7 @@ def accumulated_resid( layer, pos_slice=pos_slice, mlp_input=mlp_input, + has_batch_dim=self.has_batch_dim, recompute_ln=recompute_ln, ) if return_labels: @@ -1376,17 +1396,17 @@ def apply_ln_to_stack( # Logit lens: apply final layer norm to each component with recomputed statistics if recompute_ln and layer == self.model.cfg.n_layers and hasattr(self.model, "ln_final"): ln_final = self.model.ln_final - had_pos_dim = residual_stack.ndim == 4 results = [] for i in range(residual_stack.shape[0]): x = residual_stack[i] - # ln_final expects (batch, pos, d_model); ensure pos dim present + original_shape = x.shape + # ln_final expects (batch, pos, d_model); restore missing structural dimensions + if not has_batch_dim: + x = x.unsqueeze(0) if x.ndim == 2: x = x.unsqueeze(1) out = ln_final(x) - if not had_pos_dim: - out = out.squeeze(1) - results.append(out) + results.append(out.reshape(original_shape)) return torch.stack(results, dim=0) # Center the stack onlny if the model uses LayerNorm diff --git a/transformer_lens/HookedTransformer.py b/transformer_lens/HookedTransformer.py index 7184426b55..ee717982c1 100644 --- a/transformer_lens/HookedTransformer.py +++ b/transformer_lens/HookedTransformer.py @@ -75,6 +75,7 @@ init_xavier_uniform_, softcap_enabled, ) +from transformer_lens.utilities.architectures import POST_NORM_ARCHITECTURES from transformer_lens.utilities.devices import move_to_and_update_config from transformer_lens.weight_processing import ProcessWeights @@ -1435,18 +1436,20 @@ def from_pretrained( "Setting center_writing_weights=False instead." ) center_writing_weights = False - # OLMo 2 post-norm is incompatible with fold_ln/center_writing_weights (pre-norm only) - if cfg.original_architecture == "Olmo2ForCausalLM": + # Post-norm architectures are incompatible with fold_ln/center_writing_weights, + # both of which assume the norm gain sits on a sublayer's input. + if cfg.original_architecture in POST_NORM_ARCHITECTURES: if fold_ln: logging.warning( - "fold_ln=True is incompatible with OLMo 2's post-norm architecture. " - "Setting fold_ln=False." + f"fold_ln=True is incompatible with {cfg.original_architecture}'s " + "post-norm architecture. Setting fold_ln=False." ) fold_ln = False if center_writing_weights: logging.warning( - "center_writing_weights=True is incompatible with OLMo 2's post-norm " - "architecture. Setting center_writing_weights=False." + f"center_writing_weights=True is incompatible with " + f"{cfg.original_architecture}'s post-norm architecture. " + "Setting center_writing_weights=False." ) center_writing_weights = False if center_unembed and softcap_enabled(cfg.output_logits_soft_cap): @@ -1686,12 +1689,7 @@ def load_and_process_state_dict( state_dict = self.fill_missing_keys(state_dict) if fold_ln: - if self.cfg.num_experts and self.cfg.num_experts > 1: - logging.warning( - "You are using MoE, so the layer norm weights can't be folded! Skipping" - ) - fold_ln = False - elif self.cfg.normalization_type not in ["LN", "LNPre", "RMS", "RMSPre"]: + if self.cfg.normalization_type not in ["LN", "LNPre", "RMS", "RMSPre"]: logging.warning( "You are not using LayerNorm or RMSNorm, so the layer norm weights can't be folded! Skipping" ) diff --git a/transformer_lens/benchmarks/AGENTS.md b/transformer_lens/benchmarks/AGENTS.md index be6ad26d0b..a039bfd1e4 100644 --- a/transformer_lens/benchmarks/AGENTS.md +++ b/transformer_lens/benchmarks/AGENTS.md @@ -10,7 +10,7 @@ If an agent is here because the user asked to "update the registry" or "verify a ## What this directory IS for -- The phase-by-phase benchmark implementations (`forward_pass.py`, `generation.py`, `hook_registration.py`, `weight_processing.py`, `multimodal.py`, `audio.py`, `vision.py`, `encoder_common.py`, `text_quality.py`, `granular_weight_processing.py`, `component_outputs.py`, `backward_gradients.py`, `activation_cache.py`, `component_benchmark.py`, `hook_structure.py`). +- The phase-by-phase benchmark implementations (`forward_pass.py`, `generation.py`, `hook_registration.py`, `weight_processing.py`, `multimodal.py`, `audio.py`, `vision.py`, `encoder_common.py`, `text_quality.py`, `text_quality_profiles.py` (Phase-4 prompt-profile data + resolver), `granular_weight_processing.py`, `component_outputs.py`, `backward_gradients.py`, `activation_cache.py`, `component_benchmark.py`, `hook_structure.py`). - `main_benchmark.py` — exploratory benchmark runner for ad-hoc comparison. Useful for debugging a single model's phase scores without touching the registry. - `utils.py` — shared helpers including `BenchmarkSeverity`. diff --git a/transformer_lens/benchmarks/backward_gradients.py b/transformer_lens/benchmarks/backward_gradients.py index 4e75009d74..24d527960b 100644 --- a/transformer_lens/benchmarks/backward_gradients.py +++ b/transformer_lens/benchmarks/backward_gradients.py @@ -14,6 +14,59 @@ from transformer_lens.hook_points import HookPoint from transformer_lens.model_bridge import TransformerBridge +# Grading band for numerical (non-convention) gradient mismatches. Registering +# backward hooks forces normalization off HF's native autograd onto the python +# norm, which shifts results at float-rounding scale; measured noise is ~1e-5 +# rel_l2 with a single over-tolerance element, while injected bugs start at +# ~1e-3 rel_l2 with 60+ elements over. Valid for fp32 gradients only — the +# gradient section upcasts reduced-precision models before comparing. +REL_L2_TOLERANCE = 1e-4 +OVER_TOLERANCE_MAX_ELEMENTS = 3 + + +def needs_fp32_gradients(dtype: Optional[torch.dtype]) -> bool: + """Reduced-precision gradients cannot be graded against the fp32-calibrated + band — bf16's rounding floor alone is ~2e-3 rel_l2, inside the bug band.""" + return dtype is not None and dtype not in (torch.float32, torch.float64) + + +def gradient_mismatch_stats( + bridge_finite: torch.Tensor, + reference_finite: torch.Tensor, + abs_tolerance: float, + rel_tolerance: float, +) -> dict: + """Scale-aware statistics for grading one recorded gradient mismatch. + + A zero reference with a nonzero bridge gradient is the maximally divergent + case, not perfect agreement, so rel_l2 is inf there rather than 0. + """ + bf, rf = bridge_finite.float(), reference_finite.float() + ref_norm = torch.norm(rf) + diff_norm = torch.norm(bf - rf) + if ref_norm > 0: + rel_l2 = (diff_norm / ref_norm).item() + else: + rel_l2 = 0.0 if diff_norm == 0 else float("inf") + over_count = int( + (torch.abs(bf - rf) > abs_tolerance + rel_tolerance * torch.abs(rf)).sum().item() + ) + return {"rel_l2": rel_l2, "over_count": over_count} + + +def gradient_mismatch_is_numerical_noise(rel_l2: float, over_count: int) -> bool: + """True when a gradient mismatch is diffuse and tiny rather than a divergence. + + Elementwise worst-case cannot separate the two: one element of 55k crossing + the tolerance scores the same as a head scaled by 1%. rel_l2 separates them + by 58x or more, and the element COUNT guards the localized case rel_l2 would + dilute. A count (not a fraction) keeps the band reachable on small tensors: + detection guarantees count >= 1, so a fractional guard of 1e-4 was + arithmetically unsatisfiable below 10,000 elements (gemma-3-270m's MQA + hook_rot_k is 6,912). + """ + return rel_l2 <= REL_L2_TOLERANCE and over_count <= OVER_TOLERANCE_MAX_ELEMENTS + def benchmark_backward_hooks( bridge: TransformerBridge, @@ -119,6 +172,7 @@ def benchmark_backward_hooks( ] mismatches = [] + mismatch_stats: dict = {} for hook_name in sorted(common_hooks): if hook_name in excluded_hooks: continue @@ -148,8 +202,16 @@ def benchmark_backward_hooks( mean_diff = torch.mean(torch.abs(bf - rf)).item() rel_diff = torch.abs(bf - rf) / (torch.abs(bf) + 1e-8) mean_rel = rel_diff.mean().item() + # Scale-aware stats for grading. Elementwise worst-case alone + # cannot separate a real divergence from the float-rounding + # shift the python-norm fallback introduces when backward + # hooks force normalization off HF's native autograd path. + stats = gradient_mismatch_stats(bf, rf, abs_tolerance, rel_tolerance) + mismatch_stats[hook_name] = stats mismatches.append( - f"{hook_name}: Value mismatch - max_diff={max_diff:.6f}, mean_diff={mean_diff:.6f}, mean_rel={mean_rel:.6f}" + f"{hook_name}: Value mismatch - max_diff={max_diff:.6f}, mean_diff={mean_diff:.6f}, " + f"mean_rel={mean_rel:.6f}, rel_l2={stats['rel_l2']:.3e}, " + f"over_count={stats['over_count']}" ) tested_hooks = len(common_hooks) - len(excluded_hooks) @@ -169,6 +231,10 @@ def benchmark_backward_hooks( "k_norm", # QK norm: Bridge uses 4D, HT uses 2D (shape convention) "ln1.hook_", "ln2.hook_", + # Sandwich norms (gemma-2/3): same class as ln1/ln2 above, which + # predate them. + "ln1_post.hook_", + "ln2_post.hook_", "ln_final.hook_", "hook_resid_mid", "hook_resid_pre", @@ -180,8 +246,25 @@ def benchmark_backward_hooks( "mlp.hook_pre", "hook_mlp_out", ] + + def within_noise_band(entry: str) -> bool: + """Diffuse, tiny deviation — the fallback's rounding, not a divergence. + + Measured noise across architectures is rel_l2 ~1e-5 with a single + over-tolerance element on the rotary hooks (the only ones outside + the pattern list); injected bugs of a 1% head scale or a 0.1% + uniform scale land at rel_l2 1e-3+ with 60+ elements over. + """ + name = entry.split(":")[0] + stats = mismatch_stats.get(name) + if stats is None: + return False + return gradient_mismatch_is_numerical_noise(stats["rel_l2"], stats["over_count"]) + acceptable_mismatches = [ - m for m in mismatches if any(pattern in m for pattern in acceptable_patterns) + m + for m in mismatches + if any(pattern in m for pattern in acceptable_patterns) or within_noise_band(m) ] if len(acceptable_mismatches) == len(mismatches): @@ -402,6 +485,9 @@ def benchmark_critical_backward_hooks( "k_norm", # QK norm: Bridge uses 4D, HT uses 2D (shape convention) "ln1.hook_", "ln2.hook_", + # Sandwich norms (gemma-2/3): same class as ln1/ln2 above. + "ln1_post.hook_", + "ln2_post.hook_", "hook_resid_pre", "hook_resid_mid", "hook_resid_post", diff --git a/transformer_lens/benchmarks/component_outputs.py b/transformer_lens/benchmarks/component_outputs.py index 989f88760f..af0c0bef72 100644 --- a/transformer_lens/benchmarks/component_outputs.py +++ b/transformer_lens/benchmarks/component_outputs.py @@ -894,6 +894,16 @@ def _run_component( except AttributeError: # Skip this component raise ValueError("Cannot test pos_embed - unclear interface") + elif isinstance(getattr(component, "original_component", component), torch.nn.Embedding): + # Any other embedding table (BERT's token_type_embed) rejects the + # float default: embeddings index with integer ids. The HF side is + # the bare nn.Embedding, the bridge side wraps one. Ids are derived + # from test_input so both sides index identically. + embedding_table = getattr(component, "original_component", component) + assert isinstance(embedding_table, torch.nn.Embedding) # narrowed by the elif + num_ids = int(embedding_table.num_embeddings) + id_input = (test_input.abs().sum(dim=-1) * 1e3).long() % num_ids + return component(id_input) elif component_path == "project_in": # project_in expects word_embed_proj_dim, not d_model. word_embed_proj_dim = getattr(self.cfg, "word_embed_proj_dim", None) diff --git a/transformer_lens/benchmarks/forward_pass.py b/transformer_lens/benchmarks/forward_pass.py index 0f872e2bb0..c405f4e90b 100644 --- a/transformer_lens/benchmarks/forward_pass.py +++ b/transformer_lens/benchmarks/forward_pass.py @@ -8,12 +8,18 @@ from transformer_lens.benchmarks.utils import ( BenchmarkResult, BenchmarkSeverity, + bridge_self_target_loss, compare_scalars, compare_tensors, ) from transformer_lens.model_bridge import TransformerBridge +def _compute_self_target_loss(bridge: TransformerBridge, test_text: str) -> torch.Tensor: + """Compute loss with the tokenized input supplied as explicit labels.""" + return bridge_self_target_loss(bridge, test_text) + + def _is_encoder_decoder(model: torch.nn.Module) -> bool: """Check if a model is an encoder-decoder architecture.""" config = getattr(model, "config", None) @@ -193,7 +199,7 @@ def benchmark_loss_equivalence( BenchmarkResult with comparison details """ try: - bridge_loss = bridge(test_text, return_type="loss") + bridge_loss = _compute_self_target_loss(bridge, test_text) if reference_model is None and reference_loss is None: # No reference - just verify loss is valid diff --git a/transformer_lens/benchmarks/hook_registration.py b/transformer_lens/benchmarks/hook_registration.py index 77b819d25b..3318b65030 100644 --- a/transformer_lens/benchmarks/hook_registration.py +++ b/transformer_lens/benchmarks/hook_registration.py @@ -8,6 +8,7 @@ from transformer_lens.benchmarks.utils import ( BenchmarkResult, BenchmarkSeverity, + bridge_self_target_loss, compare_activation_dicts, compare_scalars, filter_expected_missing_hooks, @@ -696,9 +697,26 @@ def ablation_hook(activation, hook): return activation # Test bridge - bridge_original = bridge(test_text, return_type="loss") + # Encoder-decoder bridges name their stacks; a bare blocks.* hook would + # silently no-op and make the ablation vacuous. + ablation_target = next( + ( + name + for name in ( + "blocks.0.attn.hook_v", + "encoder_blocks.0.attn.hook_v", + "decoder_blocks.0.attn.hook_v", + ) + if name in bridge.hook_dict + ), + "blocks.0.attn.hook_v", + ) + bridge_original = bridge_self_target_loss(bridge, test_text) bridge_ablated = bridge.run_with_hooks( - test_text, return_type="loss", fwd_hooks=[("blocks.0.attn.hook_v", ablation_hook)] + test_text, + return_type="loss", + labels=bridge.to_tokens(test_text), + fwd_hooks=[(ablation_target, ablation_hook)], ) bridge_effect = bridge_ablated - bridge_original diff --git a/transformer_lens/benchmarks/main_benchmark.py b/transformer_lens/benchmarks/main_benchmark.py index 6e531abfd9..cc364e3ecd 100644 --- a/transformer_lens/benchmarks/main_benchmark.py +++ b/transformer_lens/benchmarks/main_benchmark.py @@ -5,10 +5,12 @@ Phase 1: HF + Bridge (unprocessed) - Compare against raw HuggingFace model Phase 2: Bridge (unprocessed) + HT (unprocessed) - Compare unprocessed models Phase 3: Bridge (processed) + HT (processed) - Full compatibility mode testing -Phase 4: Text Quality - Perplexity-based legibility scoring via GPT-2 Medium +Phase 4: Text Quality - profile prompts scored by a pinned judge's perplexity ratio Phase 5: Granular Weight Processing Tests (optional, individual flags) Phase 6: Granular Weight Processing Tests (optional, combined flags) Phase 7: Multimodal Tests (only for multimodal models with pixel_values support) +Phase 8: Audio Tests (only for audio encoder models / audio-conditioned decoders) +Phase 9: Vision Tests (only for vision-only encoder models, e.g. ViT/DeiT) """ import gc @@ -31,9 +33,11 @@ benchmark_backward_hooks, benchmark_critical_backward_hooks, benchmark_gradient_computation, + needs_fp32_gradients, ) from transformer_lens.benchmarks.component_benchmark import benchmark_all_components from transformer_lens.benchmarks.forward_pass import ( + _compute_self_target_loss, benchmark_forward_pass, benchmark_logits_equivalence, benchmark_loss_equivalence, @@ -512,17 +516,20 @@ def add_result(result: BenchmarkResult) -> None: if verbose: print("6. Backward Gradient Benchmarks") - # MPS does not support bfloat16 autograd. Upcast to float32 for gradient tests if needed. + # Gradient comparisons are graded against fp32-calibrated thresholds + # (REL_L2_TOLERANCE): bf16's rounding floor alone is ~2e-3 rel_l2, inside the + # measured bug band, so reduced-precision gradients cannot be graded at all. + # Upcast for the gradient section on every device (MPS additionally lacks + # bf16 autograd), then restore below. bridge_grad_dtype = bridge_model.cfg.dtype if hasattr(bridge_model, "cfg") else None - bridge_device = next(bridge_model.parameters()).device - mps_bf16_upcast = str(bridge_device).startswith("mps") and bridge_grad_dtype == torch.bfloat16 - if mps_bf16_upcast: + grad_fp32_upcast = needs_fp32_gradients(bridge_grad_dtype) + if grad_fp32_upcast: try: bridge_model.to(torch.float32) if reference_model is not None: reference_model.to(torch.float32) except Exception: - mps_bf16_upcast = False # Upcast failed; proceed as-is + grad_fp32_upcast = False # Upcast failed; proceed as-is if ht_available: try: @@ -561,7 +568,7 @@ def add_result(result: BenchmarkResult) -> None: if verbose: print(f"✗ Gradient benchmark failed: {e}\n") - if mps_bf16_upcast and bridge_grad_dtype is not None: + if grad_fp32_upcast and bridge_grad_dtype is not None: try: bridge_model.to(bridge_grad_dtype) if reference_model is not None: @@ -585,8 +592,9 @@ def run_benchmark_suite( test_weight_processing_individually: bool = False, phases: list[int] | None = None, trust_remote_code: bool = False, - scoring_model: PreTrainedModel | None = None, - scoring_tokenizer: PreTrainedTokenizerBase | None = None, + judge_model: PreTrainedModel | None = None, + judge_tokenizer: PreTrainedTokenizerBase | None = None, + prompt_profile: str | None = None, ) -> List[BenchmarkResult]: """Run comprehensive benchmark suite for TransformerBridge. @@ -594,7 +602,7 @@ def run_benchmark_suite( Phase 1: HF + Bridge (unprocessed) - Compare against raw HuggingFace model Phase 2: Bridge (unprocessed) + HT (unprocessed) - Compare unprocessed models Phase 3: Bridge (processed) + HT (processed) - Full compatibility mode testing - Phase 4: Text Quality - Perplexity-based legibility scoring via GPT-2 + Phase 4: Text Quality - profile prompts scored by a pinned judge's perplexity ratio Phase 5: Individual Weight Processing Flags (optional) Phase 6: Combined Weight Processing Flags (optional) @@ -617,9 +625,12 @@ def run_benchmark_suite( tests that check each processing flag individually (default: False) phases: Optional list of phase numbers to run (e.g., [1, 2, 3]). If None, runs all phases. trust_remote_code: Whether to trust remote code for custom architectures. - scoring_model: Optional pre-loaded GPT-2 scoring model for Phase 4. When - provided with scoring_tokenizer, avoids reloading for each model in batch. - scoring_tokenizer: Optional pre-loaded tokenizer for Phase 4 scoring model. + judge_model: Optional pre-loaded Phase-4 judge. When provided with + judge_tokenizer, avoids reloading for each model in batch. + judge_tokenizer: Optional pre-loaded tokenizer for the Phase-4 judge. + prompt_profile: Optional Phase-4 prompt profile (e.g. "chat", + "task:translation@en-de"). Resolved from curation + the registry + when None. Returns: List of BenchmarkResult objects @@ -1221,7 +1232,7 @@ def cleanup_model(model, model_name_str: str): with torch.no_grad(): bridge_logits = bridge_unprocessed(test_text, return_type="logits") phase1_reference.hf_logits = bridge_logits.detach().cpu().clone() - bridge_loss = bridge_unprocessed(test_text, return_type="loss") + bridge_loss = _compute_self_target_loss(bridge_unprocessed, test_text) phase1_reference.hf_loss = bridge_loss.item() phase1_reference.test_text = test_text if needs_upcast: @@ -1356,6 +1367,14 @@ def cleanup_model(model, model_name_str: str): if bridge_bos is not None: ht_prepend_bos = bridge_bos + # HookedTransformer is a causal decoder: loading a masked LM into it runs a + # bidirectional model under a causal mask, so it can never be a valid + # reference — numerical comparisons fall back to the Phase 1 HF logits. + if use_ht_reference and is_masked_lm_model(model_name, trust_remote_code=trust_remote_code): + if verbose: + print("Skipping HookedTransformer reference: masked-LM is not representable causally.") + use_ht_reference = False + # Load HookedTransformer for comparison (after generation benchmarks) ht_model_unprocessed = None if should_run_phase(2) and use_ht_reference: @@ -1426,7 +1445,7 @@ def cleanup_model(model, model_name_str: str): # (e.g., OpenELM). # ======================================================================== - # PHASE 4: Text Quality (GPT-2 perplexity scoring) + # PHASE 4: Text Quality (profile prompts, judge perplexity-ratio scoring) # Runs before Phase 3 so it can reuse bridge_unprocessed (Phase 3 # destructively processes the weights, consuming the bridge). # ======================================================================== @@ -1445,21 +1464,34 @@ def cleanup_model(model, model_name_str: str): and not is_masked_lm_model(model_name, trust_remote_code=trust_remote_code) and not is_audio_model(model_name, trust_remote_code=trust_remote_code) ): + if prompt_profile is None: + from transformer_lens.benchmarks.text_quality_profiles import ( + resolve_profile, + ) + from transformer_lens.tools.model_registry.registry_io import ( + registry_prompt_profile, + ) + + config = getattr(bridge_unprocessed, "original_model", None) + archs = getattr(getattr(config, "config", None), "architectures", None) or [] + prompt_profile = str( + resolve_profile( + model_name, archs[0] if archs else None, registry_prompt_profile(model_name) + ) + ) + if verbose: print(f"\n{'='*80}") - print("PHASE 2.5: Text Quality (GPT-2 perplexity scoring)") + print(f"PHASE 2.5: Text Quality (profile {prompt_profile}, judge ratio scoring)") print(f"{'='*80}\n") try: text_quality_result = benchmark_text_quality( bridge_unprocessed, - test_text, - max_new_tokens=50, - scoring_model_name="gpt2", - pass_threshold=85.0, - device=device, - scoring_model=scoring_model, - scoring_tokenizer=scoring_tokenizer, + prompt_profile, + judge_model=judge_model, + judge_tokenizer=judge_tokenizer, + model_name=model_name, ) text_quality_result.phase = 4 add_result(text_quality_result) @@ -1667,7 +1699,7 @@ def _cleanup_bridge_unprocessed(): _cleanup_bridge_unprocessed() _skip_phase3 = True if verbose: - print("\n⚠ Phase 3 skipped (not in phases list)\n") + print("\n⚠ Phase 3 skipped (excluded by phases filter or adapter applicable_phases)\n") elif is_encoder_decoder_model(model_name): _cleanup_bridge_unprocessed() _skip_phase3 = True @@ -1980,34 +2012,49 @@ def _cleanup_bridge_unprocessed(): return results -def update_model_registry(model_name: str, results: List[BenchmarkResult]) -> bool: +def update_model_registry( + model_name: str, results: List[BenchmarkResult], use_hf_reference: bool = False +) -> bool: """Update the model registry with benchmark results. Args: model_name: The model that was benchmarked results: List of benchmark results + use_hf_reference: Whether the run numerically compared against an HF + reference. Defaults to False so an unstated reference state records + a passing run as PROVISIONAL, never VERIFIED. Returns: True if registry was updated successfully """ from transformer_lens.tools.model_registry.registry_io import ( - STATUS_VERIFIED, + STATUS_FAILED, + STATUS_PROVISIONAL, add_verification_record, update_model_status, ) - # Calculate phase scores (percentage of passed tests per phase) - phase_results: Dict[int, List[bool]] = {1: [], 2: [], 3: []} - for result in results: - if result.phase in phase_results and result.severity != BenchmarkSeverity.SKIPPED: - phase_results[result.phase].append(result.passed) + # Threshold/note logic shared with verify_models so the two paths can't drift. + from transformer_lens.tools.model_registry.verify_models import ( + _build_verified_note, + _check_phase_scores, + _extract_phase_scores, + _extract_prompt_profile, + _pass_status, + _sanitize_note, + ) - phase_scores: Dict[int, Optional[float]] = {} - for phase, passed_list in phase_results.items(): - if passed_list: - phase_scores[phase] = round(sum(passed_list) / len(passed_list) * 100, 1) - else: - phase_scores[phase] = None + phase_scores = _extract_phase_scores(results) + + score_error = _check_phase_scores(phase_scores, results) + if score_error: + status = STATUS_FAILED + note = score_error + else: + status = _pass_status(use_hf_reference) + note = _build_verified_note(phase_scores, results) + if status == STATUS_PROVISIONAL: + note = f"Structural only (no HF reference): {note}" # Try to determine architecture architecture_id = "Unknown" @@ -2024,21 +2071,27 @@ def update_model_registry(model_name: str, results: List[BenchmarkResult]) -> bo updated = update_model_status( model_id=model_name, arch_id=architecture_id, - status=STATUS_VERIFIED, + status=status, phase_scores=phase_scores, + note=note, + sanitize_fn=_sanitize_note, + prompt_profile=_extract_prompt_profile(results), ) - add_verification_record( - model_id=model_name, - arch_id=architecture_id, - notes="Benchmark passed", - verified_by="main_benchmark", - ) + # No history record for provisional runs — VerificationHistory.is_verified() + # treats any record as verified, which would bypass the provisional gate. + if status != STATUS_PROVISIONAL: + add_verification_record( + model_id=model_name, + arch_id=architecture_id, + notes=note, + verified_by="main_benchmark", + sanitize_fn=_sanitize_note, + ) - print( - f"Updated registry for {model_name}: " - f"P1={phase_scores.get(1)}%, P2={phase_scores.get(2)}%, P3={phase_scores.get(3)}%" - ) + label = {STATUS_FAILED: "FAILED", STATUS_PROVISIONAL: "PROVISIONAL"}.get(status, "VERIFIED") + score_parts = ", ".join(f"P{p}={s}%" for p, s in sorted(phase_scores.items())) + print(f"Updated registry for {model_name} ({label}): {score_parts or 'no phase results'}") return updated @@ -2102,7 +2155,9 @@ def main(): ) if args.update_registry: - update_model_registry(args.model, results) + # Same requested-reference state verify_models feeds pass_status(): a + # --no-hf-reference run can only mint PROVISIONAL, never VERIFIED. + update_model_registry(args.model, results, use_hf_reference=not args.no_hf_reference) if __name__ == "__main__": diff --git a/transformer_lens/benchmarks/text_quality.py b/transformer_lens/benchmarks/text_quality.py index aa4c9b4bd9..b281c7dfb6 100644 --- a/transformer_lens/benchmarks/text_quality.py +++ b/transformer_lens/benchmarks/text_quality.py @@ -1,17 +1,22 @@ """Text quality benchmark for TransformerBridge. -Generates text with the bridge model from multiple diverse prompts and scores -each continuation's legibility using GPT-2 as a perplexity-based judge. -Only the generated continuation tokens are scored (prompt tokens are masked), -and a repetition penalty is applied to catch degenerate looping output. - -Generation is seeded for reproducibility, and the scoring model is loaded once -and reused across all prompts. +Generates text the way a real user of the model would (its prompt profile: +chat template, translation source, code, own-language continuation — see +``text_quality_profiles``) and scores each output against a known-good +reference completion with one pinned multilingual judge. The score derives +from the perplexity ratio PPL_judge(generated)/PPL_judge(reference), which +cancels the judge's per-language handicap; a repetition penalty catches +degenerate loops (which the ratio alone rewards) and a length penalty +catches truncated output. + +Generation is seeded per prompt for reproducibility, and the judge is loaded +once (CPU/fp32 always, so scores do not depend on the verifying machine) and +reused across all prompts. """ import gc import math -from typing import List, Optional, Tuple +from typing import Any, List, Optional, Tuple, Union import torch from transformers import ( @@ -21,6 +26,22 @@ PreTrainedTokenizerBase, ) +from transformer_lens.benchmarks.text_quality_profiles import ( + CAPTION_REFERENCES, + JUDGE_CONTEXT_KINDS, + JUDGE_R_FAIL, + LANG_ISO3, + LANG_NAMES, + MAX_NEW_TOKENS_BY_KIND, + NLLB_CODES, + PREPEND_BOS_BY_KIND, + T5_PREFIX_ARCHITECTURES, + TEMPERATURE_BY_KIND, + ProfilePrompt, + ProfileSpec, + p4_pass_threshold, + prompts_for, +) from transformer_lens.benchmarks.utils import ( BenchmarkResult, BenchmarkSeverity, @@ -28,96 +49,74 @@ ) from transformer_lens.model_bridge import TransformerBridge -# Diverse prompts used alongside the caller-provided test_text to get a robust -# quality signal across different domains and styles. -_DEFAULT_PROMPTS = [ - "The theory of relativity explains that", - "In the dense forests of the Amazon,", - "Modern computing relies heavily on", -] - - -def _load_scoring_model( - scoring_model_name: str, - device: str, -) -> Tuple[PreTrainedModel, PreTrainedTokenizerBase]: - """Load the scoring model and tokenizer. - - Separated from perplexity computation so the caller can load once and - reuse across multiple prompts. - """ - tokenizer = AutoTokenizer.from_pretrained(scoring_model_name) - model = AutoModelForCausalLM.from_pretrained(scoring_model_name) - torch.nn.Module.to(model, device) +# The one judge every model is scored with, pinned by revision so a Hub update +# can never silently move every score. Selection + measurements live in +# scripts/text_quality_judge_bakeoff.py; separation is weakest in de/ru, so +# scores there carry wider error bars. +JUDGE_MODEL_ID = "Qwen/Qwen2.5-0.5B" +JUDGE_REVISION = "060db6499f32faf8b98477b0a26969ef7d8b9987" + + +def load_judge() -> Tuple[PreTrainedModel, PreTrainedTokenizerBase]: + """Load the pinned judge on CPU in fp32 (machine-independent scores).""" + tokenizer = AutoTokenizer.from_pretrained(JUDGE_MODEL_ID, revision=JUDGE_REVISION) + model = AutoModelForCausalLM.from_pretrained( + JUDGE_MODEL_ID, revision=JUDGE_REVISION, dtype=torch.float32 + ) + torch.nn.Module.to(model, "cpu") model.eval() return model, tokenizer -def _compute_continuation_perplexity( - prompt: str, - full_text: str, - tokenizer: PreTrainedTokenizerBase, - scoring_model: PreTrainedModel, - device: str, +def _judge_perplexity( + text: str, + context: str, + tokenizer: Any, + judge: Any, ) -> Tuple[float, Optional[str]]: - """Compute perplexity of only the continuation tokens (excluding prompt). - - Prompt tokens are masked with -100 in labels so CrossEntropyLoss ignores - them. This prevents well-formed prompt text from artificially lowering - the perplexity of generated content. - - Args: - prompt: The original input prompt. - full_text: The complete text (prompt + generated continuation). - tokenizer: Pre-loaded tokenizer. - scoring_model: Pre-loaded scoring model. - device: Device string. - - Returns: - Tuple of (perplexity, error_message). error_message is None on success. - """ + """Judge perplexity of ``text``; ``context`` tokens are label-masked so only + ``text`` is scored. Returns (ppl, error).""" try: - encodings = tokenizer(full_text, return_tensors="pt") - input_ids = encodings["input_ids"].to(device) + # Tokenize the pieces separately: tokenizing the concatenated string + # lets BPE merge across the boundary and shifts the label mask into + # the scored text. + text_ids = tokenizer(text, return_tensors="pt")["input_ids"] + context_len = 0 + input_ids = text_ids + if context: + context_ids = tokenizer(context, return_tensors="pt")["input_ids"] + context_len = context_ids.shape[1] + input_ids = torch.cat([context_ids, text_ids], dim=1) + + if text_ids.shape[1] < 2: + return float("inf"), "Scored text too short (< 2 judge tokens)" - # Tokenize just the prompt to find where continuation starts - prompt_encodings = tokenizer(prompt, return_tensors="pt") - prompt_len = prompt_encodings["input_ids"].shape[1] - - # Build labels: -100 for prompt positions, actual ids for continuation labels = input_ids.clone() - labels[0, :prompt_len] = -100 - - continuation_len = input_ids.shape[1] - prompt_len - if continuation_len < 2: - return float("inf"), "Generated continuation too short (< 2 tokens)" + if context_len: + labels[0, :context_len] = -100 with torch.no_grad(): - outputs = scoring_model(input_ids, labels=labels) - loss = outputs.loss.item() - - perplexity = math.exp(loss) - return perplexity, None - + loss = judge(input_ids, labels=labels).loss.item() + return math.exp(loss), None except Exception as e: return float("inf"), f"Perplexity computation failed: {str(e)}" def _compute_repetition_penalty(text: str, ns: Tuple[int, ...] = (2, 3, 4)) -> float: - """Compute a repetition penalty based on n-gram uniqueness ratio. - - Returns a multiplier in [0.0, 1.0] where 1.0 means no repetition and - lower values penalize repetitive text. The penalty is the minimum - unique-n-gram ratio across all checked n-gram sizes. + """Minimum unique-n-gram ratio in [0, 1]; low values mean looping output. - Args: - text: The generated continuation text (prompt excluded). - ns: Tuple of n-gram sizes to check. - - Returns: - Penalty multiplier in [0.0, 1.0]. + Load-bearing under ratio scoring: a degenerate loop has LOW judge + perplexity, so without this multiplier it would score 100. """ words = text.lower().split() + # Scriptio continua (zh/ja): word n-grams are inert exactly where the + # judge rewards loops most, and a single stray space would restore the + # word path — so char n-grams whenever the text is CJK-dominated. + compact = "".join(text.split()) + if compact: + cjk = sum(1 for c in compact if 0x3040 <= ord(c) <= 0x30FF or 0x4E00 <= ord(c) <= 0x9FFF) + if cjk / len(compact) >= 0.3 and len(compact) >= 8: + words = list(compact) if len(words) < 2: return 1.0 @@ -134,23 +133,84 @@ def _compute_repetition_penalty(text: str, ns: Tuple[int, ...] = (2, 3, 4)) -> f return min_ratio -def _perplexity_to_score(perplexity: float) -> float: - """Map continuation perplexity to a 0-100 legibility score. - - Uses: score = 135 - 10 * ln(perplexity), capped to [0, 100]. - Calibrated for continuation-only perplexity (higher than full-text). - A well-functioning model typically gets ppl 40-60 -> score 94-98. - Default pass threshold of 85 corresponds to approximately ppl 150. +def _ratio_to_score(ratio: float) -> float: + """Map generated/reference perplexity ratio to 0-100. - Args: - perplexity: The perplexity value from the scoring model. - - Returns: - Score from 0.0 to 100.0. + score = 100 - 100*ln(ratio)/ln(R_FAIL), clamped: ratio<=1 (as good as the + reference) scores 100, ratio=R_FAIL scores 0, and score 50 falls at + sqrt(R_FAIL) — the geometric midpoint between reference quality and + unambiguously broken output, which keeps the registry's phase-4 floor of + 50 principled. """ - if perplexity <= 0 or math.isinf(perplexity): + if ratio <= 0 or math.isinf(ratio) or math.isnan(ratio): return 0.0 - return max(0.0, min(100.0, 135.0 - 10.0 * math.log(perplexity))) + if ratio <= 1.0: + return 100.0 + return max(0.0, min(100.0, 100.0 - 100.0 * math.log(ratio) / math.log(JUDGE_R_FAIL))) + + +_SCRIPT_RANGES: dict[str, Tuple[Tuple[int, int], ...]] = { + "zh": ((0x4E00, 0x9FFF),), + "ja": ((0x3040, 0x30FF), (0x4E00, 0x9FFF)), + "ar": ((0x0600, 0x06FF),), + "ru": ((0x0400, 0x04FF),), + "hi": ((0x0900, 0x097F),), +} + +_LATIN_STOPWORDS: dict[str, frozenset] = { + "en": frozenset("the and of to is that with for was are it in on".split()), + "fr": frozenset( + "le la les des une est que je pas dans de et il elle un en du au pour sur ne ce se".split() + ), + "es": frozenset("el los una es que no por con para como de en la y se del las".split()), + "de": frozenset( + "der die das und ist nicht ich ein eine mit den von zu im auf f\u00fcr sich".split() + ), + "it": frozenset("il la di che non per una sono del gli e in un le si con".split()), + "nl": frozenset("de het een en is niet ik van dat met op voor aan zijn".split()), + "pt": frozenset("o os uma de e que n\u00e3o para com por em um as dos da".split()), +} + + +def _wrong_language(text: str, lang: str) -> bool: + """Conservatively true only when the text is clearly NOT in ``lang``. + + Ratio scoring alone measures fluency, not language: fluent English output + beats a short German reference and clamps to 100, so an untranslated echo + would otherwise score perfectly. Non-Latin targets check script presence; + Latin targets require zero expected-language stopwords while another + covered language has several. + """ + if lang in ("code", ""): + return False + ranges = _SCRIPT_RANGES.get(lang) + if ranges is not None: + letters = [c for c in text if c.isalpha()] + if not letters: + return False + in_script = sum(1 for c in letters if any(lo <= ord(c) <= hi for lo, hi in ranges)) + return in_script / len(letters) < 0.3 + expected = _LATIN_STOPWORDS.get(lang) + if expected is None: + return False + tokens = [w.strip(".,;:!?\"'()") for w in text.lower().split()] + hits = {code: sum(1 for w in tokens if w in stops) for code, stops in _LATIN_STOPWORDS.items()} + return hits[lang] == 0 and max(hits.values(), default=0) >= 3 + + +def _length_penalty(gen_tokens: int, ref_tokens: int) -> float: + """Penalize output far shorter OR far longer than its reference. + + Neutral band [0.5x, 3x] of reference length. The old 25% floor never + fired once in four validation sweeps — a contentless 13-token chat stub + against a 41-token reference scored 93.6; at a 0.5x floor it drops below + the pass line. The 3x cap is the second net for rambling output the + repetition penalty misses.""" + if ref_tokens <= 0: + return 1.0 + under = gen_tokens / (0.5 * ref_tokens) + over = (3.0 * ref_tokens) / max(gen_tokens, 1) + return max(0.0, min(1.0, under, over)) def _build_caption_test_images(n: int = 3) -> list: @@ -184,7 +244,7 @@ def _build_caption_test_images(n: int = 3) -> list: def _generate_image_conditioned_captions( bridge: TransformerBridge, max_new_tokens: int -) -> List[Tuple[str, str]]: +) -> List[Tuple[int, str]]: """Caption synthetic images for image-conditioned seq2seq (Florence-2 emits nothing text-only, so text-only P4 is uninformative); [] if no processor/PIL.""" processor = getattr(bridge, "processor", None) @@ -202,7 +262,7 @@ def _generate_image_conditioned_captions( is_task_captioner = hasattr(processor, "post_process_generation") task = "" if is_task_captioner else "Describe this image in detail." - samples: List[Tuple[str, str]] = [] + samples: List[Tuple[int, str]] = [] for i, image in enumerate(images): try: inputs = processor(text=task, images=image, return_tensors="pt") @@ -216,48 +276,135 @@ def _generate_image_conditioned_captions( input_ids, max_new_tokens=max_new_tokens, return_type="tokens", **extra ) if isinstance(out, torch.Tensor): - text = bridge.tokenizer.decode(out[0], skip_special_tokens=True).strip() + is_encoder_decoder = bool( + getattr(getattr(bridge, "original_model", None), "config", None) + and getattr(bridge.original_model.config, "is_encoder_decoder", False) + ) + # Decoder-only VLM output is prompt + continuation; scoring the + # fluent prompt as caption text would inflate every sample. + caption_ids = out[0] if is_encoder_decoder else out[0, input_ids.shape[-1] :] + text = bridge.tokenizer.decode(caption_ids, skip_special_tokens=True).strip() if text: - samples.append((f"image_{i}", text)) + samples.append((i, text)) except Exception: continue return samples +def _architecture_id(bridge: Any) -> str: + """First HF architecture name of the wrapped model, or ''.""" + config = getattr(getattr(bridge, "original_model", None), "config", None) + architectures = getattr(config, "architectures", None) or [] + return architectures[0] if architectures else "" + + +def _resolve_lang_code(tokenizer, lang: str) -> Optional[str]: + """The tokenizer's own code string for ``lang`` ("de" / "de_DE" / + "deu_Latn"), or None. transformers 5.x NllbTokenizer exposes neither + get_lang_id nor lang_code_to_id, so candidates are probed through the + vocab as well.""" + lang = lang.lower() + lang_code_to_id = getattr(tokenizer, "lang_code_to_id", None) + if isinstance(lang_code_to_id, dict): + if lang in lang_code_to_id: + return lang + iso3 = LANG_ISO3.get(lang, "") + for code in lang_code_to_id: + code_lower = code.lower() + if code_lower.startswith(lang + "_") or (iso3 and code_lower.startswith(iso3 + "_")): + return code + # Vocab probing is only safe for DISTINCTIVE code forms ("deu_Latn"): a + # bare ISO code collides with ordinary subwords (T5's "de", Marian's "en") + # and would be injected as a forced decoder token. + nllb = NLLB_CODES.get(lang) + unk_id = getattr(tokenizer, "unk_token_id", None) + convert = getattr(tokenizer, "convert_tokens_to_ids", None) + if nllb and callable(convert): + try: + token_id = convert(nllb) + except Exception: + return None + if isinstance(token_id, int) and token_id >= 0 and token_id != unk_id: + return nllb + return None + + +def _forced_bos_for_target(tokenizer, tgt_lang: str) -> Optional[int]: + """Target-language decoder token for multilingual translators, or None.""" + get_lang_id = getattr(tokenizer, "get_lang_id", None) + if callable(get_lang_id): + try: + return int(get_lang_id(tgt_lang)) + except Exception: + return None + code = _resolve_lang_code(tokenizer, tgt_lang) + if code is None: + return None + lang_code_to_id = getattr(tokenizer, "lang_code_to_id", None) + if isinstance(lang_code_to_id, dict) and code in lang_code_to_id: + return int(lang_code_to_id[code]) + try: + token_id = tokenizer.convert_tokens_to_ids(code) + except Exception: + return None + if ( + isinstance(token_id, int) + and token_id >= 0 + and token_id != getattr(tokenizer, "unk_token_id", None) + ): + return int(token_id) + return None + + +def _build_model_input( + bridge: Any, + spec: ProfileSpec, + prompt: ProfilePrompt, + architecture_id: str, +) -> str: + """Render one profile prompt into the text this model expects.""" + if spec.kind == "chat": + return bridge.tokenizer.apply_chat_template( + [{"role": "user", "content": prompt.prompt}], + add_generation_prompt=True, + tokenize=False, + ) + if spec.kind == "task:translation" and architecture_id in T5_PREFIX_ARCHITECTURES: + src_name = LANG_NAMES.get(spec.src or "en", "English") + tgt_name = LANG_NAMES.get(spec.lang, "German") + return f"translate {src_name} to {tgt_name}: {prompt.prompt}" + if spec.kind == "task:summarization" and architecture_id in T5_PREFIX_ARCHITECTURES: + return f"summarize: {prompt.prompt}" + return prompt.prompt + + def benchmark_text_quality( - bridge: TransformerBridge, - test_text: str, - max_new_tokens: int = 50, - scoring_model_name: str = "gpt2", - pass_threshold: float = 85.0, - device: str = "cpu", - scoring_model: Optional[PreTrainedModel] = None, - scoring_tokenizer: Optional[PreTrainedTokenizerBase] = None, + bridge: Any, + profile: Union[str, ProfileSpec] = "continuation", + *, + max_new_tokens: Optional[int] = None, + judge_model: Optional[Any] = None, + judge_tokenizer: Optional[Any] = None, + model_name: Optional[str] = None, ) -> BenchmarkResult: - """Benchmark text generation quality using continuation-only perplexity scoring. - - Generates text from multiple diverse prompts, scores each continuation using - GPT-2 perplexity (prompt tokens masked), applies a repetition penalty, - and returns the averaged score. - - Args: - bridge: TransformerBridge model to test. - test_text: Primary input prompt (additional diverse prompts are also used). - max_new_tokens: Number of tokens to generate per prompt. - scoring_model_name: HuggingFace model to use as scorer. - pass_threshold: Minimum average score to pass (default 95.0). - device: Device for the scoring model. - scoring_model: Optional pre-loaded scoring model. When provided alongside - scoring_tokenizer, skips loading and avoids cleanup (caller owns lifecycle). - scoring_tokenizer: Optional pre-loaded tokenizer for the scoring model. - - Returns: - BenchmarkResult with quality score details. + """Benchmark text generation quality with profile prompts and reference-ratio scoring. + + Generates from the model's prompt-profile prompts through the real user + path (``bridge.generate``), then scores each output against the prompt's + reference completion via the pinned judge's perplexity ratio, with + repetition and length penalties. """ + if model_name is not None and model_name.lower() == JUDGE_MODEL_ID.lower(): + # Ratio scoring against the judge's own perplexity is self-grading. + return BenchmarkResult( + name="text_quality", + severity=BenchmarkSeverity.SKIPPED, + message=f"P4 skipped: {model_name} is the pinned judge — cannot self-score", + ) _loaded_locally = False - tokenizer = scoring_tokenizer + tokenizer = judge_tokenizer try: - prompts = [test_text] + _DEFAULT_PROMPTS + spec = ProfileSpec.parse(profile) if isinstance(profile, str) else profile # Diffusion LMs produce text through their native sampler; scoring that # text is as meaningful as scoring autoregressive output. @@ -271,95 +418,211 @@ def benchmark_text_quality( message="Skipped: architecture supports no text generation", ) - # Encoder-decoder models (T5/Marian/BART) emit a standalone decoder - # output (translation, summary), not a continuation of the prompt, so - # there is no prompt prefix to mask out — the whole generated text is the - # content to score. (An en-in→en translation whose output ~= the prompt - # length otherwise trips the "continuation too short" guard for every - # prompt and scores 0.) is_encoder_decoder = bool( getattr(getattr(bridge, "original_model", None), "config", None) and getattr(bridge.original_model.config, "is_encoder_decoder", False) ) - # Image-conditioned seq2seq (e.g. Florence-2) emits a 1-token EOS for a - # text-only prompt — it needs pixel_values to produce anything. For those - # we drive real caption generation from test images and score that. is_multimodal = bool(getattr(getattr(bridge, "cfg", None), "is_multimodal", False)) - image_conditioned = is_encoder_decoder and is_multimodal - # Generate text to score (prompt, full_text) - generations: List[Tuple[str, str]] = [] + # Effective-profile adjustments. Image-conditioned seq2seq (Florence-2) + # emits a bare EOS for text-only prompts — caption real images instead. + # A chat profile without a chat template downgrades to continuation; + # never the other direction (base models may ship templates). + adjustment = "" + if is_encoder_decoder and is_multimodal: + spec = ProfileSpec("caption") + elif spec.kind == "chat": + if getattr(bridge.tokenizer, "chat_template", None) is None: + spec = ProfileSpec("continuation", spec.lang) + adjustment = "chat profile downgraded: tokenizer has no chat template" + else: + try: + bridge.tokenizer.apply_chat_template( + [{"role": "user", "content": "probe"}], + add_generation_prompt=True, + tokenize=False, + ) + except Exception as template_error: + spec = ProfileSpec("continuation", spec.lang) + adjustment = f"chat profile downgraded: template raised {template_error!r}" + + denoise_style = "mask" if getattr(bridge.tokenizer, "mask_token", None) else "t5" + profile_prompts = prompts_for(spec, denoise_style=denoise_style) + if profile_prompts is None: + return BenchmarkResult( + name="text_quality", + severity=BenchmarkSeverity.SKIPPED, + message=( + f"P4 skipped: no prompts for profile '{spec}' — file a " + "TransformerLens issue to add coverage in " + "benchmarks/text_quality_profiles.py" + ), + ) + + if max_new_tokens is None: + max_new_tokens = MAX_NEW_TOKENS_BY_KIND.get(spec.kind, 50) + + architecture_id = _architecture_id(bridge) + forced_bos: Optional[int] = None + if spec.kind == "task:translation": + src_lang_attr = getattr(bridge.tokenizer, "src_lang", None) + if src_lang_attr is not None and spec.src: + src_code = _resolve_lang_code(bridge.tokenizer, spec.src) + if src_code is not None: + try: + bridge.tokenizer.src_lang = src_code + except Exception: + pass + forced_bos = _forced_bos_for_target(bridge.tokenizer, spec.lang) + + # Generate: (profile_prompt, generated_text) pairs. Token-level slicing — + # generate() decodes with skip_special_tokens, so the prompt string is + # not reliably a prefix of the output string (chat templates). + generations: List[Tuple[ProfilePrompt, str]] = [] primary_generated = "" - if image_conditioned: + if spec.kind == "caption": with deterministic_rng(): captions = _generate_image_conditioned_captions(bridge, max_new_tokens) if not captions: - # Cannot reach this model's real (image-conditioned) generation — - # skip rather than score its degenerate text-only output. return BenchmarkResult( name="text_quality", severity=BenchmarkSeverity.SKIPPED, message="Skipped: image-conditioned model; image processor/PIL unavailable", ) - # No prompt prefix to mask — the whole caption is the content (handled - # by the is_encoder_decoder path in the scoring loop below). - generations = [("", text) for _, text in captions] + generations = [ + (ProfilePrompt(prompt="", reference=CAPTION_REFERENCES[i]), text) + for i, text in captions + if i < len(CAPTION_REFERENCES) + ] primary_generated = captions[0][1] else: - with deterministic_rng(): - for i, prompt in enumerate(prompts): - generated = generator( - prompt, - max_new_tokens=max_new_tokens, - temperature=0.7, + prepend_bos = PREPEND_BOS_BY_KIND.get(spec.kind) + # Native diffusion samplers take neither return_type nor forced_bos; + # bound-method identity can't detect them (new object per access). + is_autoregressive = getattr(bridge.adapter, "supports_generation", True) + for prompt in profile_prompts: + model_input = _build_model_input(bridge, spec, prompt, architecture_id) + if is_encoder_decoder: + # Encoder input follows the tokenizer's own recipe (lang + # token + trailing ); to_tokens' BOS policy corrupts it + # (m2m100 loops on a stray ). + prompt_ids = bridge.tokenizer(model_input, return_tensors="pt")["input_ids"].to( + bridge.cfg.device ) - if not isinstance(generated, str) or len(generated.strip()) == 0: - continue - generations.append((prompt, generated)) - if i == 0: - primary_generated = generated + else: + prompt_ids = bridge.to_tokens(model_input, prepend_bos=prepend_bos) + gen_kwargs: dict = { + "max_new_tokens": max_new_tokens, + "temperature": TEMPERATURE_BY_KIND.get(spec.kind, 0.7), + } + if is_autoregressive: + gen_kwargs["return_type"] = "tokens" + if forced_bos is not None: + gen_kwargs["forced_bos_token_id"] = forced_bos + # Seeded per prompt so each sample stream is independent of the + # previous prompt's length. + with deterministic_rng(): + out = generator(prompt_ids, **gen_kwargs) + if not isinstance(out, torch.Tensor): + continue + generated_ids = out[0, 1:] if is_encoder_decoder else out[0, prompt_ids.shape[-1] :] + generated = bridge.tokenizer.decode(generated_ids, skip_special_tokens=True) + if spec.kind == "task:denoise" and denoise_style == "t5": + # Splice the fill back so both ratio sides are full + # sentences (bare fragments judge in the thousands). An + # EMPTY fill must stay empty or a dead model inherits the + # near-reference sentence and a free 100. + if generated.strip(): + generated = prompt.prompt.replace("", generated.strip()) + # Empty output is a scored failure (0), not a dropped sample — + # dropping it would average only over the prompts that worked. + generations.append((prompt, generated)) + if not primary_generated: + primary_generated = generated if len(generations) == 0: return BenchmarkResult( name="text_quality", severity=BenchmarkSeverity.DANGER, - message="Generation produced empty output for all prompts", + message="Generation produced no scoreable output for any prompt", passed=False, ) - # Load scoring model if not pre-loaded by caller - if scoring_model is None or tokenizer is None: - scoring_model, tokenizer = _load_scoring_model(scoring_model_name, device) + if judge_model is None or tokenizer is None: + judge_model, tokenizer = load_judge() _loaded_locally = True - # Score each continuation + # Judge context per kind is JUDGE_CONTEXT_KINDS' call. Translation is + # scored jointly: per-sentence judge perplexity on the short pivots is + # unstable (measured spread 4.8-3497), so the samples concatenate into + # one gen/ref pair. + # Captured pre-merge so translation keeps its per-sentence texts. + all_generated_texts = [text for _, text in generations] + + if spec.kind == "task:translation" and len(generations) > 1: + joiner = "" if spec.lang in ("zh", "ja") else " " + joint = ProfilePrompt( + prompt="", + reference=joiner.join(g[0].reference for g in generations), + lang=spec.lang, + ) + generations = [(joint, joiner.join(g[1] for g in generations))] + + sample_lang = spec.lang if spec.kind != "caption" else "en" per_prompt_scores = [] - per_prompt_perplexities = [] + per_prompt_ratios = [] per_prompt_penalties = [] prompt_details_parts = [] - for prompt, full_text in generations: - # For encoder-decoder output there is no prompt-in-continuation to - # mask; score the entire generated sequence. - score_prompt = "" if is_encoder_decoder else prompt - perplexity, error = _compute_continuation_perplexity( - score_prompt, full_text, tokenizer, scoring_model, device - ) - if error is not None: + for prompt, generated in generations: + context = prompt.prompt if spec.kind in JUDGE_CONTEXT_KINDS else "" + + gen_token_count = len(tokenizer(generated)["input_ids"]) if generated.strip() else 0 + if gen_token_count < 2: + # Empty or one-token output is a scored failure, not a dropped + # sample (Florence-style bare EOS, dead generation). + per_prompt_scores.append(0.0) + per_prompt_ratios.append(float("inf")) + per_prompt_penalties.append(0.0) + prompt_details_parts.append("score=0.0 (output < 2 tokens)") + continue + check_lang = prompt.lang if spec.kind != "task:translation" else sample_lang + if _wrong_language(generated, check_lang): + # Fluency-only ratio scoring would rate untranslated or + # wrong-language output above the reference; hard zero. + per_prompt_scores.append(0.0) + per_prompt_ratios.append(float("inf")) + per_prompt_penalties.append(0.0) + prompt_details_parts.append(f"score=0.0 (output not in '{check_lang}')") continue - raw_score = _perplexity_to_score(perplexity) + gen_ppl, gen_err = _judge_perplexity(generated, context, tokenizer, judge_model) + ref_ppl, ref_err = _judge_perplexity(prompt.reference, context, tokenizer, judge_model) + if gen_err is not None: + # The model's own output was unjudgeable — scored failure. + per_prompt_scores.append(0.0) + per_prompt_ratios.append(float("inf")) + per_prompt_penalties.append(0.0) + prompt_details_parts.append(f"score=0.0 ({gen_err})") + continue + if ref_err is not None: + # Our reference failed to judge — a data problem, not the + # model's; exclude the sample and say so. + prompt_details_parts.append(f"excluded (reference: {ref_err})") + continue - # Repetition penalty on continuation only - continuation = full_text[len(score_prompt) :] - rep_penalty = _compute_repetition_penalty(continuation) - adjusted_score = raw_score * rep_penalty + ratio = gen_ppl / ref_ppl if ref_ppl > 0 else float("inf") + rep_penalty = _compute_repetition_penalty(generated) + ref_token_count = len(tokenizer(prompt.reference)["input_ids"]) + len_penalty = _length_penalty(gen_token_count, ref_token_count) + adjusted_score = _ratio_to_score(ratio) * rep_penalty * len_penalty per_prompt_scores.append(adjusted_score) - per_prompt_perplexities.append(perplexity) + per_prompt_ratios.append(ratio) per_prompt_penalties.append(rep_penalty) prompt_details_parts.append( - f"ppl={perplexity:.1f} score={adjusted_score:.1f} rep={rep_penalty:.2f}" + f"ratio={ratio:.2f} ppl={gen_ppl:.1f} ref_ppl={ref_ppl:.1f} " + f"rep={rep_penalty:.2f} len={len_penalty:.2f} score={adjusted_score:.1f}" ) if len(per_prompt_scores) == 0: @@ -372,19 +635,26 @@ def benchmark_text_quality( ) avg_score = sum(per_prompt_scores) / len(per_prompt_scores) - avg_perplexity = sum(per_prompt_perplexities) / len(per_prompt_perplexities) + finite_ratios = [r for r in per_prompt_ratios if math.isfinite(r)] + avg_ratio = sum(finite_ratios) / len(finite_ratios) if finite_ratios else float("inf") avg_rep_penalty = sum(per_prompt_penalties) / len(per_prompt_penalties) + pass_threshold = p4_pass_threshold() details = { "score": round(avg_score, 1), - "avg_perplexity": round(avg_perplexity, 2), + "prompt_profile": str(spec), + "judge_model": JUDGE_MODEL_ID, + "judge_revision": JUDGE_REVISION, + "avg_ratio": round(avg_ratio, 3) if math.isfinite(avg_ratio) else "inf", "avg_repetition_penalty": round(avg_rep_penalty, 2), "num_prompts": len(per_prompt_scores), "per_prompt": " | ".join(prompt_details_parts), - "scoring_model": scoring_model_name, "max_new_tokens": max_new_tokens, "generated_text": primary_generated, + "generated_texts": all_generated_texts, } + if adjustment: + details["profile_adjustment"] = adjustment if avg_score >= pass_threshold: return BenchmarkResult( @@ -392,18 +662,17 @@ def benchmark_text_quality( severity=BenchmarkSeverity.INFO, message=( f"Text quality score: {avg_score:.1f}/100 " - f"(avg perplexity: {avg_perplexity:.1f}, " - f"{len(per_prompt_scores)} prompts)" + f"(profile {spec}, {len(per_prompt_scores)} prompts)" ), details=details, ) - elif avg_score >= 80.0: + elif avg_score >= pass_threshold / 2: return BenchmarkResult( name="text_quality", severity=BenchmarkSeverity.WARNING, message=( f"Text quality score: {avg_score:.1f}/100 " - f"(below {pass_threshold}, avg perplexity: {avg_perplexity:.1f})" + f"(below {pass_threshold:.0f}, profile {spec})" ), details=details, passed=False, @@ -414,8 +683,7 @@ def benchmark_text_quality( severity=BenchmarkSeverity.DANGER, message=( f"Text quality score: {avg_score:.1f}/100 " - f"(avg perplexity: {avg_perplexity:.1f}) " - f"— generated text may be incoherent" + f"(profile {spec}) — generated text may be incoherent" ), details=details, passed=False, @@ -431,13 +699,8 @@ def benchmark_text_quality( finally: if _loaded_locally: - if scoring_model is not None: - del scoring_model + if judge_model is not None: + del judge_model if tokenizer is not None: del tokenizer gc.collect() - if device != "cpu" and torch.cuda.is_available(): - torch.cuda.empty_cache() - if device == "mps" and hasattr(torch, "mps") and hasattr(torch.mps, "empty_cache"): - torch.mps.synchronize() - torch.mps.empty_cache() diff --git a/transformer_lens/benchmarks/text_quality_profiles.py b/transformer_lens/benchmarks/text_quality_profiles.py new file mode 100644 index 0000000000..2b09083bcb --- /dev/null +++ b/transformer_lens/benchmarks/text_quality_profiles.py @@ -0,0 +1,1141 @@ +"""Prompt profiles and reference data for the Phase-4 text-quality benchmark. + +Each verified model is scored on prompts a real user would feed it (its +``prompt_profile``): chat models get their chat template, translation models get +source sentences, code models get code, multilingual models get their own +language. Every prompt carries a known-good reference completion; scoring is the +ratio of judge perplexities PPL(generated)/PPL(reference), which cancels the +judge's per-language handicap. + +Profile resolution is curation-first because Hub metadata is unreliable +(observed live 2026-08-20): ``bigscience/mt0-base`` is mis-tagged +``text-generation``; ``facebook/m2m100_418M`` and ``google/long-t5-tglobal-base`` +have no ``pipeline_tag`` at all; the ``conversational`` tag is added by HF for +*any* repo shipping a chat template, including base models like +``Qwen/Qwen2.5-0.5B``; Helsinki-NLP language tags are unordered, so Marian +direction must come from the model id. Precedence: per-model override > +architecture rule > fetched HF signals > stored registry value > default. + +Pivot sentences are from Tatoeba (https://tatoeba.org, CC BY 2.0 FR); source +sentence ids are noted inline. Everything else is hand-authored. + +This module stays stdlib-only: the registry scraper imports it at scan time. + +Language x kind coverage (prompts exist where marked; uncovered combinations +SKIP with a file-an-issue message, they never score against wrong-language +data): + + kind en fr es de zh ja ru ar hi it nl pt ro code + continuation x x x x x x x x - - - - - x + chat x x x x x x x x - - - - - - + task:instruction x x - - x - - - - - - - - - + task:summarization x x - - x - - - - - - - - - + task:denoise x - - - - - - - - - - - - - + PIVOT (translation) x x x x x x x x x x x x - - + +hi/it/nl/pt have pivot coverage only (translation targets); ro exists only in +NLLB_CODES. Filling continuation/chat for those plus it/nl/pt/hi bake-off +calibration is tracked as a follow-up. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Optional + +PROFILE_KINDS = ( + "continuation", + "chat", + "task:instruction", + "task:translation", + "task:summarization", + "task:denoise", + "caption", +) + + +@dataclass(frozen=True) +class ProfileSpec: + """A parsed prompt profile: what to feed the model and in which language.""" + + kind: str + lang: str = "en" + src: Optional[str] = None # translation source language + + @classmethod + def parse(cls, spec: str) -> "ProfileSpec": + """Parse ``kind[@lang]`` (translation: ``@src-tgt``); '@' because task kinds contain ':'.""" + kind, _, lang = spec.partition("@") + if kind not in PROFILE_KINDS: + raise ValueError(f"Unknown profile kind {kind!r} in {spec!r}") + if not lang: + return cls(kind=kind) + if kind == "task:translation": + src, sep, tgt = lang.partition("-") + if not sep or not src or not tgt: + raise ValueError(f"Translation profile needs '@src-tgt', got {spec!r}") + return cls(kind=kind, lang=tgt, src=src) + return cls(kind=kind, lang=lang) + + def __str__(self) -> str: + if self.kind == "task:translation" and self.src: + return f"{self.kind}@{self.src}-{self.lang}" + if self.lang != "en": + return f"{self.kind}@{self.lang}" + return self.kind + + +@dataclass(frozen=True) +class ProfilePrompt: + """One scored sample: model input and a known-good reference completion.""" + + prompt: str + reference: str + lang: str = "en" + + +DEFAULT_PROFILE = ProfileSpec("continuation", "en") + + +def is_default_profile(profile) -> bool: + """One sparse-encoding rule for every registry writer: the bare default + continuation@en profile is never stored (a lang-tagged continuation is).""" + if isinstance(profile, ProfileSpec): + return profile == DEFAULT_PROFILE + try: + return ProfileSpec.parse(str(profile)) == DEFAULT_PROFILE + except ValueError: + return False + + +# --------------------------------------------------------------------------- +# Pivot sentences (Tatoeba, CC BY 2.0 FR) — index-aligned across languages. +# English #1277 / #1284 / #1315; per-language ids in row comments. +# Feed translation pairs and the judge bake-off's fluent corpus. +# --------------------------------------------------------------------------- + +PIVOT_SENTENCES: dict[str, tuple[str, str, str]] = { + "en": ( # 1277, 1284, 1315 + "I have to go to sleep.", + "I will be back soon.", + "I can't live that kind of life.", + ), + "fr": ( # 373908, 3099, 3131 + "Je dois aller dormir.", + "Je serai bientôt de retour.", + "Je ne peux pas vivre comme ça.", + ), + "es": ( # 2482, 2489, 2521 + "Tengo que irme a dormir.", + "Volveré pronto.", + "No puedo vivir así.", + ), + "de": ( # 1195088, 85, 117 + "Ich muss schlafen.", + "Ich werde bald zurück sein.", + "Ich kann so ein Leben nicht leben.", + ), + "it": ( # 4369, 375118, 2733911 + "Devo andare a dormire.", + "Torno subito.", + "Non posso vivere quel tipo di vita.", + ), + "nl": ( # 5966, 5984, 378741 + "Ik moet gaan slapen.", + "Ik ben zo terug.", + "Ik kan zo niet leven.", + ), + "pt": ( # 182184, 331974, 405254 + "Preciso ir dormir.", + "Voltarei em breve.", + "Eu não posso viver esse tipo de vida.", + ), + "ru": ( # 5410, 374353, 5449 + "Мне пора идти спать.", + "Я скоро вернусь.", + "Я так жить не могу.", + ), + "zh": ( # 2, 9 (Hans transcription), 35 — one script; mixing traditional + # into a simplified-dominant judge destroys that row's zero point. + "我该去睡觉了。", + "我很快就会回来。", + "我不能这样活着。", + ), + "ja": ( # 4703, 4709, 4742 + "私は眠らなければなりません。", + "すぐに戻ります。", + "私はそんな風には生きられない。", + ), + "ar": ( # 372962, 400781, 549626 + "عليّ أن أنام.", + "سأعود قريباً.", + "لا أستطيع أن أعيش حياة كتلك.", + ), + "hi": ( # 3792910, 3793971, 11371181 + "मुझे सोना है।", + "मैं जल्द लौटूंगी।", + "मैं ऐसी जिंदगी नहीं जी सकता।", + ), +} + +# --------------------------------------------------------------------------- +# Continuation prompts. English is seeded from the pre-rework default prompts so +# control-model scores stay comparable. "code" is a language here: code models +# continue code the way prose models continue prose. +# --------------------------------------------------------------------------- + +CONTINUATION_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + "en": ( + ProfilePrompt( + "The theory of relativity explains that", + " time and space are not absolute but depend on the observer's " + "motion, so clocks moving at high speed tick more slowly than " + "clocks at rest.", + ), + ProfilePrompt( + "In the dense forests of the Amazon,", + " thousands of plant and animal species live in a delicate " + "balance, and scientists continue to discover new ones every year.", + ), + ProfilePrompt( + "Modern computing relies heavily on", + " fast processors and large amounts of memory, which allow " + "software to handle enormous quantities of data in real time.", + ), + ProfilePrompt( + "The city library opens early on weekdays, and", + # Judge PPL 8.7 (en median 8.9). References must stay within + # ~3.5x of the language median or this prompt's bar loosens + # proportionally; the integration test pins the band. + " many people stop by in the morning to read or borrow books before work.", + ), + ), + "fr": ( + ProfilePrompt( + "La tour Eiffel est l'un des monuments", + " les plus célèbres du monde, et des millions de visiteurs " + "montent chaque année à son sommet pour admirer Paris.", + lang="fr", + ), + ProfilePrompt( + "Chaque matin, le boulanger du village", + " prépare du pain frais et des croissants que les habitants " + "viennent acheter dès l'ouverture de la boutique.", + lang="fr", + ), + ProfilePrompt( + "La science moderne repose sur", + " l'observation, l'expérience et le raisonnement, qui permettent " + "de comprendre les lois de la nature.", + lang="fr", + ), + ProfilePrompt( + "Pendant l'hiver, les montagnes", + " se couvrent de neige et attirent de nombreux skieurs venus de " "toute l'Europe.", + lang="fr", + ), + ), + "es": ( + ProfilePrompt( + "El clima de la región mediterránea es", + " templado, con veranos secos y calurosos e inviernos suaves y " + "lluviosos, ideal para el cultivo de olivos.", + lang="es", + ), + ProfilePrompt( + "Cada domingo por la mañana, el mercado", + " se llena de gente que compra fruta fresca, verduras y flores a " + "los vendedores locales.", + lang="es", + ), + ProfilePrompt( + "La historia de América Latina está marcada por", + " una gran diversidad cultural, fruto del encuentro entre pueblos " + "indígenas, europeos y africanos.", + lang="es", + ), + ProfilePrompt( + "Los avances de la medicina moderna permiten", + " tratar enfermedades que hace pocas décadas se consideraban " + "incurables, y prolongar la vida de millones de personas.", + lang="es", + ), + ), + "de": ( + ProfilePrompt( + "Der Schwarzwald ist bekannt für", + " seine dichten Wälder, tiefen Täler und traditionellen " + "Bauernhäuser, die jedes Jahr viele Wanderer anziehen.", + lang="de", + ), + ProfilePrompt( + "Jeden Morgen fährt der Zug", + " pünktlich um sieben Uhr vom Hauptbahnhof ab und bringt die " + "Pendler in die umliegenden Städte zur Arbeit.", + lang="de", + ), + ProfilePrompt( + "Die deutsche Sprache hat", + " viele lange zusammengesetzte Wörter, die Lernende oft " + "überraschen, aber einer klaren Logik folgen.", + lang="de", + ), + ProfilePrompt( + "In der modernen Industrie spielen Roboter", + " eine immer größere Rolle, weil sie schwere und gefährliche " + "Arbeiten schneller und sicherer erledigen können.", + lang="de", + ), + ), + "zh": ( + ProfilePrompt( + "长城是中国古代", + "伟大的防御工程，绵延数千公里，每年吸引大量游客前来参观。", + lang="zh", + ), + ProfilePrompt( + "每天早晨，公园里", + "有许多老人打太极拳、散步和下棋，气氛十分热闹。", + lang="zh", + ), + ProfilePrompt( + "现代科技的发展使得", + "人们的生活越来越方便，购物、学习和工作都可以在网上完成。", + lang="zh", + ), + ProfilePrompt( + "春天到了，山上的", + "花都开了，许多家庭趁着周末去郊外踏青赏花。", + lang="zh", + ), + ), + "ja": ( + ProfilePrompt( + "日本の四季は", + "それぞれ美しく、春には桜、秋には紅葉を楽しむために多くの人が旅行に出かけます。", + lang="ja", + ), + ProfilePrompt( + "毎朝、駅の周りには", + "通勤や通学の人々が行き交い、店が次々と開き始めます。", + lang="ja", + ), + ProfilePrompt( + "現代の技術の進歩により、", + "私たちの生活はますます便利になり、買い物も勉強も家にいながらできるようになりました。", + lang="ja", + ), + ProfilePrompt( + "図書館は静かな場所で、", + "学生たちが本を読んだり、勉強したりするのに最適です。", + lang="ja", + ), + ), + "ru": ( + ProfilePrompt( + "Зимой в Сибири", + " очень холодно, температура часто опускается ниже сорока " + "градусов, но местные жители привыкли к таким морозам.", + lang="ru", + ), + ProfilePrompt( + "Каждое утро студенты", + " спешат на занятия в университет, а вечером собираются в " + "библиотеке, чтобы готовиться к экзаменам.", + lang="ru", + ), + ProfilePrompt( + "Современная наука позволяет", + " лечить болезни, которые раньше считались неизлечимыми, и " + "продлевать жизнь миллионам людей.", + lang="ru", + ), + ProfilePrompt( + "Русская литература известна", + " во всём мире благодаря произведениям Толстого, Достоевского и " + "Чехова, которые переведены на десятки языков.", + lang="ru", + ), + ), + "ar": ( + ProfilePrompt( + "تشتهر مدينة القاهرة", + " بتاريخها العريق ومساجدها القديمة وأسواقها الشعبية التي يزورها " + "السياح من جميع أنحاء العالم.", + lang="ar", + ), + ProfilePrompt( + "في كل صباح يذهب الطلاب", + " إلى المدرسة مبكرين، ويقضون اليوم في تعلم القراءة والكتابة " "والعلوم.", + lang="ar", + ), + ProfilePrompt( + "يساعد التقدم العلمي الحديث", + " الأطباء على علاج أمراض كانت تعتبر مستعصية قبل عقود قليلة.", + lang="ar", + ), + ProfilePrompt( + "تعتبر اللغة العربية", + " من أقدم اللغات الحية في العالم، ويتحدث بها ملايين الناس في " "الوطن العربي وخارجه.", + lang="ar", + ), + ), + "code": ( + ProfilePrompt( + 'def is_prime(n):\n """Return True if n is a prime number."""\n', + " if n < 2:\n return False\n" + " for i in range(2, int(n ** 0.5) + 1):\n" + " if n % i == 0:\n return False\n" + " return True\n", + lang="code", + ), + ProfilePrompt( + 'def count_words(text):\n """Count occurrences of each word in text."""\n', + " counts = {}\n for word in text.split():\n" + " counts[word] = counts.get(word, 0) + 1\n" + " return counts\n", + lang="code", + ), + ProfilePrompt( + "def fibonacci(n):\n" ' """Return the first n Fibonacci numbers as a list."""\n', + " result = []\n a, b = 0, 1\n" + " for _ in range(n):\n result.append(a)\n" + " a, b = b, a + b\n return result\n", + lang="code", + ), + ProfilePrompt( + "// Return the largest number in the array.\n" "function findMax(numbers) {\n", + " let max = numbers[0];\n" + " for (const n of numbers) {\n" + " if (n > max) max = n;\n }\n" + " return max;\n}\n", + lang="code", + ), + ), +} + +# --------------------------------------------------------------------------- +# Chat prompts: realistic user turns (rendered through the tokenizer's chat +# template at run time) with a good assistant reply as reference. +# --------------------------------------------------------------------------- + +CHAT_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + "en": ( + ProfilePrompt( + "How do I keep basil alive indoors?", + "Keep basil in a warm spot with at least six hours of sunlight a " + "day, water it when the top of the soil feels dry, and pinch off " + "flower buds so the plant keeps producing leaves.", + ), + ProfilePrompt( + "What's a good way to remember people's names?", + "Repeat the name right after you hear it, use it once or twice in " + "conversation, and link it to something memorable about the " + "person, like their job or where you met.", + ), + ProfilePrompt( + "Explain why the sky is blue in simple terms.", + "Sunlight is made of many colors, and the air scatters blue light " + "more than the other colors because blue travels in shorter " + "waves. When you look up, that scattered blue light is what you " + "see.", + ), + ), + "fr": ( + ProfilePrompt( + "Comment préparer un bon café à la maison ?", + "Utilisez du café fraîchement moulu, une eau à environ 90 degrés " + "et un dosage d'une cuillère à soupe par tasse. Laissez infuser " + "quelques minutes avant de servir.", + lang="fr", + ), + ProfilePrompt( + "Quels sont les avantages de la lecture quotidienne ?", + "Lire chaque jour enrichit le vocabulaire, améliore la " + "concentration et réduit le stress. C'est aussi un excellent " + "moyen de découvrir de nouvelles idées.", + lang="fr", + ), + ProfilePrompt( + "Explique-moi simplement pourquoi les feuilles tombent en automne.", + "En automne, les jours raccourcissent et les arbres reçoivent " + "moins de lumière. Ils cessent de nourrir leurs feuilles, qui " + "changent de couleur puis tombent pour économiser l'énergie " + "pendant l'hiver.", + lang="fr", + ), + ), + "es": ( + ProfilePrompt( + "¿Cómo puedo mejorar mi memoria para estudiar?", + "Estudia en sesiones cortas y regulares, repasa lo aprendido al " + "día siguiente y explica el tema en voz alta con tus propias " + "palabras. Dormir bien también ayuda mucho a fijar los " + "recuerdos.", + lang="es", + ), + ProfilePrompt( + "¿Qué debo tener en cuenta al adoptar un gato?", + "Prepara un espacio tranquilo con comida, agua y un arenero " + "limpio. Dale tiempo para adaptarse, llévalo al veterinario para " + "sus vacunas y juega con él todos los días.", + lang="es", + ), + ProfilePrompt( + "Explícame de forma sencilla cómo funciona un molino de viento.", + "El viento empuja las aspas del molino y las hace girar. Ese giro " + "mueve un eje conectado a una máquina o a un generador, que " + "convierte el movimiento en trabajo útil o en electricidad.", + lang="es", + ), + ), + "de": ( + ProfilePrompt( + "Wie kann ich beim Einkaufen Geld sparen?", + "Schreiben Sie vorher eine Einkaufsliste und halten Sie sich " + "daran, vergleichen Sie Preise und kaufen Sie saisonale " + "Produkte. Große Packungen lohnen sich nur, wenn Sie alles " + "verbrauchen.", + lang="de", + ), + ProfilePrompt( + "Was ist ein guter Weg, eine neue Sprache zu lernen?", + "Üben Sie jeden Tag ein wenig, hören Sie Podcasts oder Musik in " + "der Sprache und sprechen Sie so früh wie möglich mit " + "Muttersprachlern. Regelmäßigkeit ist wichtiger als lange " + "Lerneinheiten.", + lang="de", + ), + ProfilePrompt( + "Erkläre mir einfach, warum es Ebbe und Flut gibt.", + "Der Mond zieht mit seiner Schwerkraft am Wasser der Ozeane. Auf " + "der dem Mond zugewandten Seite der Erde hebt sich das Wasser, " + "und während sich die Erde dreht, wandert dieser Wasserberg — so " + "entstehen Ebbe und Flut.", + lang="de", + ), + ), + "zh": ( + ProfilePrompt( + "怎样才能养成早起的习惯？", + "每天固定同一时间睡觉和起床，睡前少看手机，把闹钟放在离床远一点的地方。坚持两三个星期，身体就会慢慢适应新的作息。", + lang="zh", + ), + ProfilePrompt( + "第一次做饭应该注意什么？", + "先从简单的菜开始，提前准备好所有材料，注意用火安全，切菜时小心手指。做完后记得关闭燃气，慢慢积累经验就会越来越熟练。", + lang="zh", + ), + ProfilePrompt( + "请用简单的话解释为什么会下雨。", + "太阳把地面上的水晒热，水变成水蒸气升到天上，遇冷凝结成小水滴，聚在一起形成云。当水滴越来越重，云托不住它们时，就落下来变成雨。", + lang="zh", + ), + ), + "ja": ( + ProfilePrompt( + "朝型の生活に変えるにはどうすればいいですか？", + "毎日同じ時間に寝起きし、寝る前はスマートフォンを見ないようにしましょう。朝に日光を浴びると体内時計が整い、二、三週間続ければ自然に朝型になります。", + lang="ja", + ), + ProfilePrompt( + "初めての一人暮らしで気をつけることは何ですか?", + "毎月の家賃や食費など生活費の計画を立て、無理のない範囲で貯金をしましょう。防犯のために戸締まりを忘れず、近所のスーパーや病院の場所も早めに確認しておくと安心です。", + lang="ja", + ), + ProfilePrompt( + "虹がどうしてできるのか、簡単に説明してください。", + "雨上がりの空気中には小さな水滴がたくさん残っています。太陽の光がその水滴の中で曲がって反射すると、光が七つの色に分かれて見えます。これが虹です。", + lang="ja", + ), + ), + "ru": ( + ProfilePrompt( + "Как научиться рано вставать?", + "Ложитесь и вставайте в одно и то же время каждый день, не " + "смотрите в телефон перед сном и ставьте будильник подальше от " + "кровати. Через пару недель организм привыкнет к новому режиму.", + lang="ru", + ), + ProfilePrompt( + "Что почитать, чтобы полюбить чтение?", + "Начните с коротких книг на темы, которые вам действительно " + "интересны, — детективы, приключения или научно-популярные " + "рассказы. Главное — читать понемногу каждый день и не " + "заставлять себя дочитывать скучное.", + lang="ru", + ), + ProfilePrompt( + "Объясни простыми словами, почему летом жарко, а зимой холодно.", + "Земля вращается вокруг Солнца с наклонённой осью. Летом наше " + "полушарие наклонено к Солнцу, лучи падают прямее и сильнее " + "нагревают землю. Зимой оно отклонено от Солнца, лучи идут под " + "углом и греют слабее.", + lang="ru", + ), + ), + "ar": ( + ProfilePrompt( + "كيف أنظم وقتي أثناء الدراسة؟", + "قسّم يومك إلى فترات قصيرة للدراسة مع فترات راحة منتظمة، وابدأ " + "بأصعب المواد عندما يكون ذهنك صافياً. اكتب قائمة بالمهام كل صباح " + "والتزم بها قدر الإمكان.", + lang="ar", + ), + ProfilePrompt( + "ما هي فوائد المشي اليومي؟", + "المشي كل يوم يقوي القلب والعضلات ويساعد على تخفيف التوتر " + "وتحسين المزاج. كما أنه يساعد على النوم بشكل أفضل ولا يحتاج إلى " + "أي معدات خاصة.", + lang="ar", + ), + ProfilePrompt( + "اشرح لي ببساطة كيف تصنع النحلة العسل.", + "تجمع النحلة رحيق الأزهار وتخزنه في معدة خاصة، ثم تعود إلى " + "الخلية وتسلمه لنحلات أخرى تضيف إليه مواد تحوله إلى عسل. بعد ذلك " + "يوضع العسل في الأقراص الشمعية ويجفف بتحريك الأجنحة حتى ينضج.", + lang="ar", + ), + ), +} + +# --------------------------------------------------------------------------- +# Task prompts. +# --------------------------------------------------------------------------- + +SUMMARIZATION_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + "en": ( + ProfilePrompt( + "The city council voted on Tuesday to approve funding for a new " + "public library in the downtown district. The project, which has " + "been debated for over two years, will cost an estimated twelve " + "million dollars and is expected to open in the spring of 2028. " + "Supporters argued that the current library, built in 1962, is " + "too small and lacks modern facilities. Opponents raised " + "concerns about the cost and the loss of a parking lot at the " + "proposed site. The mayor said the new building would include " + "community meeting rooms, a children's wing, and free computer " + "access for residents.", + "The city council approved a twelve million dollar downtown " + "library, expected to open in spring 2028, replacing the " + "outdated 1962 building despite concerns over cost and parking.", + ), + ProfilePrompt( + "Researchers at a European university have published a study " + "showing that regular walking can significantly improve sleep " + "quality in adults over sixty. The study followed four hundred " + "participants for one year, half of whom walked for thirty " + "minutes a day while the other half kept their usual habits. " + "Those in the walking group fell asleep faster, woke less often " + "during the night, and reported feeling more rested in the " + "morning. The researchers noted that the benefits appeared " + "within the first two months and lasted for the rest of the " + "study.", + "A year-long study of four hundred older adults found that " + "walking thirty minutes daily improved sleep quality within two " + "months, helping participants fall asleep faster and wake less " + "often.", + ), + ProfilePrompt( + "A severe storm swept through the coastal region on Friday " + "night, leaving thousands of homes without electricity and " + "forcing the closure of the main highway. Emergency crews worked " + "through the weekend to clear fallen trees and restore power " + "lines. Officials said no serious injuries were reported, though " + "several boats were damaged in the harbor. Schools in the area " + "remained closed on Monday while cleanup continued, and " + "residents were advised to avoid the beachfront until inspectors " + "declared it safe.", + "A Friday night storm cut power to thousands of coastal homes " + "and closed the main highway; crews restored services over the " + "weekend with no serious injuries reported.", + ), + ), +} + +INSTRUCTION_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + "en": ( + ProfilePrompt( + "List three things to pack for a day hike.", + "Water, snacks, and a map of the trail.", + ), + ProfilePrompt( + "Write one sentence describing what a lighthouse does.", + "A lighthouse shines a bright light to guide ships safely along " "the coast at night.", + ), + ProfilePrompt( + "Name the four seasons of the year.", + "Spring, summer, autumn, and winter.", + ), + ), +} + +# Pretrained-only seq2seq models (T5, BART) were trained to fill masked spans, +# not to follow instructions; feed them their native denoising format. +DENOISE_PROMPTS: dict[str, tuple[ProfilePrompt, ...]] = { + # ONE sentinel per "t5" prompt: the whole special-stripped output is the + # fill, spliced back by the runner so both ratio sides are full sentences + # (bare span fragments judge in the thousands). + "t5": ( + ProfilePrompt( + "The children in the park until the sun went down.", + "The children played happily in the park until the sun went down.", + ), + ProfilePrompt( + "Every morning she drinks a cup of and reads the newspaper.", + "Every morning she drinks a cup of coffee and reads the newspaper.", + ), + ProfilePrompt( + "The old bridge across the was built many years ago.", + "The old bridge across the river was built many years ago.", + ), + ), + "mask": ( + ProfilePrompt( + "The children played in the park until the sun went down.", + "The children played happily in the park until the sun went down.", + ), + ProfilePrompt( + "Every morning she drinks a cup of and reads the newspaper.", + "Every morning she drinks a cup of coffee and reads the newspaper.", + ), + ProfilePrompt( + "The old bridge across the was built many years ago.", + "The old bridge across the river was built many years ago.", + ), + ), +} + +# References for the synthetic caption images built by the text-quality +# benchmark (index-aligned with _build_caption_test_images). +CAPTION_REFERENCES: tuple[str, ...] = ( + "The image shows a blue rectangle and a green oval on a white background.", + "The image shows a large yellow circle on a black background.", + "The image shows a dark green rectangle and an orange oval on a light " "blue background.", +) + +# --------------------------------------------------------------------------- +# Per-kind generation and judging knobs. +# --------------------------------------------------------------------------- + +MAX_NEW_TOKENS_BY_KIND: dict[str, int] = { + "continuation": 50, + "chat": 64, + "task:instruction": 48, + "task:translation": 48, + "task:summarization": 48, + "task:denoise": 24, + "caption": 50, +} + +# Chat prompts arrive pre-templated (the template supplies its own BOS); +# everything else follows the adapter default. +PREPEND_BOS_BY_KIND: dict[str, Optional[bool]] = { + "chat": False, +} + +# Bake-off-measured scoring anchors; scripts/text_quality_judge_bakeoff.py +# regenerates them (it prints these names verbatim; last run 2026-08-20, full +# 13-domain corpus). R_FAIL = geo-mean of per-language MEDIAN corrupted/fluent +# ratios — a low percentile degenerates below 1 in weak-separation languages. +# R_GOOD = the paraphrase noise floor; score(R_GOOD) is the pass line. +JUDGE_R_FAIL = 18.1 +JUDGE_R_GOOD = 3.74 + + +# Known scale properties (measured during the 2026-08 output audit): +# - Saturation: any ratio <= 1 scores 100 — "at least reference-fluent" is the +# top of the scale, with no resolution above it. +# - Judge family-favoring: the pinned Qwen judge rates Qwen-family models a +# few points friendlier than others; watch Qwen entries in campaign reruns. + + +def p4_pass_threshold() -> float: + """The P4 pass line, derived from the bake-off noise floor. The registry + floor imports this so [floor, pass) can never silently diverge again.""" + return round(100.0 - 100.0 * math.log(JUDGE_R_GOOD) / math.log(JUDGE_R_FAIL), 1) + + +# Registry scale marker for phase4_score. Absent = v1 (unpinned GPT-2, +# 135-10*ln(ppl), pass 85); 2 = pinned-judge reference-ratio scale (pass 56). +# The column mixes populations until the backlog is re-run, so every P4 write +# stamps the scale it was measured on. +P4_SCORING_VERSION = 2 + +# Task output is generated greedily — that is how users run translators and +# summarizers, and it removes sampling variance from a single-sample score. +# Open-ended kinds keep sampling (greedy makes base models loop). +TEMPERATURE_BY_KIND: dict[str, float] = { + "continuation": 0.7, + "chat": 0.7, + "task:instruction": 0.0, + "task:translation": 0.0, + "task:summarization": 0.0, + "task:denoise": 0.0, + "caption": 0.0, +} + +# Kinds whose judge PPL is conditioned on the prompt — the relevance signal: +# unconditioned, fluent-but-off-topic or hallucinated output judges as well as +# a real answer. Translation stays unconditioned (cross-lingual conditioning +# is noisy; the language check covers it); caption's source is an image the +# judge cannot read. +JUDGE_CONTEXT_KINDS = frozenset( + {"continuation", "chat", "task:instruction", "task:summarization", "task:denoise"} +) + +# T5-family checkpoints expect a natural-language task prefix on the source. +T5_PREFIX_ARCHITECTURES = frozenset( + { + "T5ForConditionalGeneration", + "T5WithLMHeadModel", + "MT5ForConditionalGeneration", + "LongT5ForConditionalGeneration", + "SwitchTransformersForConditionalGeneration", + "UMT5ForConditionalGeneration", + } +) + +# Full NLLB (flores-200) codes for covered languages; transformers 5.x +# NllbTokenizer resolves them only via convert_tokens_to_ids. +NLLB_CODES: dict[str, str] = { + "en": "eng_Latn", + "fr": "fra_Latn", + "es": "spa_Latn", + "de": "deu_Latn", + "it": "ita_Latn", + "nl": "nld_Latn", + "pt": "por_Latn", + "ru": "rus_Cyrl", + "zh": "zho_Hans", + "ja": "jpn_Jpan", + "ar": "arb_Arab", + "hi": "hin_Deva", + "ro": "ron_Latn", +} + +# ISO 639-3 equivalents for NLLB-style language codes ("deu_Latn"). +LANG_ISO3: dict[str, str] = { + "en": "eng", + "fr": "fra", + "es": "spa", + "de": "deu", + "it": "ita", + "nl": "nld", + "pt": "por", + "ru": "rus", + "zh": "zho", + "ja": "jpn", + "ar": "ara", + "hi": "hin", + "ro": "ron", +} + +LANG_NAMES: dict[str, str] = { + "en": "English", + "fr": "French", + "es": "Spanish", + "de": "German", + "it": "Italian", + "nl": "Dutch", + "pt": "Portuguese", + "ru": "Russian", + "zh": "Chinese", + "ja": "Japanese", + "ar": "Arabic", + "hi": "Hindi", + "ro": "Romanian", +} + +# --------------------------------------------------------------------------- +# Curation: architecture rules and per-model overrides. +# --------------------------------------------------------------------------- + +# Architectures whose task is unambiguous. T5/BART/Switch and Falcon/MPT are +# deliberately absent: their task depends on the checkpoint, so they resolve +# through overrides or fetched Hub signals. +ARCHITECTURE_PROFILE_KINDS: dict[str, str] = { + "MarianMTModel": "task:translation", + "M2M100ForConditionalGeneration": "task:translation", + "PegasusForConditionalGeneration": "task:summarization", + "LEDForConditionalGeneration": "task:summarization", + "BlenderbotForConditionalGeneration": "chat", + "BlenderbotSmallForConditionalGeneration": "chat", + "GPTBigCodeForCausalLM": "continuation@code", + "CodeGenForCausalLM": "continuation@code", +} + +# An unlabelled seq2seq model cannot continue text; its pretraining task is the +# only prompt it understands. +SEQ2SEQ_FALLBACK_KIND = "task:denoise" + +MODEL_PROFILE_OVERRIDES: dict[str, str] = { + # T5 v1.0 checkpoints were multitask-trained with task prefixes; the WMT + # en-de pair is their canonical supervised task. + "google-t5/t5-small": "task:translation@en-de", + "google-t5/t5-base": "task:translation@en-de", + "google-t5/t5-large": "task:translation@en-de", + "t5-small": "task:translation@en-de", + "t5-base": "task:translation@en-de", + "t5-large": "task:translation@en-de", + # mt0 is instruction-tuned MT5 (Hub mis-tags it text-generation). + "bigscience/mt0-small": "task:instruction", + "bigscience/mt0-base": "task:instruction", + "bigscience/mt0-large": "task:instruction", + # Pretrained-only checkpoints: denoising is their only language. + "google/long-t5-tglobal-base": "task:denoise", + "google/long-t5-local-base": "task:denoise", + # Base model that ships a chat template (Hub tags it conversational). + "Qwen/Qwen2.5-0.5B": "continuation", + # Task depends on the checkpoint for BART (arch rule deliberately absent); + # without a scraped registry profile these canonical ones need curation. + "facebook/bart-large-cnn": "task:summarization", + "facebook/bart-large-xsum": "task:summarization", + # MBart has no arch rule (base checkpoints are denoising pretrains, task + # varies by fine-tune) — the canonical translators are curated instead. + "facebook/mbart-large-50-many-to-many-mmt": "task:translation@en-de", + "facebook/mbart-large-50-one-to-many-mmt": "task:translation@en-de", + "facebook/mbart-large-50-many-to-one-mmt": "task:translation@de-en", + # Indic-language denoiser; English denoise prompts measure the wrong + # thing, so this skips until Indic coverage exists. + "ai4bharat/IndicBART": "task:denoise@hi", + # Code checkpoints on general-purpose architectures. + "Salesforce/codegen-350M-mono": "continuation@code", + "bigcode/starcoderbase-1b": "continuation@code", + "replit/replit-code-v1-3b": "continuation@code", +} + +# --------------------------------------------------------------------------- +# Hub-signal distillation and profile resolution. +# --------------------------------------------------------------------------- + +# Full ISO 639-1 code set, used to pick language codes out of unstructured Hub +# tag lists. Complete on purpose: a dropped code silently reroutes a model to +# English prompts. +ISO_639_1 = frozenset( + "aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce " + "ch co cr cs cu cv cy da de dv dz ee el en eo es et eu fa ff fi fj fo fr " + "fy ga gd gl gn gu gv ha he hi ho hr ht hu hy hz ia id ie ig ii ik io is " + "it iu ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln " + "lo lt lu lv mg mh mi mk ml mn mr ms mt my na nb nd ne ng nl nn no nr nv " + "ny oc oj om or os pa pi pl ps pt qu rm rn ro ru rw sa sc sd se sg si sk " + "sl sm sn so sq sr ss st su sv sw ta te tg th ti tk tl tn to tr ts tt tw " + "ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu".split() +) + +_PIPELINE_TAG_KINDS: dict[str, str] = { + "translation": "task:translation", + "summarization": "task:summarization", + "text2text-generation": "task:denoise", + "text-generation": "continuation", + "image-text-to-text": "caption", + "image-to-text": "caption", +} + +# Hub tags that mark code models (`conversational` is deliberately NOT mapped +# to chat: HF adds it for any repo shipping a chat template, base models +# included). +_CODE_TAGS = frozenset({"code", "code-generation", "coding"}) + + +@dataclass(frozen=True) +class HFSignals: + """Distilled Hub metadata for one model, as fetched by the scraper.""" + + pipeline_tag: Optional[str] = None + languages: tuple[str, ...] = () + tags: tuple[str, ...] = () + + +def extract_languages(card_data_language: object, tags: object) -> tuple[str, ...]: + """Normalize cardData.language (str or list) plus tag-list ISO codes, noise dropped.""" + langs: list[str] = [] + if isinstance(card_data_language, str): + langs.append(card_data_language.lower()) + elif isinstance(card_data_language, (list, tuple)): + langs.extend(str(item).lower() for item in card_data_language) + if isinstance(tags, (list, tuple)): + langs.extend(str(t).lower() for t in tags) + seen: list[str] = [] + for lang in langs: + # The ISO gate alone filters framework/task tag noise ("pytorch", + # "marian", "multilingual" are not ISO 639-1 codes). + if lang in ISO_639_1 and lang not in seen: + seen.append(lang) + if len(seen) >= 8: + break + return tuple(seen) + + +def _marian_pair_from_model_id(model_id: str) -> Optional[tuple[str, str]]: + """Parse opus-mt-{src}-{tgt} from the id; Helsinki-NLP language tags are unordered.""" + name = model_id.rsplit("/", 1)[-1].lower() + if not name.startswith("opus-mt-"): + return None + parts = name[len("opus-mt-") :].split("-") + if len(parts) == 2 and all(len(p) in (2, 3) for p in parts): + return parts[0], parts[1] + return None + + +_CHAT_ID_MARKERS = ("instruct", "-chat", "_chat") + + +def _id_says_chat(model_id: str) -> bool: + """Instruction-tuned checkpoints are used through their chat template; the + id is the only reliable signal (HF's `conversational` tag also covers base + models, and no architecture distinguishes tuned from base).""" + name = model_id.rsplit("/", 1)[-1].lower() + if name.endswith("-it") or "-it-" in name: + return True + return any(marker in name for marker in _CHAT_ID_MARKERS) + + +def _first_covered_language(languages: tuple[str, ...], table: dict) -> Optional[str]: + for lang in languages: + if lang in table: + return lang + return None + + +def profile_from_hf_signals( + model_id: str, + architecture_id: str, + signals: HFSignals, +) -> Optional[ProfileSpec]: + """Distill fetched Hub metadata into a profile, or None when it says nothing.""" + tags_lower = {t.lower() for t in signals.tags} + if tags_lower & _CODE_TAGS: + return ProfileSpec("continuation", "code") + kind = _PIPELINE_TAG_KINDS.get((signals.pipeline_tag or "").lower()) + if kind is None: + return None + if kind == "task:translation": + pair = _marian_pair_from_model_id(model_id) + if pair is not None: + return ProfileSpec(kind, lang=pair[1], src=pair[0]) + non_en = [lang for lang in signals.languages if lang != "en"] + if "en" in signals.languages and non_en: + return ProfileSpec(kind, lang=non_en[0], src="en") + # Direction unknowable from tags alone (tag lists are unordered); + # fall through rather than guess a reversed or identity pair. + return None + lang = _first_covered_language(signals.languages, CONTINUATION_PROMPTS) or "en" + if kind == "continuation": + return ProfileSpec(kind, lang) + return ProfileSpec(kind) + + +def resolve_profile( + model_id: str, + architecture_id: Optional[str], + registry_profile: Optional[str] = None, + signals: Optional[HFSignals] = None, +) -> ProfileSpec: + """Resolve a model's profile: override > architecture rule > live signals > + stored registry value > default (seq2seq falls back to denoising).""" + override = MODEL_PROFILE_OVERRIDES.get(model_id) + if override is not None: + return ProfileSpec.parse(override) + + # Instruction-tuned ids get the chat profile (the runtime downgrades to + # continuation when no chat template actually exists). Checked before the + # signals layer: the `conversational` tag is deliberately not mapped. + if _id_says_chat(model_id) and ARCHITECTURE_PROFILE_KINDS.get(architecture_id or "") is None: + # The heuristic fixes only the KIND; a stored chat profile keeps its + # language or writeback would flatten curation to @en. + if registry_profile: + try: + stored = ProfileSpec.parse(registry_profile) + if stored.kind == "chat": + return stored + except ValueError: + pass + lang = "en" + if signals is not None: + lang = _first_covered_language(signals.languages, CHAT_PROMPTS) or "en" + return ProfileSpec("chat", lang) + + arch_kind = ARCHITECTURE_PROFILE_KINDS.get(architecture_id or "") + if arch_kind is not None: + arch_spec = ProfileSpec.parse(arch_kind) + # The arch rule fixes only the KIND; a stored same-kind profile keeps + # its language so curation survives the writeback round-trip. + if registry_profile: + try: + stored = ProfileSpec.parse(registry_profile) + if stored.kind == arch_spec.kind: + if arch_spec.kind != "task:translation": + return stored + except ValueError: + pass + if arch_spec.kind == "task:translation": + pair = _marian_pair_from_model_id(model_id) + if pair is not None: + return ProfileSpec(arch_spec.kind, lang=pair[1], src=pair[0]) + if signals is not None: + from_signals = profile_from_hf_signals(model_id, architecture_id or "", signals) + if from_signals is not None and from_signals.kind == "task:translation": + return from_signals + if registry_profile: + try: + stored = ProfileSpec.parse(registry_profile) + if stored.kind == "task:translation": + return stored + except ValueError: + pass + return ProfileSpec(arch_spec.kind, lang="de", src="en") + return arch_spec + + if signals is not None: + from_signals = profile_from_hf_signals(model_id, architecture_id or "", signals) + if from_signals is not None: + return from_signals + + if registry_profile: + try: + return ProfileSpec.parse(registry_profile) + except ValueError: + pass + + try: + from transformer_lens.utilities.architectures import classify_architecture + + if architecture_id and classify_architecture(architecture_id) == "seq2seq": + return ProfileSpec.parse(SEQ2SEQ_FALLBACK_KIND) + except ImportError: # pragma: no cover - torch-free scraper environments + pass + return DEFAULT_PROFILE + + +def prompts_for( + spec: ProfileSpec, denoise_style: str = "t5" +) -> Optional[tuple[ProfilePrompt, ...]]: + """Prompt set for a profile, or None when coverage is missing (caller skips + with a file-an-issue message naming the gap).""" + if spec.kind == "continuation": + return CONTINUATION_PROMPTS.get(spec.lang) + if spec.kind == "chat": + return CHAT_PROMPTS.get(spec.lang) + if spec.kind == "task:instruction": + return INSTRUCTION_PROMPTS.get(spec.lang) + if spec.kind == "task:summarization": + return SUMMARIZATION_PROMPTS.get(spec.lang) + if spec.kind == "task:denoise": + # Denoise prompts are English-only; a non-en denoise profile + # (IndicBART) is a coverage gap, not a zero. + if spec.lang not in ("en", ""): + return None + return DENOISE_PROMPTS.get(denoise_style) + if spec.kind == "task:translation": + src = spec.src or "en" + if src not in PIVOT_SENTENCES or spec.lang not in PIVOT_SENTENCES: + return None + return tuple( + ProfilePrompt(prompt=s, reference=t, lang=spec.lang) + for s, t in zip(PIVOT_SENTENCES[src], PIVOT_SENTENCES[spec.lang]) + ) + if spec.kind == "caption": + return tuple(ProfilePrompt(prompt="", reference=ref) for ref in CAPTION_REFERENCES) + return None diff --git a/transformer_lens/benchmarks/utils.py b/transformer_lens/benchmarks/utils.py index 7fa1069ca0..19295e0e0b 100644 --- a/transformer_lens/benchmarks/utils.py +++ b/transformer_lens/benchmarks/utils.py @@ -459,3 +459,13 @@ def format_results(results: List[BenchmarkResult]) -> str: output.append("=" * 80) return "\n".join(output) + + +def bridge_self_target_loss(bridge, test_text: str): + """Loss with the tokenized input as explicit labels. + + Seq2seq bridges refuse label-less return_type="loss" (encoder input_ids are + not decoder targets), so every benchmark loss call routes through here. + """ + labels = bridge.to_tokens(test_text) + return bridge(test_text, labels=labels, return_type="loss") diff --git a/transformer_lens/benchmarks/weight_processing.py b/transformer_lens/benchmarks/weight_processing.py index b755e832dc..71ee968c55 100644 --- a/transformer_lens/benchmarks/weight_processing.py +++ b/transformer_lens/benchmarks/weight_processing.py @@ -8,6 +8,7 @@ from transformer_lens.benchmarks.utils import ( BenchmarkResult, BenchmarkSeverity, + bridge_self_target_loss, is_tiny_test_model, safe_allclose, ) @@ -147,7 +148,7 @@ def benchmark_weight_sharing( """ try: # Get baseline loss - bridge_original = bridge(test_text, return_type="loss") + bridge_original = bridge_self_target_loss(bridge, test_text) if reference_model is not None: reference_original = reference_model(test_text, return_type="loss") @@ -212,7 +213,7 @@ def benchmark_weight_sharing( reference_model.blocks[bridge_attn_idx].attn.W_V[0, :, :] = 0 # Test modified losses - bridge_modified = bridge(test_text, return_type="loss") + bridge_modified = bridge_self_target_loss(bridge, test_text) reference_modified = reference_model(test_text, return_type="loss") bridge_change = bridge_modified - bridge_original @@ -254,7 +255,7 @@ def benchmark_weight_sharing( with torch.no_grad(): ws_attn_block.attn.W_V[0, :, :] = 0 - bridge_modified = bridge(test_text, return_type="loss") + bridge_modified = bridge_self_target_loss(bridge, test_text) change = abs(bridge_modified - bridge_original) # Restore weights @@ -302,7 +303,7 @@ def benchmark_weight_modification( """ try: # Get original loss - original_loss = bridge(test_text, return_type="loss") + original_loss = bridge_self_target_loss(bridge, test_text) # Find first block with attention (hybrid models may not have attn on block 0) wm_attn_blocks = bridge.blocks_with("attn") @@ -334,7 +335,7 @@ def benchmark_weight_modification( # Get modified loss (with error handling to restore weights) try: - modified_loss = bridge(test_text, return_type="loss") + modified_loss = bridge_self_target_loss(bridge, test_text) except Exception as forward_error: # Restore weights before reporting error with torch.no_grad(): @@ -369,7 +370,7 @@ def benchmark_weight_modification( with torch.no_grad(): original_mlp_w = mlp_block.mlp.out.weight.clone() mlp_block.mlp.out.weight[0, :] = 0 - mlp_modified_loss = bridge(test_text, return_type="loss") + mlp_modified_loss = bridge_self_target_loss(bridge, test_text) with torch.no_grad(): mlp_block.mlp.out.weight.copy_(original_mlp_w) mlp_change = abs(mlp_modified_loss - original_loss) diff --git a/transformer_lens/components/transformer_block.py b/transformer_lens/components/transformer_block.py index 7017d0acba..e1ebda8428 100644 --- a/transformer_lens/components/transformer_block.py +++ b/transformer_lens/components/transformer_block.py @@ -25,6 +25,7 @@ from transformer_lens.factories.mlp_factory import MLPFactory from transformer_lens.hook_points import HookPoint from transformer_lens.utilities import repeat_along_head_dimension +from transformer_lens.utilities.architectures import POST_NORM_ARCHITECTURES class TransformerBlock(nn.Module): @@ -154,7 +155,7 @@ def forward( key_input = attn_in value_input = attn_in - if self.cfg.original_architecture in ("Olmo2ForCausalLM", "Olmo3ForCausalLM"): + if self.cfg.original_architecture in POST_NORM_ARCHITECTURES: attn_out = self.attn( query_input=query_input, key_input=key_input, @@ -182,7 +183,7 @@ def forward( # and before the hook. We do it before the hook so hook_attn_out captures "that which # is added to the residual stream" attn_out = self.ln1_post(attn_out) - if self.cfg.original_architecture in ("Olmo2ForCausalLM", "Olmo3ForCausalLM"): + if self.cfg.original_architecture in POST_NORM_ARCHITECTURES: # OLMo 2/3 post-norm: ln1 applies before the residual add, so it must # precede the hook for hook_attn_out to capture the additive contribution. attn_out = self.ln1(attn_out) @@ -196,7 +197,7 @@ def forward( mlp_in = ( resid_mid if not self.cfg.use_hook_mlp_in else self.hook_mlp_in(resid_mid.clone()) ) - if self.cfg.original_architecture in ("Olmo2ForCausalLM", "Olmo3ForCausalLM"): + if self.cfg.original_architecture in POST_NORM_ARCHITECTURES: # Post-norm: apply_mlp applies ln2 before hook_mlp_out internally. mlp_out = self.apply_mlp(mlp_in) else: @@ -228,7 +229,7 @@ def apply_mlp( mlp_out = self.mlp(normalized_resid) # [batch, pos, d_model] if self.cfg.use_normalization_before_and_after: mlp_out = self.ln2_post(mlp_out) - if self.cfg.original_architecture in ("Olmo2ForCausalLM", "Olmo3ForCausalLM"): + if self.cfg.original_architecture in POST_NORM_ARCHITECTURES: # OLMo 2/3 post-norm: ln2 applies before the residual add, so it must # precede the hook for hook_mlp_out to capture the additive contribution. mlp_out = self.ln2(mlp_out) diff --git a/transformer_lens/config/transformer_bridge_config.py b/transformer_lens/config/transformer_bridge_config.py index e09d55e75a..28a3d4c2f3 100644 --- a/transformer_lens/config/transformer_bridge_config.py +++ b/transformer_lens/config/transformer_bridge_config.py @@ -1,7 +1,10 @@ """Configuration class for TransformerBridge.""" -from typing import Optional +import warnings +import weakref +from typing import Any, Optional +import numpy as np import torch from transformer_lens.utilities.activation_functions import SOFTCAP_DISABLED @@ -18,6 +21,17 @@ class TransformerBridgeConfig(TransformerLensConfig): Also includes all HookedTransformerConfig fields for compatibility. """ + __slots__ = ("_bridge_ref",) + + _BRIDGE_MANAGED_HOOK_FLAGS = frozenset( + { + "use_attn_result", + "use_attn_in", + "use_hook_mlp_in", + "use_split_qkv_input", + } + ) + def __init__( self, d_model: int, @@ -109,6 +123,7 @@ def __init__( **kwargs, ): """Initialize TransformerBridgeConfig.""" + object.__setattr__(self, "_bridge_ref", None) super().__init__( d_model=d_model, d_head=d_head, @@ -204,9 +219,48 @@ def __init__( self.vision_num_layers = vision_num_layers self.vision_num_heads = vision_num_heads self.mm_tokens_per_image = mm_tokens_per_image - self.__post_init__() + def __setattr__(self, name: str, value: Any) -> None: + """Route live Bridge hook-flag assignments through their public setters.""" + if name in self._BRIDGE_MANAGED_HOOK_FLAGS: + bridge_ref = getattr(self, "_bridge_ref", None) + bridge = bridge_ref() if bridge_ref is not None else None + if bridge is not None: + getattr(bridge, f"set_{name}")(value) + return + super().__setattr__(name, value) + + def __getstate__(self) -> dict[str, Any]: + """Serialize config data without retaining its live Bridge binding.""" + return self.__dict__.copy() + + def __setstate__(self, state: dict[str, Any]) -> None: + """Restore an unbound config copy.""" + self.__dict__.update(state) + object.__setattr__(self, "_bridge_ref", None) + + def _bind_bridge(self, bridge: Any) -> None: + """Bind runtime hook-flag assignments to a constructed Bridge.""" + bridge_ref = getattr(self, "_bridge_ref", None) + bound_bridge = bridge_ref() if bridge_ref is not None else None + if bound_bridge is None: + object.__setattr__(self, "_bridge_ref", weakref.ref(bridge)) + elif bound_bridge is not bridge: + warnings.warn( + "TransformerBridgeConfig is already bound to another live " + "TransformerBridge; declining to bind it to this instance. " + "Direct assignments to Bridge-managed hook flags will continue " + "to configure the existing TransformerBridge.", + stacklevel=3, + ) + + def _set_bridge_managed_hook_flag(self, name: str, value: bool) -> None: + """Set a managed flag without re-entering the Bridge setter.""" + if name not in self._BRIDGE_MANAGED_HOOK_FLAGS: + raise ValueError(f"Unknown Bridge-managed hook flag: {name}") + object.__setattr__(self, name, value) + def __post_init__(self): """Post-initialization processing.""" # dtype is guaranteed to be set at this point @@ -219,6 +273,18 @@ def __post_init__(self): ): raise ValueError(f"architecture must be a string, got {type(self.architecture)}") + # Resolve the initializer_range sentinel (-1.0 means "not set by the user"). + # Mirrors HookedTransformerConfig.__post_init__ (hooked_transformer_config.py). + # Guarded with getattr: this method also runs once from the dataclass + # parent's __init__, before self.initializer_range is assigned below. + if getattr(self, "initializer_range", None) is not None: + if self.initializer_range < 0 and self.init_mode == "gpt2": + # Roughly copy the GPT-2 value, but proportional to sqrt(1/d_model) + self.initializer_range = 0.8 / np.sqrt(self.d_model) + if self.initializer_range < 0 and self.init_mode != "gpt2": + # This is the gain parameter for the weight initialisation + self.initializer_range = 1.0 + # Call parent's __post_init__ after our validation if hasattr(super(), "__post_init__"): super().__post_init__() diff --git a/transformer_lens/loading_from_pretrained.py b/transformer_lens/loading_from_pretrained.py index 7e5bc58f1d..3b8cac7619 100644 --- a/transformer_lens/loading_from_pretrained.py +++ b/transformer_lens/loading_from_pretrained.py @@ -58,6 +58,7 @@ convert_t5_weights, ) from transformer_lens.supported_models import MODEL_ALIASES, OFFICIAL_MODEL_NAMES +from transformer_lens.utilities.architectures import POST_NORM_ARCHITECTURES from transformer_lens.utilities.heterogeneous_config import het_safe_view from transformer_lens.utilities.hf_utils import get_rotary_pct_from_config from transformer_lens.utilities.quantization import ( @@ -1785,11 +1786,12 @@ def get_pretrained_model_config( ) fold_ln = False - # OLMo 2 uses post-norm (norm after attention/MLP, not before), so folding - # the norm weights into adjacent linear layers is not mathematically valid. - if cfg_dict.get("original_architecture") == "Olmo2ForCausalLM" and fold_ln: + # Post-norm blocks normalize the sublayer output, so folding the norm weights + # into adjacent linear layers is not mathematically valid. + architecture = cfg_dict.get("original_architecture") + if architecture in POST_NORM_ARCHITECTURES and fold_ln: logging.warning( - "fold_ln=True is incompatible with OLMo 2's post-norm architecture. " + f"fold_ln=True is incompatible with {architecture}'s post-norm architecture. " "Setting fold_ln=False." ) fold_ln = False diff --git a/transformer_lens/model_bridge/bridge.py b/transformer_lens/model_bridge/bridge.py index 94896edc45..670e7ef139 100644 --- a/transformer_lens/model_bridge/bridge.py +++ b/transformer_lens/model_bridge/bridge.py @@ -17,6 +17,7 @@ Callable, Dict, FrozenSet, + Iterable, Iterator, List, Literal, @@ -32,6 +33,8 @@ import torch import tqdm from torch import nn +from torch.nn import functional as F +from transformers.tokenization_utils_base import PreTrainedTokenizerBase from transformer_lens import utilities as utils from transformer_lens.ActivationCache import ActivationCache @@ -39,7 +42,10 @@ from transformer_lens.FactoredMatrix import FactoredMatrix from transformer_lens.hook_points import HookIntrospectionMixin, HookPoint from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter -from transformer_lens.model_bridge.component_setup import set_original_components +from transformer_lens.model_bridge.component_setup import ( + refresh_container_state_owners, + set_original_components, +) from transformer_lens.model_bridge.composition_scores import CompositionScores from transformer_lens.model_bridge.exceptions import StopAtLayerException from transformer_lens.model_bridge.generalized_components.base import ( @@ -192,18 +198,13 @@ def __init__(self, model: nn.Module, adapter: ArchitectureAdapter, tokenizer: An tokenizer: The tokenizer to use (required) """ super().__init__() + self._n_params_total = sum(parameter.numel() for parameter in model.parameters()) self.__dict__["original_model"] = model self.adapter = adapter self.cfg = adapter.cfg - self.tokenizer = tokenizer - if self.cfg.d_vocab == -1 and self.tokenizer is not None: - if hasattr(self.tokenizer, "get_vocab"): - vocab = self.tokenizer.get_vocab() - self.cfg.d_vocab = max(vocab.values()) + 1 - elif hasattr(self.tokenizer, "vocab"): - self.cfg.d_vocab = max(self.tokenizer.vocab.values()) + 1 - else: - self.cfg.d_vocab = getattr(self.tokenizer, "vocab_size", 50257) + self._tokenizer = None + if tokenizer is not None: + self.tokenizer = tokenizer # Use the property setter if self.cfg.d_vocab_out == -1: self.cfg.d_vocab_out = self.cfg.d_vocab self.compatibility_mode = False @@ -241,6 +242,63 @@ def __init__(self, model: nn.Module, adapter: ArchitectureAdapter, tokenizer: An # train() recurses, so this stamps the wrappers with the model's mode. original_model.train(original_model.training) self.train(original_model.training) + self.cfg._bind_bridge(self) + + def __setstate__(self, state: dict[str, Any]) -> None: + """Restore runtime config routing after deepcopy or deserialization.""" + super().__setstate__(state) + self.cfg._bind_bridge(self) + + @property + def tokenizer(self) -> Any: + """The tokenizer used for encoding/decoding text.""" + return self._tokenizer + + @tokenizer.setter + def tokenizer(self, value: Any) -> None: + """Set tokenizer and re-run wiring (d_vocab, BOS/EOS detection, padding). + + On initial assignment (during __init__), the boot path has already run + setup_tokenizer, so we skip calling it again. However, we still infer + d_vocab if it wasn't set from the model config (d_vocab == -1). + + On reassignment, we re-run the tokenizer wiring and update d_vocab to + keep cfg in sync with the new tokenizer. + """ + is_reassignment = getattr(self, "_tokenizer", None) is not None + cfg = getattr(self, "cfg", None) + if value is not None and cfg is not None: + if is_reassignment: + from transformer_lens.model_bridge.sources._bridge_builder import ( + detect_tokenizer_bos_eos, + ) + from transformer_lens.model_bridge.sources.transformers import ( + setup_tokenizer, + ) + + value = setup_tokenizer( + value, default_padding_side=getattr(cfg, "default_padding_side", None) + ) + cfg.tokenizer_prepends_bos, cfg.tokenizer_appends_eos = detect_tokenizer_bos_eos( + value + ) + + # Infer d_vocab: on initial assignment only if not set (-1), + # on reassignment always update to match new tokenizer. + # Use getattr for cfg attributes since tests may use SimpleNamespace. + d_vocab = getattr(cfg, "d_vocab", None) + if d_vocab == -1 or is_reassignment: + if hasattr(value, "get_vocab"): + vocab = value.get_vocab() + cfg.d_vocab = max(vocab.values()) + 1 + elif hasattr(value, "vocab"): + cfg.d_vocab = max(value.vocab.values()) + 1 + else: + cfg.d_vocab = getattr(value, "vocab_size", 50257) + d_vocab_out = getattr(cfg, "d_vocab_out", None) + if d_vocab_out == -1 or is_reassignment: + cfg.d_vocab_out = getattr(cfg, "d_vocab", d_vocab_out) + self._tokenizer = value @classmethod def boot_transformers( @@ -336,10 +394,34 @@ def boot_transformers( checkpoint_value=checkpoint_value, ) + @overload + @classmethod + def boot_native( + cls, + config: TransformerBridgeConfig, + tokenizer: Optional[Any] = None, + device: Optional[Union[str, torch.device]] = None, + dtype: Optional[torch.dtype] = None, + model_name: str = "native", + ) -> "TransformerBridge": + ... + + @overload + @classmethod + def boot_native( + cls, + config: Dict[str, Any], + tokenizer: Optional[Any] = None, + device: Optional[Union[str, torch.device]] = None, + dtype: Optional[torch.dtype] = None, + model_name: str = "native", + ) -> "TransformerBridge": + ... + @classmethod def boot_native( cls, - config: Union[TransformerBridgeConfig, dict], + config: Any, tokenizer: Optional[Any] = None, device: Optional[Union[str, torch.device]] = None, dtype: Optional[torch.dtype] = None, @@ -350,6 +432,15 @@ def boot_native( No HuggingFace Hub call, no ``transformers`` import. ``config.init_mode`` and ``config.seed`` control reproducibility. """ + # Impl signature stays Any so this guard is reachable — a Union hint + # would have beartype reject foreign configs with its own error first. + if not isinstance(config, (TransformerBridgeConfig, dict)): + raise TypeError( + "boot_native expected a TransformerBridgeConfig or dict, " + f"got {type(config).__name__}. Construct a TransformerBridgeConfig " + "with the same fields." + ) + import copy as _copy from transformer_lens.config import TransformerBridgeConfig as _Cfg @@ -383,14 +474,16 @@ def boot_native( # Fork RNG around construction + init when seeded so neither nn.Linear's # default reset_parameters nor our scoped init perturb the caller's RNG. - # Unseeded calls let global RNG advance normally. - if cfg.seed is not None: + # When custom init is disabled, construction keeps PyTorch's normal global + # RNG semantics and cfg.seed has no initialization work to control. + if cfg.init_weights and cfg.seed is not None: with torch.random.fork_rng(devices=[]): model = NativeModel(cfg) initialize_native_model(model, cfg) else: model = NativeModel(cfg) - initialize_native_model(model, cfg) + if cfg.init_weights: + initialize_native_model(model, cfg) if device is not None: model = model.to(device) @@ -407,6 +500,22 @@ def boot_native( model_name=model_name, ) + def init_weights(self) -> None: + """Reinitialize a TL-native model in place using the bridge config.""" + from transformer_lens.model_bridge.sources.native.init import ( + initialize_native_model, + ) + from transformer_lens.model_bridge.sources.native.model import NativeModel + + model = self.original_model + if not isinstance(model, NativeModel): + raise RuntimeError( + "TransformerBridge.init_weights() is only supported for TL-native " + "bridges created with TransformerBridge.boot_native(...); this bridge " + f"wraps {type(model).__name__}." + ) + initialize_native_model(model, self.cfg) + @property def original_model(self) -> nn.Module: """Return the wrapped underlying model; raises AttributeError if it was never set.""" @@ -792,18 +901,17 @@ def hook_dict(self) -> dict[str, HookPoint]: @property def n_params_total(self) -> int: - """Total number of parameters in the model, including embeddings, biases, - and layer norm weights. + """Number of parameters in the wrapped model before bridge instrumentation. - Mirrors :attr:`HookedTransformer.n_params_total`. Use this when you want - the actual parameter count for memory budgeting, comparison with - HuggingFace's ``model.num_parameters()``, or alignment with reported - model sizes in papers (e.g. the Pythia suite). + This follows PyTorch's parameter iteration semantics, counting tied + parameters once. Bridge-created split views and synthetic zero tensors + are excluded, so the result can differ from + :attr:`HookedTransformer.n_params_total` and :meth:`tl_parameters`. Returns: - int: ``sum(p.numel() for p in self.parameters())`` + int: Parameter count of the uninstrumented wrapped model. """ - return sum(p.numel() for p in self.parameters()) + return self._n_params_total def clear_hook_registry(self) -> None: """Clear the hook registry and force re-initialization.""" @@ -882,6 +990,10 @@ def __getattr__(self, name: str) -> Any: # Use __dict__ directly to avoid recursion if "_modules" in self.__dict__ and name in self.__dict__["_modules"]: # type: ignore[arg-type] return self.__dict__["_modules"][name] + adapter = self.__dict__.get("adapter") + component_mapping = getattr(adapter, "component_mapping", None) + if component_mapping is not None and name in component_mapping: + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") if "original_model" in self.__dict__ and self.__dict__["original_model"] is not None: try: name_split = name.split(".") @@ -1220,13 +1332,23 @@ def to_tokens( padding_side = getattr(self.tokenizer, "padding_side", "right") tokenizer_prepends_bos = getattr(self.cfg, "tokenizer_prepends_bos", True) if prepend_bos and (not tokenizer_prepends_bos): - input = utils.get_input_with_manually_prepended_bos(self.tokenizer.bos_token, input) + bos = self.tokenizer.bos_token + encodes_atomically = ( + bos is not None + and len(self.tokenizer(bos, add_special_tokens=False)["input_ids"]) == 1 + ) + if encodes_atomically: + input = utils.get_input_with_manually_prepended_bos(bos, input) + # else: the fallback BOS is not an atom in this vocab (e.g. + # '<|endoftext|>' installed on BERT); prepending the string would + # tokenize to subword garbage, so skip rather than pollute the input. if isinstance(input, str): input = [input] tokens = self.tokenizer( input, return_tensors="pt", padding=True, + padding_side=padding_side, truncation=truncate, max_length=self.cfg.n_ctx if truncate else None, )["input_ids"] @@ -1239,7 +1361,9 @@ def to_tokens( while tokens.shape[-1] > 1 and (tokens[:, -1] == self.tokenizer.eos_token_id).all(): tokens = tokens[:, :-1] if not prepend_bos and tokenizer_prepends_bos: - tokens = utils.get_tokens_with_bos_removed(self.tokenizer, tokens) + tokens = utils.get_tokens_with_bos_removed( + self.tokenizer, tokens, padding_side=padding_side + ) if move_to_device: tokens = tokens.to(self.cfg.device) return tokens @@ -1504,6 +1628,26 @@ def _reshape_o(self, w: torch.Tensor) -> torch.Tensor: return w.reshape(self.cfg.n_heads, d_head, self.cfg.d_model) return w + def _expand_kv_heads(self, w: torch.Tensor) -> torch.Tensor: + """Expand stacked grouped K/V weights along the head axis to n_heads. + + GQA models store one K/V projection per key-value head while W_Q/W_O are + per-query-head, so weight circuits must repeat the grouped K/V up to + n_heads before factoring: query head h reads kv head + h // (n_heads // n_kv_heads), i.e. repeat_interleave — the same layout + GroupedQueryAttention.W_K/W_V expose on HookedTransformer. No-op for MHA, + where the head axes already match. + """ + if w.ndim != 4 or w.shape[1] == self.cfg.n_heads: + return w + n_kv_heads = w.shape[1] + if self.cfg.n_heads % n_kv_heads != 0: + raise ValueError( + f"Cannot expand {n_kv_heads} key-value heads to {self.cfg.n_heads} " + f"query heads: n_heads must be a multiple of n_kv_heads." + ) + return w.repeat_interleave(self.cfg.n_heads // n_kv_heads, dim=1) + @property def W_K(self) -> torch.Tensor: """Stack the key weights across all layers.""" @@ -1589,24 +1733,24 @@ def W_E(self) -> torch.Tensor: @property def QK(self): """QK circuit. On hybrids, returns attn layers only (with warning). See QK_for_attn_layers().""" - return FactoredMatrix(self.W_Q, self.W_K.transpose(-2, -1)) + return FactoredMatrix(self.W_Q, self._expand_kv_heads(self.W_K).transpose(-2, -1)) @property def OV(self): """OV circuit. On hybrids, returns attn layers only (with warning). See OV_for_attn_layers().""" - return FactoredMatrix(self.W_V, self.W_O) + return FactoredMatrix(self._expand_kv_heads(self.W_V), self.W_O) def QK_for_attn_layers(self) -> Tuple[List[int], FactoredMatrix]: """QK circuit for attention layers only. Returns (layer_indices, FactoredMatrix).""" q_indices, W_Q = self.stack_params_for("attn", "attn.W_Q", self._reshape_qkv) _, W_K = self.stack_params_for("attn", "attn.W_K", self._reshape_qkv) - return q_indices, FactoredMatrix(W_Q, W_K.transpose(-2, -1)) + return q_indices, FactoredMatrix(W_Q, self._expand_kv_heads(W_K).transpose(-2, -1)) def OV_for_attn_layers(self) -> Tuple[List[int], FactoredMatrix]: """OV circuit for attention layers only. Returns (layer_indices, FactoredMatrix).""" v_indices, W_V = self.stack_params_for("attn", "attn.W_V", self._reshape_qkv) _, W_O = self.stack_params_for("attn", "attn.W_O", self._reshape_o) - return v_indices, FactoredMatrix(W_V, W_O) + return v_indices, FactoredMatrix(self._expand_kv_heads(W_V), W_O) # ------------------------------------------------------------------ # Mechanistic interpretability analysis methods @@ -1733,17 +1877,17 @@ def _stack(attr_path: str, reshape_fn: Optional[Callable] = None) -> torch.Tenso weights = [w.to(target_device) for w in weights] return torch.stack(weights, dim=0) - W_V = _stack("attn.W_V", self._reshape_qkv) + W_V = self._expand_kv_heads(_stack("attn.W_V", self._reshape_qkv)) W_O = _stack("attn.W_O", self._reshape_o) left = FactoredMatrix(W_V, W_O) if mode == "Q": W_Q = _stack("attn.W_Q", self._reshape_qkv) - W_K = _stack("attn.W_K", self._reshape_qkv) + W_K = self._expand_kv_heads(_stack("attn.W_K", self._reshape_qkv)) right = FactoredMatrix(W_Q, W_K.transpose(-2, -1)) elif mode == "K": W_Q = _stack("attn.W_Q", self._reshape_qkv) - W_K = _stack("attn.W_K", self._reshape_qkv) + W_K = self._expand_kv_heads(_stack("attn.W_K", self._reshape_qkv)) right = FactoredMatrix(W_Q, W_K.transpose(-2, -1)).T elif mode == "V": right = left @@ -1869,6 +2013,79 @@ def tl_named_parameters(self) -> Iterator[tuple[str, torch.Tensor]]: """ return iter(self.get_params().items()) + def _accepts_derived_position_ids(self) -> bool: + """Whether it is safe to hand the wrapped model a mask-derived ``position_ids``. + + Two families of model must be left alone, so the injection below is + gated on the target the same way ``output_attentions`` is in + :meth:`run_with_cache`: + + * **Fixed-signature models.** Remote-code forwards such as + ``LLaDAModelLM.forward`` take neither ``position_ids`` nor + ``**kwargs``, so passing it raises ``TypeError`` where the model + previously returned logits. + * **Models that own their position derivation.** mRoPE architectures + (Qwen2-VL, Qwen2.5-VL, Qwen3-VL, GLM-4V) build a 3-D temporal / + height / width index in ``get_rope_index``, and only while + ``position_ids is None``; a supplied 2-D tensor is silently expanded + across all three streams instead. Their derivation already scatters + positions onto attended slots only, so it handles left padding + correctly on its own and needs no help from us. + * **Mask-consuming positional embeddings.** OPT's + ``OPTLearnedPositionalEmbedding.forward`` takes the mask and derives + the same positions we would, so injection buys nothing — but it does + replace the model's own padding-slot convention with ours, which + shows up as a whole-tensor diff. + """ + underlying = getattr(self, "original_model", None) + if underlying is None: + return False + + cached = self.__dict__.get("_derived_position_ids_ok") + if cached is not None and cached[0] is underlying: + return bool(cached[1]) + + def verdict() -> bool: + fwd_params = inspect.signature(underlying.forward).parameters + if "position_ids" not in fwd_params and not any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in fwd_params.values() + ): + return False + + # ``get_rope_index`` lives on the inner text model, not the + # ForConditionalGeneration wrapper that is usually original_model. + for module in ( + underlying, + getattr(underlying, "model", None), + getattr(underlying, "language_model", None), + ): + if module is not None and hasattr(module, "get_rope_index"): + return False + + # Config-level backstop for mRoPE models that spell the derivation + # differently; the section list is what makes positions 3-D. + config = getattr(underlying, "config", None) + for candidate in (config, getattr(config, "text_config", None)): + scaling = getattr(candidate, "rope_scaling", None) + if isinstance(scaling, dict) and "mrope_section" in scaling: + return False + + # A positional embedding that takes the mask derives positions for + # itself. Only embeddings that override nn.Embedding.forward are + # worth inspecting, which keeps this to a handful per model. + for module in underlying.modules(): + if not isinstance(module, nn.Embedding): + continue + if type(module).forward is nn.Embedding.forward: + continue + if "attention_mask" in inspect.signature(module.forward).parameters: + return False + return True + + accepts = verdict() + self.__dict__["_derived_position_ids_ok"] = (underlying, accepts) + return accepts + def forward( self, input: Union[str, List[str], torch.Tensor], @@ -1877,6 +2094,7 @@ def forward( prepend_bos: Optional[bool] = None, padding_side: Optional[str] = None, attention_mask: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, start_at_layer: Optional[int] = None, stop_at_layer: Optional[int] = None, pixel_values: Optional[torch.Tensor] = None, @@ -1891,6 +2109,8 @@ def forward( loss_per_token: Whether to return loss per token prepend_bos: Whether to prepend BOS token padding_side: Which side to pad on + labels: Explicit language-model targets. Encoder-decoder models require + labels for loss; decoder-only models fall back to input IDs when omitted. start_at_layer: Not implemented in TransformerBridge. The bridge delegates to HuggingFace's model.forward() which owns the layer iteration loop, making start_at_layer infeasible without monkey-patching HF internals @@ -1910,7 +2130,18 @@ def forward( Model output based on return_type """ - if return_type in ("loss", "both") and not self.adapter.supports_causal_loss: + model_config = getattr(self.original_model, "config", None) + is_encoder_decoder = bool(getattr(model_config, "is_encoder_decoder", False)) + if return_type in ("loss", "both") and is_encoder_decoder and labels is None: + raise ValueError( + "labels are required for seq2seq return_type='loss' or 'both'; " + "encoder input_ids are not decoder targets" + ) + if ( + return_type in ("loss", "both") + and not is_encoder_decoder + and not self.adapter.supports_causal_loss + ): architecture = self.cfg.architecture or type(self.adapter).__name__ raise NotImplementedError( f"{architecture} does not support TransformerBridge's shifted causal " @@ -1918,6 +2149,9 @@ def forward( "masked-token objective explicitly." ) + if labels is not None: + kwargs["labels"] = labels + if start_at_layer is not None: raise NotImplementedError( "start_at_layer is not supported in TransformerBridge. " @@ -1940,8 +2174,11 @@ def forward( "The bridge only supports stop_at_layer on 'blocks'." ) if hasattr(self, "blocks"): + effective_stop_at_layer = ( + len(self.blocks) + stop_at_layer if stop_at_layer < 0 else stop_at_layer + ) for block in self.blocks: - block._stop_at_layer_idx = stop_at_layer + block._stop_at_layer_idx = effective_stop_at_layer # Map HookedEncoderDecoder-style kwargs to HF-compatible names if "decoder_input" in kwargs: @@ -1952,16 +2189,17 @@ def forward( else: kwargs.pop("one_zero_attention_mask") - # Detect batched list input that will need padding. For this case we force - # left-padding internally and auto-compute attention_mask + position_ids - # (unless the caller passed them explicitly) so pad tokens don't contaminate - # attention or position embeddings. + # Detect batched list input that may need padding. Forward follows the + # requested/tokenizer side; generation separately forces left-padding. _is_batched_list = ( isinstance(input, list) and len(input) > 1 and not getattr(self.cfg, "is_audio_model", False) and not getattr(self.cfg, "is_visual_model", False) ) + _resolved_padding_side = padding_side + if _resolved_padding_side is None and self.tokenizer is not None: + _resolved_padding_side = getattr(self.tokenizer, "padding_side", "right") try: if isinstance(input, (str, list)): @@ -1975,20 +2213,9 @@ def forward( "Visual models require tensor input (pixel values), not text. " "Pass a torch.Tensor or use the pixel_values parameter." ) - if _is_batched_list and padding_side is None: - # Force left-padding so real tokens are flush-right. - _orig_padding_side = self.tokenizer.padding_side - self.tokenizer.padding_side = "left" - try: - input_ids = self.to_tokens( - input, prepend_bos=prepend_bos, padding_side=padding_side - ) - finally: - self.tokenizer.padding_side = _orig_padding_side - else: - input_ids = self.to_tokens( - input, prepend_bos=prepend_bos, padding_side=padding_side - ) + input_ids = self.to_tokens( + input, prepend_bos=prepend_bos, padding_side=padding_side + ) else: input_ids = input # Promote 1D integer token tensors to 2D [batch=1, seq] to match @@ -2007,40 +2234,78 @@ def forward( isinstance(input_ids, torch.Tensor) and input_ids.is_floating_point() ) - # Auto-compute attention_mask + position_ids for batched list input - # when the caller didn't supply them. Matches HF generation convention. + # Left padding needs a mask and corrected positions. Right padding is + # harmless for causal real-token positions and remains unmasked to + # match HookedTransformer; bidirectional/encoder inputs still need it. if ( _is_batched_list and attention_mask is None and self.tokenizer is not None and self.tokenizer.pad_token_id is not None and not _is_inputs_embeds + and ( + _resolved_padding_side == "left" + or is_encoder_decoder + or not self.adapter.supports_causal_loss + ) ): - _prev_side = self.tokenizer.padding_side - self.tokenizer.padding_side = "left" - try: - attention_mask = utils.get_attention_mask( - self.tokenizer, - input_ids, - prepend_bos=getattr(self.cfg, "default_prepend_bos", True), - ).to(self.cfg.device) - finally: - self.tokenizer.padding_side = _prev_side - if "position_ids" not in kwargs: + attention_mask = utils.get_attention_mask( + self.tokenizer, + input_ids, + prepend_bos=getattr(self.cfg, "default_prepend_bos", True), + padding_side=_resolved_padding_side, + ).to(self.cfg.device) + # Gated on the target for the same reason the derivation below is: + # a fixed-signature forward raises TypeError on the kwarg, and a + # model that owns its own position derivation is overridden by it + # (#1626). + if "position_ids" not in kwargs and self._accepts_derived_position_ids(): position_ids = attention_mask.long().cumsum(-1) - 1 position_ids.masked_fill_(attention_mask == 0, 1) kwargs["position_ids"] = position_ids + # Any masked-out token shifts the absolute position of every real token + # after it, so positions must be derived from the mask rather than left + # to HF's default arange. This is the same derivation HookedTransformer + # applies in pos_embed; without it the bridge silently returns wrong + # logits. An all-ones mask reduces to arange, so this is a no-op there. + # + # The mask spans any cached prefix as well as the new tokens, so it is + # offset back to just the tokens actually being passed — matching how + # AbstractAttention/PosEmbed use past_kv_pos_offset. + if ( + attention_mask is not None + and "position_ids" not in kwargs + and not _is_inputs_embeds + and attention_mask.ndim == 2 + and isinstance(input_ids, torch.Tensor) + and input_ids.ndim == 2 + and attention_mask.shape[1] >= input_ids.shape[1] + and self._accepts_derived_position_ids() + ): + # .long() because callers may hand in a float 0/1 mask, and + # positions index an embedding table. + _derived = utils.get_offset_position_ids(0, attention_mask.long()) + _arange = torch.arange(attention_mask.shape[1], device=_derived.device) + # Decide per row, not per batch. A row only needs the derived + # positions when its mask actually moves one of its attended + # tokens off the default position — i.e. a masked token precedes + # a real one (left padding, or an interior gap). Rows that are + # unpadded or purely right-padded keep arange verbatim, so one + # left-padded row in a batch cannot perturb its neighbours. + _needs = ((_derived != _arange) & (attention_mask != 0)).any(dim=1, keepdim=True) + if bool(_needs.any()): + _positions = torch.where(_needs, _derived, _arange.expand_as(_derived)) + kwargs["position_ids"] = _positions[ + :, attention_mask.shape[1] - input_ids.shape[1] : + ] + if attention_mask is not None: kwargs["attention_mask"] = attention_mask if kwargs.pop("use_past_kv_cache", False) or kwargs.get("use_cache", False): kwargs["use_cache"] = True # Auto-generate decoder_input_ids for encoder-decoder models - if ( - "decoder_input_ids" not in kwargs - and hasattr(self.original_model, "config") - and getattr(self.original_model.config, "is_encoder_decoder", False) - ): + if "decoder_input_ids" not in kwargs and labels is None and is_encoder_decoder: decoder_start_token_id = getattr( self.original_model.config, "decoder_start_token_id", None ) @@ -2116,7 +2381,11 @@ def forward( if hasattr(output, "logits"): logits = output.logits elif isinstance(output, tuple) and len(output) > 0: - logits = output[0] + # With labels forwarded, HF tuple outputs are (loss, logits, ...). + if labels is not None and len(output) > 1: + logits = output[1] + else: + logits = output[0] elif hasattr(output, "last_hidden_state"): # Bare encoder models (ViTModel, DeiTModel, BertModel, etc. without # a task head) return e.g. BaseModelOutput/BaseModelOutputWithPooling, @@ -2131,6 +2400,18 @@ def forward( elif return_type == "logits_and_cache": past_key_values = getattr(output, "past_key_values", None) return (logits, past_key_values) + elif is_encoder_decoder and return_type in ("loss", "both"): + assert isinstance( + logits, torch.Tensor + ), f"Expected seq2seq logits tensor, got {type(logits)}" + assert isinstance(labels, torch.Tensor) + return self._finalize_seq2seq_return( + return_type, + logits, + labels, + output, + loss_per_token=loss_per_token, + ) elif return_type == "loss": if getattr(self.cfg, "is_audio_model", False): raise ValueError( @@ -2145,7 +2426,7 @@ def forward( "yourself from the returned logits, or use hf_generate()-style " "direct access to self.original_model for HF's own loss." ) - if _is_inputs_embeds: + if _is_inputs_embeds and labels is None: raise ValueError( "Cannot compute loss with inputs_embeds — token IDs required for labels." ) @@ -2155,21 +2436,46 @@ def forward( assert isinstance( logits, torch.Tensor ), f"Expected logits tensor, got {type(logits)}" - return self.loss_fn(logits, input_ids, per_token=loss_per_token) + if labels is not None: + return self._causal_labels_loss( + logits, + labels, + attention_mask=attention_mask, + per_token=loss_per_token, + ) + return self.loss_fn( + logits, + input_ids, + attention_mask=attention_mask, + per_token=loss_per_token, + ) elif return_type == "both": if getattr(self.cfg, "is_audio_model", False): raise ValueError( "Audio models do not support return_type='both'. " "CTC loss requires aligned frame-level labels." ) - if _is_inputs_embeds: + if _is_inputs_embeds and labels is None: raise ValueError( "Cannot compute loss with inputs_embeds — token IDs required for labels." ) assert isinstance( logits, torch.Tensor ), f"Expected logits tensor, got {type(logits)}" - loss = self.loss_fn(logits, input_ids, per_token=loss_per_token) + if labels is not None: + loss = self._causal_labels_loss( + logits, + labels, + attention_mask=attention_mask, + per_token=loss_per_token, + ) + else: + loss = self.loss_fn( + logits, + input_ids, + attention_mask=attention_mask, + per_token=loss_per_token, + ) return (logits, loss) elif return_type == "predictions": assert ( @@ -2256,8 +2562,147 @@ def loss_fn( """ if tokens.device != logits.device: tokens = tokens.to(logits.device) + if attention_mask is not None: + if attention_mask.device != logits.device: + attention_mask = attention_mask.to(logits.device) + attention_mask = self._prepare_loss_attention_mask(attention_mask, tokens) return lm_cross_entropy_loss(logits, tokens, attention_mask, per_token) + def _causal_labels_loss( + self, + logits: torch.Tensor, + labels: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + per_token: bool = False, + ) -> torch.Tensor: + """Compute shifted causal loss against explicit labels, ignoring ``-100``.""" + if labels.device != logits.device: + labels = labels.to(logits.device) + if labels.shape != logits.shape[:-1]: + raise ValueError( + "causal labels must match the logits batch and position dimensions, " + f"got labels {tuple(labels.shape)} and logits {tuple(logits.shape)}" + ) + + losses = F.cross_entropy( + logits[:, :-1].flatten(0, 1), + labels[:, 1:].flatten(), + reduction="none", + ignore_index=-100, + ).view_as(labels[:, 1:]) + valid_targets = labels[:, 1:] != -100 + if attention_mask is not None: + if attention_mask.device != logits.device: + attention_mask = attention_mask.to(logits.device) + token_mask = self._prepare_loss_attention_mask(attention_mask, labels) + valid_targets &= token_mask[:, :-1] & token_mask[:, 1:] + losses = losses.masked_fill(~valid_targets, 0.0) + return losses if per_token else losses.sum() / valid_targets.sum() + + @staticmethod + def _seq2seq_loss( + logits: torch.Tensor, + labels: torch.Tensor, + native_loss: Any, + *, + per_token: bool, + ) -> torch.Tensor: + """Return encoder-decoder loss without the causal LM token shift.""" + if labels.device != logits.device: + labels = labels.to(logits.device) + if labels.shape != logits.shape[:-1]: + raise ValueError( + "seq2seq labels must match the decoder logits batch and position " + f"dimensions, got labels {tuple(labels.shape)} and logits " + f"{tuple(logits.shape)}" + ) + if not per_token and isinstance(native_loss, torch.Tensor): + return native_loss + + losses = F.cross_entropy( + logits.flatten(0, 1), + labels.flatten(), + reduction="none" if per_token else "mean", + ignore_index=-100, + ) + return losses.view_as(labels) if per_token else losses + + def _finalize_seq2seq_return( + self, + return_type: str, + logits: torch.Tensor, + labels: torch.Tensor, + native_output: Any, + *, + loss_per_token: bool, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + loss = self._seq2seq_loss( + logits, + labels, + getattr(native_output, "loss", None), + per_token=loss_per_token, + ) + return (logits, loss) if return_type == "both" else loss + + @staticmethod + def _prepare_loss_attention_mask( + attention_mask: torch.Tensor, tokens: torch.Tensor + ) -> torch.Tensor: + """Reduce a forward attention mask to the token window scored by the loss.""" + batch, pos = tokens.shape + if attention_mask.ndim not in (2, 4): + raise ValueError( + "attention_mask must be 2D [batch, key_pos] or 4D " + f"[batch, *, query_pos, key_pos], got shape {tuple(attention_mask.shape)}" + ) + if attention_mask.shape[0] != batch: + raise ValueError( + "attention_mask batch dimension must match tokens, " + f"got {attention_mask.shape[0]} and {batch}" + ) + + if attention_mask.ndim == 2: + if attention_mask.shape[1] < pos: + raise ValueError( + "attention_mask must cover every scored token, " + f"got length {attention_mask.shape[1]} for {pos} tokens" + ) + return attention_mask[:, -pos:].bool() + + query_pos, key_pos = attention_mask.shape[-2:] + if key_pos < pos: + raise ValueError( + "attention_mask must cover every scored token, " + f"got key length {key_pos} for {pos} tokens" + ) + + blocked = attention_mask if attention_mask.dtype is torch.bool else attention_mask < -1.0 + if query_pos == 1: + # Broadcast key-only masks use one query row for the full sequence. + keep = ~blocked[..., 0, -pos:] + else: + if query_pos < pos: + raise ValueError( + "attention_mask must contain a query row for every scored token, " + f"got {query_pos} rows for {pos} tokens" + ) + # The aligned diagonal excludes causal masking while retaining padding. + diagonal = torch.diagonal( + blocked, + offset=key_pos - query_pos, + dim1=-2, + dim2=-1, + ) + if diagonal.shape[-1] < pos: + raise ValueError( + "attention_mask diagonal must cover every scored token, " + f"got length {diagonal.shape[-1]} for {pos} tokens" + ) + keep = ~diagonal[..., -pos:] + + # A token is padding only when every broadcast/head mask blocks its key. + return keep.reshape(batch, -1, pos).any(dim=1) + @overload def run_with_cache( self, @@ -2414,6 +2859,7 @@ def cache_hook(tensor: torch.Tensor, *, hook: Any) -> Optional[torch.Tensor]: effective_stop_layer = len(self.blocks) + stop_at_layer else: effective_stop_layer = stop_at_layer + gated_names_skipped: List[str] = [] for hook_name, hook in hook_dict.items(): if names_filter_fn(hook_name): if effective_stop_layer is not None: @@ -2424,7 +2870,26 @@ def cache_hook(tensor: torch.Tensor, *, hook: Any) -> Optional[torch.Tensor]: continue except (IndexError, ValueError): pass + + # Only validate gated hooks when the caller explicitly supplied + # a names_filter. The default filter matches every hook and must + # not cause gated hooks to be treated as explicitly requested. + if names_filter is not None: + try: + self.check_hooks_to_add(hook_name) + except ValueError: + gated_names_skipped.append(hook_name) + continue + hooks.append((hook, hook_name)) + + if names_filter is not None and gated_names_skipped: + warnings.warn( + f"run_with_cache: skipped {len(gated_names_skipped)} gated-off hook name(s) " + f"that will never be cached: {gated_names_skipped}. Call the relevant " + "set_use_*(True) setter first to enable them.", + stacklevel=2, + ) self.context_level += 1 context_level = self.context_level try: @@ -2440,13 +2905,15 @@ def cache_hook(tensor: torch.Tensor, *, hook: Any) -> Optional[torch.Tensor]: processed_args = [input] if processed_args and isinstance(processed_args[0], str): assert self.tokenizer is not None, "Tokenizer must be set to pass string input." - input_ids = self.to_tokens(processed_args[0]) + prepend_bos = kwargs.pop("prepend_bos", None) + input_ids = self.to_tokens(processed_args[0], prepend_bos=prepend_bos) input_ids = input_ids.to(next(self.original_model.parameters()).device) kwargs["input_ids"] = input_ids processed_args = processed_args[1:] elif "input" in kwargs and isinstance(kwargs["input"], str): assert self.tokenizer is not None, "Tokenizer must be set to pass string input." - input_ids = self.to_tokens(kwargs["input"]) + prepend_bos = kwargs.pop("prepend_bos", None) + input_ids = self.to_tokens(kwargs["input"], prepend_bos=prepend_bos) input_ids = input_ids.to(next(self.original_model.parameters()).device) kwargs["input_ids"] = input_ids del kwargs["input"] @@ -2618,7 +3085,12 @@ def run_with_hooks( effective_stop_layer = stop_at_layer def add_hook_to_point( - hook_point: HookPoint, hook_fn: Callable, name: str, dir: Literal["fwd", "bwd"] = "fwd" + hook_point: HookPoint, + hook_fn: Callable, + name: str, + dir: Literal["fwd", "bwd"] = "fwd", + *, + is_explicit: bool = True, ): if effective_stop_layer is not None and name.startswith("blocks."): try: @@ -2627,6 +3099,15 @@ def add_hook_to_point( return except (IndexError, ValueError): pass + if is_explicit: + self.check_hooks_to_add(name) + elif self._gated_hook_reason(name) is not None: + warnings.warn( + f"run_with_hooks(): filter matched gated-off hook name '{name}', skipped. " + "Call the relevant set_use_*(True) setter first to enable it.", + stacklevel=2, + ) + return if self.compatibility_mode and name != hook_point.name: alias_names_list: list[str] = [] if hook_point.name is not None: @@ -2665,7 +3146,11 @@ def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): actual_hook_name = aliases[hook_name_or_filter] if actual_hook_name in hook_dict: add_hook_to_point( - hook_dict[actual_hook_name], hook_fn, actual_hook_name, direction + hook_dict[actual_hook_name], + hook_fn, + actual_hook_name, + direction, + is_explicit=True, ) else: hook_dict = self.hook_dict @@ -2677,7 +3162,13 @@ def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): continue seen_hooks.add(hook_id) hook_name_to_use = hook_point.name if hook_point.name else name - add_hook_to_point(hook_point, hook_fn, hook_name_to_use, direction) + add_hook_to_point( + hook_point, + hook_fn, + hook_name_to_use, + direction, + is_explicit=False, + ) try: self.context_level = context_level @@ -2777,6 +3268,21 @@ def _resolve_stopping_criteria( return criteria if len(criteria) > 0 else None + def _encdec_ngram_processor(self) -> Optional[Any]: + """generation_config.no_repeat_ngram_size as transformers' own + processor, or None. HF applies it by default; parity for models whose + greedy decode needs it to escape token attractors.""" + size = getattr( + getattr(self.original_model, "generation_config", None), + "no_repeat_ngram_size", + None, + ) + if not size: + return None + from transformers.generation.logits_process import NoRepeatNGramLogitsProcessor + + return NoRepeatNGramLogitsProcessor(size) + def _generate_tokens( self, current_tokens: torch.Tensor, @@ -2808,6 +3314,10 @@ def _generate_tokens( multimodal_kwargs: Dict[str, Any], verbose: bool, stopping_criteria_list: Optional[Any] = None, + initial_attention_mask: Optional[torch.Tensor] = None, + min_decoder_length: Optional[int] = None, + ngram_processor: Optional[Any] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, ) -> Generator[Tuple[torch.Tensor, torch.Tensor, bool], None, None]: """Core generation loop. Yields (sampled_tokens, final_logits, all_finished) per step. @@ -2823,6 +3333,23 @@ def _generate_tokens( # A row may finish via EOS and/or any of the configured stopping criteria. any_stop_active = stop_at_eos or stopping_criteria_list is not None + # Models that own their position derivation (the gate refuses them) cache + # mRoPE deltas on the module between calls; a text-only prefill never + # refreshes them, so a stale delta from an earlier multimodal forward gets + # added to every cached-step position. HF's generate recomputes them at + # prefill via prepare_inputs_for_generation, which this loop bypasses — + # so match it by clearing before the prompt pass. A multimodal prefill + # recomputes its own fresh deltas regardless. + if not self._accepts_derived_position_ids(): + underlying = getattr(self, "original_model", None) + for module in ( + underlying, + getattr(underlying, "model", None), + getattr(underlying, "language_model", None), + ): + if module is not None and hasattr(module, "rope_deltas"): + module.rope_deltas = None + # Pure-SSM models (Mamba-1/2) take the stateful cache as `cache_params`; # modern hybrids (Bamba, NemotronH, FalconH1) take `past_key_values` and # would receive a duplicate cache_params via **kwargs cascade otherwise. @@ -2835,17 +3362,44 @@ def _generate_tokens( for gen_step_idx in tqdm.tqdm(range(max_new_tokens), disable=not verbose): with torch.no_grad(): if is_encoder_decoder: + assert encoder_input is not None + encdec_kwargs: Dict[str, Any] = {} + if encoder_attention_mask is not None: + encdec_kwargs["attention_mask"] = encoder_attention_mask.to( + encoder_input.device + ) logits = self( encoder_input, return_type="logits", decoder_input=decoder_tokens, + **encdec_kwargs, ) else: forward_kwargs: Dict[str, Any] = {} + # A prompt mask covers only the prompt, so extend it by one + # attended column per token generated so far. position_ids are + # left to forward(), which derives them from the mask for the + # models that can take them. + running_attention_mask: Optional[torch.Tensor] = None + if initial_attention_mask is not None: + n_generated = current_tokens.shape[1] - initial_attention_mask.shape[1] + running_attention_mask = torch.cat( + [ + initial_attention_mask.to(current_tokens.device), + torch.ones( + (current_tokens.shape[0], n_generated), + dtype=initial_attention_mask.dtype, + device=current_tokens.device, + ), + ], + dim=1, + ) + forward_kwargs["attention_mask"] = running_attention_mask # Compute attention mask and position_ids for batched # inputs with padding. if ( - _is_batched_list + initial_attention_mask is None + and _is_batched_list and self.tokenizer is not None and self.tokenizer.pad_token_id is not None ): @@ -2858,9 +3412,12 @@ def _generate_tokens( ).to(self.cfg.device) self.tokenizer.padding_side = _prev_side forward_kwargs["attention_mask"] = attn_mask - position_ids = attn_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attn_mask == 0, 1) - forward_kwargs["position_ids"] = position_ids + # Same target gate as the forward() path: the mask is safe + # for every model, the derived positions are not (#1626). + if self._accepts_derived_position_ids(): + position_ids = attn_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attn_mask == 0, 1) + forward_kwargs["position_ids"] = position_ids if gen_step_idx == 0: if pixel_values is not None: forward_kwargs["pixel_values"] = pixel_values @@ -2914,17 +3471,32 @@ def _generate_tokens( dtype=torch.long, device=device, ) - if "position_ids" in forward_kwargs: - forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ - :, -1: - ] - else: - forward_kwargs["position_ids"] = torch.full( - (batch_size, 1), - total_len - 1, - dtype=torch.long, - device=device, - ) + # Gated as a whole (#1626): every branch below supplies + # position_ids, so gating only the prompt derivation + # above would divert a refused model into the + # total_len - 1 fallback, which counts pad slots and is + # wrong per row for a left-padded batch. A model that + # owns its position derivation gets the mask alone, + # matching the uncached path. + if self._accepts_derived_position_ids(): + if "position_ids" in forward_kwargs: + forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ + :, -1: + ] + elif running_attention_mask is not None: + # total_len - 1 counts pad slots, so it is wrong + # for a left-padded prompt. Derive the new token's + # position from the mask instead. + forward_kwargs["position_ids"] = utils.get_offset_position_ids( + 0, running_attention_mask.long() + )[:, -1:] + else: + forward_kwargs["position_ids"] = torch.full( + (batch_size, 1), + total_len - 1, + dtype=torch.long, + device=device, + ) logits = self( current_tokens[:, -1:], return_type="logits", @@ -2949,6 +3521,23 @@ def _generate_tokens( if _generate_from_embeds and generated_token_ids else None ) + # transformers' own NoRepeatNGramLogitsProcessor, honoring + # generation_config (bart-large-cnn pins 3; without it greedy + # decoding falls into a BOS attractor and emits nothing). + if ngram_processor is not None and decoder_tokens is not None: + final_logits = ngram_processor(decoder_tokens, final_logits) + # HF's generate() suppresses EOS below generation_config.min_length + # (bart-large-cnn pins 56); without this the loop can EOS on step + # one and emit an empty summary. + if ( + min_decoder_length is not None + and is_encoder_decoder + and decoder_tokens is not None + and decoder_tokens.shape[1] < min_decoder_length + and stop_tokens + ): + final_logits = final_logits.clone() + final_logits[:, stop_tokens] = float("-inf") if do_sample: sampled_tokens = utils.sample_logits( final_logits, @@ -3083,6 +3672,8 @@ def generate( pixel_values: Optional[torch.Tensor] = None, stop_strings: Optional[Union[str, List[str]]] = None, stopping_criteria: Optional[Any] = None, + attention_mask: Optional[torch.Tensor] = None, + forced_bos_token_id: Optional[int] = None, **multimodal_kwargs, ) -> ( str @@ -3159,6 +3750,24 @@ def generate( Stateful/SSM models raise only when run with use_past_kv_cache=False (the default keeps them on the hooked loop). Each error names the supported alternative. + attention_mask: Optional ``[batch, pos]`` 0/1 mask over the prompt, marking + which prompt tokens are real. Required to generate correctly from an + already-padded token tensor: without it the pad tokens are treated as + real context and every real token's position is shifted, so the + continuation differs from the same prompt unpadded. The mask is extended + by one attended column per generated token. Takes precedence over the + ``padding_side`` heuristic, and unlike it can express an interior gap or + a pad id that also occurs as a real token. Passing ``padding_side`` + instead reads the padding off the pad token, which is enough for the + common single-edge case, and raises if this bridge has no tokenizer + or pad id to read it from. On the encoder-decoder and inputs_embeds + paths the mask is forwarded to the model as-is rather than grown per + step, which is what processors emitting one alongside + ``pixel_values`` expect. + forced_bos_token_id: Optional token id seeded as the first decoder token + after ``decoder_start`` on encoder-decoder models. Multilingual + translators (M2M100/MBart/NLLB) select their target language this way. + Raises ValueError on decoder-only models. Returns: Generated sequence as string, list of strings, or tensor depending on input type and return_type. @@ -3189,22 +3798,44 @@ def generate( use_past_kv_cache = self._resolve_generation_caching(use_past_kv_cache, _is_batched_list) _generate_from_embeds = False + _encdec_early = hasattr(self.original_model, "config") and getattr( + self.original_model.config, "is_encoder_decoder", False + ) if isinstance(input, str): - input_tokens = self.to_tokens( - input, prepend_bos=prepend_bos, move_to_device=True, truncate=False - ) + if _encdec_early: + # Deliberate divergence: prepend_bos is IGNORED for enc-dec + # string/list input. Encoder input follows the tokenizer's own + # recipe (lang token + trailing ); to_tokens' decoder-style + # BOS policy corrupts it — m2m100 degenerates to loops. + input_tokens = self.tokenizer(input, return_tensors="pt")["input_ids"].to( + self.cfg.device + ) + else: + input_tokens = self.to_tokens( + input, prepend_bos=prepend_bos, move_to_device=True, truncate=False + ) input_type = "str" elif isinstance(input, list): - # Force left-padding for batched generation so real tokens are - # flush-right and logits[:, -1, :] is always the last real token. - if _is_batched_list: - _orig_padding_side = self.tokenizer.padding_side - self.tokenizer.padding_side = "left" - input_tokens = self.to_tokens( - input, prepend_bos=prepend_bos, move_to_device=True, truncate=False - ) - if _is_batched_list: - self.tokenizer.padding_side = _orig_padding_side + if _encdec_early: + # Same native-recipe rule as the str branch: to_tokens' BOS + # policy corrupts encoder inputs (stray , dropped ). + # Keep the tokenizer's mask too — unequal rows otherwise + # attend over pads in the encoder. + _enc_batch = self.tokenizer(input, return_tensors="pt", padding=True) + input_tokens = _enc_batch["input_ids"].to(self.cfg.device) + if attention_mask is None and "attention_mask" in _enc_batch: + attention_mask = _enc_batch["attention_mask"].to(self.cfg.device) + else: + # Force left-padding for batched generation so real tokens are + # flush-right and logits[:, -1, :] is always the last real token. + if _is_batched_list: + _orig_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = "left" + input_tokens = self.to_tokens( + input, prepend_bos=prepend_bos, move_to_device=True, truncate=False + ) + if _is_batched_list: + self.tokenizer.padding_side = _orig_padding_side input_type = "list" elif isinstance(input, torch.Tensor) and input.is_floating_point(): # inputs_embeds: pre-computed embeddings (e.g., from multimodal models) @@ -3215,6 +3846,63 @@ def generate( input_tokens = input.to(self.cfg.device) input_type = "tokens" + # Without one of these a pre-padded tensor generates as though its pads were + # real context, shifting every real token's position (#1612). An explicit + # mask wins; otherwise the padding is read off the tokens, but only when the + # caller asked for that by passing padding_side. Deriving a mask on the + # default path would silently change behaviour for every existing caller, + # and would demand a real tokenizer where today none is required. + initial_attention_mask: Optional[torch.Tensor] = attention_mask + if initial_attention_mask is not None and ( + _generate_from_embeds + or getattr(getattr(self.original_model, "config", None), "is_encoder_decoder", False) + ): + # Growing the mask per step only means something for decoder-only token + # generation. On these paths the mask used to arrive via + # **multimodal_kwargs and be forwarded to the model untouched — as + # processors emit it alongside pixel_values — so keep doing that rather + # than reject a call that worked before this parameter existed. + multimodal_kwargs = {**multimodal_kwargs, "attention_mask": initial_attention_mask} + initial_attention_mask = None + if initial_attention_mask is not None: + if initial_attention_mask.shape != input_tokens.shape: + raise ValueError( + f"attention_mask shape {tuple(initial_attention_mask.shape)} does not " + f"match the prompt shape {tuple(input_tokens.shape)}. Pass a 0/1 mask " + "covering exactly the prompt tokens; generate() extends it itself." + ) + initial_attention_mask = initial_attention_mask.to(self.cfg.device) + elif padding_side is not None and input_type == "tokens": + # Reading the padding off the tokens needs a tokenizer with a pad id. + # Without one the argument would be inert, leaving exactly the bug this + # fixes — silently, on a bridge booted without a tokenizer. Say so + # rather than generate something quietly wrong. + if not isinstance(self.tokenizer, PreTrainedTokenizerBase): + raise ValueError( + "generate(padding_side=...) reads the padding off the pad token, " + "which needs a tokenizer; this bridge has none. Pass " + "attention_mask=... to state the padding directly instead." + ) + if self.tokenizer.pad_token_id is None: + raise ValueError( + "generate(padding_side=...) reads the padding off the pad token, " + "but this tokenizer has no pad_token_id. Set one, or pass " + "attention_mask=... to state the padding directly instead." + ) + _prepend = self.cfg.default_prepend_bos if prepend_bos is None else prepend_bos + _orig_side = self.tokenizer.padding_side + self.tokenizer.padding_side = padding_side + try: + initial_attention_mask = utils.get_attention_mask( + self.tokenizer, input_tokens, _prepend + ).to(self.cfg.device) + finally: + self.tokenizer.padding_side = _orig_side + # An all-ones mask is what the model assumes anyway; skipping it keeps + # the unpadded path byte-identical to before. + if initial_attention_mask is not None and bool(initial_attention_mask.all()): + initial_attention_mask = None + # Determine return type if return_type == "input": if input_type in ["str", "list"]: @@ -3266,6 +3954,18 @@ def generate( is_encoder_decoder = hasattr(self.original_model, "config") and getattr( self.original_model.config, "is_encoder_decoder", False ) + if forced_bos_token_id is None and is_encoder_decoder: + # HF's generate() applies generation_config defaults; bart-large-cnn + # pins forced_bos_token_id=0 there and degrades without it. + forced_bos_token_id = getattr( + getattr(self.original_model, "generation_config", None), + "forced_bos_token_id", + None, + ) + if forced_bos_token_id is not None and not is_encoder_decoder: + # Raise before any state mutation (_capture_hf_cache) and before + # the stateful hf_generate early-return would drop the kwarg. + raise ValueError("forced_bos_token_id is only meaningful for encoder-decoder models") # return_cache recomputes run_with_cache on the generated output (see issue #697). # That is well-defined only for single-sequence, decoder-only text generation, so @@ -3434,6 +4134,16 @@ def generate( dtype=input_tokens.dtype, device=self.cfg.device, ) + if forced_bos_token_id is not None: + # Multilingual seq2seq (M2M100/MBart/NLLB) selects the target + # language via the first decoder token after decoder_start. + forced = torch.full( + (batch_size, 1), + forced_bos_token_id, + dtype=input_tokens.dtype, + device=self.cfg.device, + ) + decoder_tokens = torch.cat([decoder_tokens, forced], dim=1) try: for sampled_tokens, final_logits, all_finished in self._generate_tokens( @@ -3465,6 +4175,18 @@ def generate( multimodal_kwargs=multimodal_kwargs if multimodal_kwargs else {}, verbose=verbose, stopping_criteria_list=stopping_criteria_list, + initial_attention_mask=initial_attention_mask, + min_decoder_length=( + getattr( + getattr(self.original_model, "generation_config", None), + "min_length", + None, + ) + if is_encoder_decoder + else None + ), + ngram_processor=(self._encdec_ngram_processor() if is_encoder_decoder else None), + encoder_attention_mask=(attention_mask if is_encoder_decoder else None), ): sampled_tokens_list.append(sampled_tokens.unsqueeze(1)) if logits_seq_list is not None: @@ -3480,7 +4202,8 @@ def generate( sampled_tokens = torch.cat(sampled_tokens_list, dim=1) if is_encoder_decoder: # Reconstruct full decoder sequence: start token + generated tokens - output_tokens = torch.cat([decoder_tokens[:, :1], sampled_tokens], dim=1) + decoder_seed_len = 2 if forced_bos_token_id is not None else 1 + output_tokens = torch.cat([decoder_tokens[:, :decoder_seed_len], sampled_tokens], dim=1) elif _generate_from_embeds: # For inputs_embeds, we only have the generated token IDs (no input token IDs) output_tokens = sampled_tokens @@ -3677,22 +4400,38 @@ def generate_stream( _is_batched_list = isinstance(input, list) and len(input) > 1 use_past_kv_cache = self._resolve_generation_caching(use_past_kv_cache, _is_batched_list) + _encdec_early = hasattr(self.original_model, "config") and getattr( + self.original_model.config, "is_encoder_decoder", False + ) if isinstance(input, str): - input_tokens = self.to_tokens( - input, prepend_bos=prepend_bos, move_to_device=True, truncate=False - ) + if _encdec_early: + # Native recipe: to_tokens' BOS policy corrupts encoder inputs. + input_tokens = self.tokenizer(input, return_tensors="pt")["input_ids"].to( + self.cfg.device + ) + else: + input_tokens = self.to_tokens( + input, prepend_bos=prepend_bos, move_to_device=True, truncate=False + ) input_type = "str" elif isinstance(input, list): - if _is_batched_list: + if _encdec_early: + input_tokens = self.tokenizer(input, return_tensors="pt", padding=True)[ + "input_ids" + ].to(self.cfg.device) + elif _is_batched_list: _orig_ps = self.tokenizer.padding_side self.tokenizer.padding_side = "left" - try: + try: + input_tokens = self.to_tokens( + input, prepend_bos=prepend_bos, move_to_device=True, truncate=False + ) + finally: + self.tokenizer.padding_side = _orig_ps + else: input_tokens = self.to_tokens( input, prepend_bos=prepend_bos, move_to_device=True, truncate=False ) - finally: - if _is_batched_list: - self.tokenizer.padding_side = _orig_ps input_type = "list" else: input_tokens = input.to(self.cfg.device) @@ -4156,6 +4895,37 @@ def train(self, mode: bool = True) -> "TransformerBridge": original.train(mode) return self + def _gated_hook_reason(self, hook_point_name: str) -> Optional[str]: + """Return the disabled setter name if hook_point_name is gated off, else None.""" + if hook_point_name.endswith("attn.hook_result") and not self.cfg.use_attn_result: + return "use_attn_result" + if ( + hook_point_name.endswith(("hook_q_input", "hook_k_input", "hook_v_input")) + and not self.cfg.use_split_qkv_input + ): + return "use_split_qkv_input" + if hook_point_name.endswith("mlp_in") and not self.cfg.use_hook_mlp_in: + return "use_hook_mlp_in" + if hook_point_name.endswith("attn_in") and not self.cfg.use_attn_in: + return "use_attn_in" + return None + + def check_hooks_to_add(self, hook_point_name: str) -> None: + """Raise a clear error if a hook is being explicitly added to a gated-off hook point. + + Mirrors HookedTransformer.check_hooks_to_add, but raises a ValueError + naming the setter to call, instead of a bare assert. Only for explicit, + user-named hook points — a filter/callable matching a gated name uses + _gated_hook_reason directly and skips with a warning instead, since the + filter was not necessarily targeting that name on purpose. + """ + reason = self._gated_hook_reason(hook_point_name) + if reason is not None: + raise ValueError( + f"Cannot add hook {hook_point_name} because {reason} is False. " + f"Call set_{reason}(True) first." + ) + def add_hook( self, name: Union[str, Callable[[str], bool]], @@ -4177,13 +4947,24 @@ def add_hook( if callable(name) and not isinstance(name, str): hook_dict = self.hook_dict seen_hooks: set[int] = set() + gated_names_skipped: List[str] = [] for hook_name, hook_point in hook_dict.items(): if name(hook_name): hook_id = id(hook_point) if hook_id in seen_hooks: continue seen_hooks.add(hook_id) + if self._gated_hook_reason(hook_name) is not None: + gated_names_skipped.append(hook_name) + continue hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + if gated_names_skipped: + warnings.warn( + f"add_hook: filter matched {len(gated_names_skipped)} gated-off hook " + f"name(s) that were skipped: {gated_names_skipped}. Call the relevant " + "set_use_*(True) setter first to enable them.", + stacklevel=2, + ) return component = self @@ -4197,6 +4978,7 @@ def add_hook( if hasattr(component, hook_name): hook_point = getattr(component, hook_name) if isinstance(hook_point, HookPoint): + self.check_hooks_to_add(name) hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) else: raise AttributeError( @@ -4219,16 +5001,52 @@ def add_perma_hook( """ self.add_hook(name, hook_fn, dir=dir, is_permanent=True) - def reset_hooks(self, clear_contexts=True): - """Remove all hooks from the model.""" + def hook_points(self) -> Iterable[HookPoint]: + """All registered :class:`HookPoint` instances.""" + return self._hook_registry.values() - def remove_hooks_recursive(module): - if isinstance(module, GeneralizedComponent): - module.remove_hooks() - for child in module.children(): - remove_hooks_recursive(child) + def clear_contexts(self) -> None: + """Clear the stored ``ctx`` on every registered hook point.""" + for hp in self._hook_registry.values(): + hp.clear_context() - remove_hooks_recursive(self) + def remove_all_hook_fns( + self, + direction: Literal["fwd", "bwd", "both"] = "both", + including_permanent: bool = False, + level: Optional[int] = None, + ) -> None: + """Remove hook functions from every registered hook point.""" + for hp in self._hook_registry.values(): + hp.remove_hooks(dir=direction, including_permanent=including_permanent, level=level) + + def reset_hooks( + self, + clear_contexts: bool = True, + direction: Literal["fwd", "bwd", "both"] = "both", + including_permanent: bool = False, + level: Optional[int] = None, + ) -> None: + """Remove hooks from the model; mirrors ``HookedRootModule.reset_hooks``. + + Clears through the hook registry (which holds hook points the component + walk cannot reach, e.g. alias-registered points) and, on a full reset, + additionally walks the component tree — dev's registry is not asserted + canonical, so both passes run belt-and-suspenders. + """ + if clear_contexts: + self.clear_contexts() + self.remove_all_hook_fns(direction, including_permanent=including_permanent, level=level) + + if direction == "both" and level is None: + + def remove_hooks_recursive(module): + if isinstance(module, GeneralizedComponent): + module.remove_hooks() + for child in module.children(): + remove_hooks_recursive(child) + + remove_hooks_recursive(self) def hooks(self, fwd_hooks=[], bwd_hooks=[], reset_hooks_end=True, clear_contexts=False): """Context manager for temporarily adding hooks. @@ -4258,7 +5076,18 @@ def add_hook_to_point( hook_fn: Callable, name: str, dir: Literal["fwd", "bwd"] = "fwd", + *, + is_explicit: bool = True, ): + if is_explicit: + self.check_hooks_to_add(name) + elif self._gated_hook_reason(name) is not None: + warnings.warn( + f"hooks(): filter matched gated-off hook name '{name}', skipped. " + "Call the relevant set_use_*(True) setter first to enable it.", + stacklevel=2, + ) + return if self.compatibility_mode and name != hook_point.name: alias_names_list: list[str] = [] if hook_point.name is not None: @@ -4282,7 +5111,11 @@ def apply_hooks(hooks: List[Tuple[Union[str, Callable], Callable]], is_fwd: bool actual_hook_name = aliases[hook_name_or_filter] if actual_hook_name in hook_dict: add_hook_to_point( - hook_dict[actual_hook_name], hook_fn, actual_hook_name, direction + hook_dict[actual_hook_name], + hook_fn, + actual_hook_name, + direction, + is_explicit=True, ) else: hook_dict = self.hook_dict @@ -4294,7 +5127,13 @@ def apply_hooks(hooks: List[Tuple[Union[str, Callable], Callable]], is_fwd: bool continue seen_hooks.add(hook_id) hook_name_to_use = hook_point.name if hook_point.name else name - add_hook_to_point(hook_point, hook_fn, hook_name_to_use, direction) + add_hook_to_point( + hook_point, + hook_fn, + hook_name_to_use, + direction, + is_explicit=False, + ) try: self.context_level = context_level @@ -4320,7 +5159,7 @@ def set_use_attn_result(self, use_attn_result: bool): """ if use_attn_result: self._validate_attention_fork_supported("use_attn_result") - self.cfg.use_attn_result = use_attn_result + self.cfg._set_bridge_managed_hook_flag("use_attn_result", use_attn_result) self._propagate_attention_flag("use_attn_result", use_attn_result) def set_use_split_qkv_input(self, use_split_qkv_input: bool): @@ -4335,7 +5174,7 @@ def set_use_split_qkv_input(self, use_split_qkv_input: bool): "Call set_use_attn_in(False) before enabling use_split_qkv_input." ) self._validate_attention_fork_supported("use_split_qkv_input") - self.cfg.use_split_qkv_input = use_split_qkv_input + self.cfg._set_bridge_managed_hook_flag("use_split_qkv_input", use_split_qkv_input) self._propagate_attention_flag("use_split_qkv_input", use_split_qkv_input) def set_use_attn_in(self, use_attn_in: bool): @@ -4353,7 +5192,7 @@ def set_use_attn_in(self, use_attn_in: bool): "Call set_use_split_qkv_input(False) before enabling use_attn_in." ) self._validate_attention_fork_supported("use_attn_in") - self.cfg.use_attn_in = use_attn_in + self.cfg._set_bridge_managed_hook_flag("use_attn_in", use_attn_in) self._propagate_attention_flag("use_attn_in", use_attn_in) def set_use_hook_mlp_in(self, use_hook_mlp_in: bool) -> None: @@ -4362,18 +5201,26 @@ def set_use_hook_mlp_in(self, use_hook_mlp_in: bool) -> None: See :py:meth:`HookedTransformer.set_use_hook_mlp_in`. """ - self.cfg.use_hook_mlp_in = use_hook_mlp_in + self.cfg._set_bridge_managed_hook_flag("use_hook_mlp_in", use_hook_mlp_in) if not hasattr(self, "blocks"): return for block in self.blocks: block_cfg = getattr(block, "config", None) if block_cfg is not None and block_cfg is not self.cfg: try: - block_cfg.use_hook_mlp_in = use_hook_mlp_in - except Exception: + self._write_propagated_hook_flag(block_cfg, "use_hook_mlp_in", use_hook_mlp_in) + except (AttributeError, TypeError): pass block._use_hook_mlp_in = use_hook_mlp_in + @staticmethod + def _write_propagated_hook_flag(config: Any, flag_name: str, value: bool) -> None: + """Write a cloned config flag without dispatching through its live Bridge.""" + if isinstance(config, TransformerBridgeConfig): + config._set_bridge_managed_hook_flag(flag_name, value) + else: + object.__setattr__(config, flag_name, value) + def _propagate_attention_flag(self, flag_name: str, value: bool) -> None: """Mirror `bridge.cfg.` onto every block's attention config. @@ -4394,11 +5241,10 @@ def _propagate_attention_flag(self, flag_name: str, value: bool) -> None: attn_cfg = getattr(attn, "config", None) if attn_cfg is not None and attn_cfg is not self.cfg: try: - setattr(attn_cfg, flag_name, value) - except Exception: - # Some cfg objects may be frozen/immutable. Skip silently — - # the block simply won't honor the flag, which is the - # same outcome as before this fix. + self._write_propagated_hook_flag(attn_cfg, flag_name, value) + except (AttributeError, TypeError): + # Some config-like objects reject attributes even when + # bypassing their custom __setattr__ implementation. pass def _validate_attention_fork_supported(self, flag_name: str) -> None: @@ -4567,8 +5413,8 @@ def _normalize_bridge_key_to_hf(self, key: str) -> str: block_list_names = {"blocks", "L_blocks", "H_blocks", "encoder_blocks", "decoder_blocks"} for tl_name, component in component_mapping.items(): if component.name and tl_name not in block_list_names: - # Skip if TL name is already a suffix of the HF path (avoids doubling). - if tl_name != component.name and not component.name.endswith("." + tl_name): + # Skip if TL name is already a segment of its HF path (avoids doubling). + if tl_name != component.name and tl_name not in component.name.split("."): attr_to_hf[tl_name] = component.name # Map block-level components (ln1, ln2, attn, mlp) for all block lists @@ -4600,9 +5446,10 @@ def state_dict(self, destination=None, prefix="", keep_vars=False): Converts HuggingFace format keys to TransformerLens format and filters out _original_component references and nested HuggingFace components. - This returns a clean state dict with only bridge component paths converted to TL format, - excluding nested HF components (like c_fc, c_proj, c_attn) that exist inside - original_component modules. + A direct no-argument call returns a clean state dict with bridge component + paths converted to TL format. Calls that supply ``destination`` or + ``prefix`` use standard ``nn.Module`` recursive semantics so a Bridge can + compose inside a parent module. Args: destination: Optional dict to store state dict in @@ -4610,14 +5457,17 @@ def state_dict(self, destination=None, prefix="", keep_vars=False): keep_vars: Whether to keep variables as Variables instead of tensors Returns: - Dict containing the state dict with TransformerLens format keys + Direct calls return TransformerLens-format keys; recursive calls + return the supplied destination with standard module-tree keys. """ - if destination is not None: - raw_state_dict = self.original_model.state_dict( - destination=destination, prefix=prefix, keep_vars=keep_vars + if destination is not None or prefix: + return super().state_dict( + destination=destination, + prefix=prefix, + keep_vars=keep_vars, ) - else: - raw_state_dict = self.original_model.state_dict(prefix=prefix, keep_vars=keep_vars) + + raw_state_dict = self.original_model.state_dict(keep_vars=keep_vars) # Clean _original_component references and convert to TL format # Also filter out nested HuggingFace components that are wrapped by bridge components @@ -4648,9 +5498,39 @@ def state_dict(self, destination=None, prefix="", keep_vars=False): return tl_state_dict + def _tl_key_to_actual_keys(self) -> dict[str, list[str]]: + """Inverse of the renaming state_dict() applies: map each TL-format key + back to every raw parameter/buffer path that represents it. + + Mirrors the filtering and key-conversion in state_dict() exactly, except + it keeps every raw key for a given TL key instead of only the first-seen + one. Bridge components frequently expose the same underlying parameter + through more than one attribute path (e.g. GPT-2's split q/k/v weights + are views into the wrapped module's combined c_attn weight, reachable + both via a block-level shortcut and via the nested _original_component + chain) - all of those aliases must be written for the round trip to + actually change what forward() reads, not just what state_dict() shows. + """ + mapping: dict[str, list[str]] = {} + for actual_key in self.original_model.state_dict(): + if actual_key == "_original_component" or actual_key.startswith("_original_component."): + continue + clean_key = actual_key.replace("._original_component", "") + if not self._is_valid_bridge_path(clean_key): + continue + hf_key = self._normalize_bridge_key_to_hf(clean_key) + tl_key = self.adapter.convert_hf_key_to_tl_key(hf_key) + mapping.setdefault(tl_key, []).append(actual_key) + return mapping + def load_state_dict(self, state_dict, strict=True, assign=False): """Load state dict into the model, handling both clean keys and original keys with _original_component references. + Accepts three key formats: TL-format keys as emitted by state_dict() + (e.g. "blocks.0.attn.q.weight"), raw native parameter paths (e.g. for + ``boot_native`` / tracr-style loading), and raw paths with + "_original_component" segments stripped. + Args: state_dict: Dictionary containing a whole state of the module strict: Whether to strictly enforce that the keys in state_dict match the keys returned by this module's state_dict() function @@ -4661,27 +5541,60 @@ def load_state_dict(self, state_dict, strict=True, assign=False): """ current_state_dict = self.original_model.state_dict() clean_to_actual = {} - actual_to_clean = {} for actual_key in current_state_dict.keys(): if actual_key != "_original_component": - clean_key = actual_key.replace("._original_component", "") - clean_to_actual[clean_key] = actual_key - actual_to_clean[actual_key] = clean_key + clean_to_actual[actual_key.replace("._original_component", "")] = actual_key + + tl_to_actual = self._tl_key_to_actual_keys() + mapped_state_dict = {} + unexpected_keys = [] for input_key, value in state_dict.items(): if input_key in current_state_dict: mapped_state_dict[input_key] = value - else: - if input_key in clean_to_actual: - actual_key = clean_to_actual[input_key] + elif input_key in clean_to_actual: + mapped_state_dict[clean_to_actual[input_key]] = value + elif input_key in tl_to_actual: + for actual_key in tl_to_actual[input_key]: mapped_state_dict[actual_key] = value - else: - mapped_state_dict[input_key] = value - effective_strict = strict and len(mapped_state_dict) == len(current_state_dict) - return self.original_model.load_state_dict( - mapped_state_dict, strict=effective_strict, assign=assign + else: + unexpected_keys.append(input_key) + + # A TL key's actual-key aliases share the same underlying storage (see + # _tl_key_to_actual_keys), so writing any one of them already updates + # what forward() reads for all of them. Treat the group as satisfied + # if any alias was written -- e.g. a caller supplying clean/raw keys + # (the branch above maps each clean key to exactly one actual key) + # shouldn't have the *other*, unwritten aliases reported as missing. + missing_keys = sorted( + actual_key + for actual_keys in tl_to_actual.values() + if not any(k in mapped_state_dict for k in actual_keys) + for actual_key in actual_keys ) + if strict and (missing_keys or unexpected_keys): + error_msgs = [] + if unexpected_keys: + error_msgs.append( + "Unexpected key(s) in state_dict: " + + ", ".join(f'"{k}"' for k in sorted(unexpected_keys)) + ) + if missing_keys: + error_msgs.append( + "Missing key(s) in state_dict: " + ", ".join(f'"{k}"' for k in missing_keys) + ) + raise RuntimeError( + "Error(s) in loading state_dict for {}:\n\t{}".format( + type(self.original_model).__name__, "\n\t".join(error_msgs) + ) + ) + + result = self.original_model.load_state_dict(mapped_state_dict, strict=False, assign=assign) + if assign: + refresh_container_state_owners(self) + return type(result)(missing_keys=missing_keys, unexpected_keys=unexpected_keys) + def get_params(self): """Access to model parameters in the format expected by SVDInterpreter. diff --git a/transformer_lens/model_bridge/component_setup.py b/transformer_lens/model_bridge/component_setup.py index c628c643d5..b5ac1017cc 100644 --- a/transformer_lens/model_bridge/component_setup.py +++ b/transformer_lens/model_bridge/component_setup.py @@ -20,6 +20,63 @@ pass +class _ContainerStateOwner(nn.Module): + """Registered view of state owned directly by an unwrapped container.""" + + def __init__(self, original_container: nn.Module) -> None: + super().__init__() + self.__dict__["_original_container"] = original_container + + def _sync_original_container(self) -> None: + original_container = self.__dict__["_original_container"] + original_container._parameters.update(self._parameters) + original_container._buffers.update(self._buffers) + + def _refresh_from_original_container(self) -> None: + original_container = self.__dict__["_original_container"] + for name in self._parameters: + self._parameters[name] = original_container._parameters[name] + for name in self._buffers: + self._buffers[name] = original_container._buffers[name] + + def _apply(self, fn: Any, recurse: bool = True) -> "_ContainerStateOwner": + self._refresh_from_original_container() + super()._apply(fn, recurse=recurse) + self._sync_original_container() + return self + + def _load_from_state_dict( + self, + state_dict: dict[str, Any], + prefix: str, + local_metadata: dict[str, Any], + strict: bool, + missing_keys: list[str], + unexpected_keys: list[str], + error_msgs: list[str], + ) -> None: + super()._load_from_state_dict( + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, + ) + self._sync_original_container() + + +def refresh_container_state_owners(bridge_module: nn.Module) -> None: + """Refresh registered container state from the original model tree.""" + root_owner = bridge_module._modules.get("_container_state_owners") + if not isinstance(root_owner, _ContainerStateOwner): + return + for owner in root_owner.modules(): + if isinstance(owner, _ContainerStateOwner): + owner._refresh_from_original_container() + + def replace_remote_component( replacement_component: nn.Module, remote_path: str, remote_model: RemoteModel ) -> None: @@ -56,6 +113,69 @@ def set_original_components( """ component_mapping = architecture_adapter.get_component_mapping() setup_components(component_mapping, bridge_module, architecture_adapter, original_model) + if isinstance(original_model, nn.Module): + _register_unowned_container_state(bridge_module, original_model) + + +def _register_unowned_container_state(bridge_module: nn.Module, original_model: nn.Module) -> None: + """Make direct container parameters and buffers reachable through the Bridge tree.""" + registered_parameter_ids = { + id(parameter) + for module in bridge_module.modules() + for parameter in module._parameters.values() + if parameter is not None + } + registered_buffer_ids = { + id(buffer) + for module in bridge_module.modules() + for buffer in module._buffers.values() + if buffer is not None + } + missing_parameters: list[tuple[str, nn.Module, str, nn.Parameter]] = [] + missing_buffers: list[tuple[str, nn.Module, str, Any]] = [] + + for module_path, module in original_model.named_modules(): + if any(child is not None for child in module._modules.values()): + for parameter_name, parameter in module._parameters.items(): + if parameter is not None and id(parameter) not in registered_parameter_ids: + missing_parameters.append((module_path, module, parameter_name, parameter)) + registered_parameter_ids.add(id(parameter)) + for buffer_name, buffer in module._buffers.items(): + if buffer is not None and id(buffer) not in registered_buffer_ids: + missing_buffers.append((module_path, module, buffer_name, buffer)) + registered_buffer_ids.add(id(buffer)) + + if not missing_parameters and not missing_buffers: + return + + owner_by_path: dict[str, _ContainerStateOwner] = {"": _ContainerStateOwner(original_model)} + root_owner = owner_by_path[""] + original_modules = dict(original_model.named_modules()) + + def get_owner(module_path: str) -> _ContainerStateOwner: + current_path = "" + current_owner = root_owner + for path_part in module_path.split(".") if module_path else (): + child_path = f"{current_path}.{path_part}" if current_path else path_part + if child_path not in owner_by_path: + child_owner = _ContainerStateOwner(original_modules[child_path]) + current_owner.add_module(path_part, child_owner) + owner_by_path[child_path] = child_owner + current_owner = owner_by_path[child_path] + current_path = child_path + return current_owner + + for module_path, _, parameter_name, parameter in missing_parameters: + get_owner(module_path).register_parameter(parameter_name, parameter) + + for module_path, module, buffer_name, buffer in missing_buffers: + get_owner(module_path).register_buffer( + buffer_name, + buffer, + persistent=buffer_name not in module._non_persistent_buffers_set, + ) + + bridge_module.add_module("_container_state_owners", root_owner) def setup_submodules( @@ -286,6 +406,8 @@ def setup_blocks_bridge( block_bridge.name = f"{blocks_template.name}.{i}" block_bridge.set_original_component(original_block) setup_submodules(block_bridge, architecture_adapter, original_block) + if hasattr(block_bridge, "_wire_ln1_module"): + block_bridge._wire_ln1_module() bridged_blocks.append(block_bridge) replace_remote_component(bridged_blocks, blocks_template.name, original_model) return bridged_blocks diff --git a/transformer_lens/model_bridge/generalized_components/alibi_joint_qkv_attention.py b/transformer_lens/model_bridge/generalized_components/alibi_joint_qkv_attention.py index acb1c6a537..971d5fb0a1 100644 --- a/transformer_lens/model_bridge/generalized_components/alibi_joint_qkv_attention.py +++ b/transformer_lens/model_bridge/generalized_components/alibi_joint_qkv_attention.py @@ -118,6 +118,7 @@ def _reconstruct_attention( # Add attention mask attention_mask = kwargs.get("attention_mask", None) if attention_mask is not None: + attention_mask = self._normalize_compatibility_mask_sentinel(attention_mask) attn_scores = attn_scores + attention_mask[:, :, :, : attn_scores.shape[-1]] attn_scores = self.hook_attn_scores(attn_scores) diff --git a/transformer_lens/model_bridge/generalized_components/attention.py b/transformer_lens/model_bridge/generalized_components/attention.py index 3fc9af1b69..2f072da52c 100644 --- a/transformer_lens/model_bridge/generalized_components/attention.py +++ b/transformer_lens/model_bridge/generalized_components/attention.py @@ -429,14 +429,19 @@ def _update_kv_cache( present in kwargs, K and V are returned unchanged. """ past_key_values = kwargs.get("past_key_values", None) + if past_key_values is None: + # GPT-NeoX/GPT-J/Bloom/Falcon/MPT/CodeGen/GPTBigCode still name the + # cache `layer_past`; missing it leaves the cache empty, so every + # decode step attends to itself alone and generation ignores the prompt. + past_key_values = kwargs.get("layer_past", None) if past_key_values is None: return k, v layer_idx = getattr(self, "_layer_idx", None) if layer_idx is None: logger.warning( "%s: past_key_values provided but _layer_idx is None " - "(HF component missing layer_idx attribute). " - "KV cache update skipped — generation will be slow.", + "(HF component missing layer_idx attribute). KV cache update " + "skipped — cached generation will ignore earlier tokens.", self.name, ) return k, v @@ -518,10 +523,25 @@ def _softmax_dropout_pattern( attn_weights = torch.nn.functional.softmax(attn_scores, dim=-1) if target_dtype is not None: attn_weights = attn_weights.to(target_dtype) + attn_weights = self._scrub_compatibility_pattern_nans(attn_weights) attn_weights = self._apply_attn_dropout(attn_weights) attn_weights = self.hook_pattern(attn_weights) return attn_weights + def _scrub_compatibility_pattern_nans(self, pattern: torch.Tensor) -> torch.Tensor: + """Match HookedTransformer for fully masked attention rows.""" + if self.compatibility_mode: + pattern = torch.where(torch.isnan(pattern), torch.zeros_like(pattern), pattern) + return pattern + + def _normalize_compatibility_mask_sentinel(self, attention_mask: torch.Tensor) -> torch.Tensor: + """Normalize additive mask sentinels before dtype conversion or addition.""" + if self.compatibility_mode and attention_mask.is_floating_point(): + attention_mask = attention_mask.masked_fill( + attention_mask <= torch.finfo(attention_mask.dtype).min, -torch.inf + ) + return attention_mask + def _reshape_attn_output( self, attn_output: torch.Tensor, @@ -558,6 +578,7 @@ def _apply_reconstruct_attention_mask( if q_seq_len is None: q_seq_len = seq_len min_dtype = torch.finfo(attn_scores.dtype).min + mask_value = -torch.inf if self.compatibility_mode else min_dtype use_direct_hf_mask = attention_mask is not None and attention_mask.ndim >= 4 # Bidirectional attention (encoders) and cross-attention have no causal # structure, so only synthesize the triangular mask for causal self-attention. @@ -569,29 +590,29 @@ def _apply_reconstruct_attention_mask( q_seq_len, seq_len, device=attn_scores.device, dtype=torch.bool ) causal_mask = torch.tril(causal_mask, diagonal=seq_len - q_seq_len) - attn_scores = attn_scores.masked_fill(~causal_mask, min_dtype) - - if attention_mask is None: - return attn_scores - - if attention_mask.shape[-1] != seq_len: - attention_mask = attention_mask[..., :seq_len] - if attention_mask.ndim >= 3 and attention_mask.shape[-2] != q_seq_len: - # Extra query rows mean a full-sequence mask on a cached decode step, - # where the live queries are the LAST rows; taking the first hands - # every step position 0 (Baichuan-13B fuses ALiBi slopes in here). - attention_mask = attention_mask[..., -q_seq_len:, :] - - if attention_mask.dtype == torch.bool: - attention_mask = torch.where( - attention_mask, - torch.zeros((), dtype=attn_scores.dtype, device=attn_scores.device), - torch.full((), min_dtype, dtype=attn_scores.dtype, device=attn_scores.device), - ) - else: - attention_mask = attention_mask.to(dtype=attn_scores.dtype) + attn_scores = attn_scores.masked_fill(~causal_mask, mask_value) + + if attention_mask is not None: + if attention_mask.shape[-1] != seq_len: + attention_mask = attention_mask[..., :seq_len] + if attention_mask.ndim >= 3 and attention_mask.shape[-2] != q_seq_len: + # Extra query rows mean a full-sequence mask on a cached decode step, + # where the live queries are the LAST rows; taking the first hands + # every step position 0 (Baichuan-13B fuses ALiBi slopes in here). + attention_mask = attention_mask[..., -q_seq_len:, :] + + if attention_mask.dtype == torch.bool: + attention_mask = torch.where( + attention_mask, + torch.zeros((), dtype=attn_scores.dtype, device=attn_scores.device), + torch.full((), mask_value, dtype=attn_scores.dtype, device=attn_scores.device), + ) + else: + attention_mask = self._normalize_compatibility_mask_sentinel(attention_mask) + attention_mask = attention_mask.to(dtype=attn_scores.dtype) + attn_scores = attn_scores + attention_mask - return attn_scores + attention_mask + return attn_scores def _get_n_heads(self, use_kv: bool = False) -> int: """Resolve the number of attention heads from config. @@ -792,42 +813,14 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: raise RuntimeError( f"Original component not set for {self.name}. Call set_original_component() first." ) - # Skip non-fp params: quantized weights (bnb uint8/int8, GPTQ/AWQ int32, - # HQQ, torchao) are stored in integer dtypes and dequantized internally - # during matmul. The compute dtype must come from a fp parameter; casting - # fp inputs to an integer storage dtype destroys precision. - target_dtype = None - for p in self.original_component.parameters(): - if not p.dtype.is_floating_point: - continue - target_dtype = p.dtype - break if "query_input" in kwargs: hooked = self.hook_in(kwargs["query_input"]) - if ( - target_dtype is not None - and isinstance(hooked, torch.Tensor) - and hooked.is_floating_point() - ): - hooked = hooked.to(dtype=target_dtype) kwargs["query_input"] = hooked elif "hidden_states" in kwargs: hooked = self.hook_in(kwargs["hidden_states"]) - if ( - target_dtype is not None - and isinstance(hooked, torch.Tensor) - and hooked.is_floating_point() - ): - hooked = hooked.to(dtype=target_dtype) kwargs["hidden_states"] = hooked elif len(args) > 0 and isinstance(args[0], torch.Tensor): hooked = self.hook_in(args[0]) - if ( - target_dtype is not None - and isinstance(hooked, torch.Tensor) - and hooked.is_floating_point() - ): - hooked = hooked.to(dtype=target_dtype) args = (hooked,) + args[1:] # try/finally so the captured tensor (and its autograd graph) is # released even if original_component raises. diff --git a/transformer_lens/model_bridge/generalized_components/base.py b/transformer_lens/model_bridge/generalized_components/base.py index af8087282a..3d092489a6 100644 --- a/transformer_lens/model_bridge/generalized_components/base.py +++ b/transformer_lens/model_bridge/generalized_components/base.py @@ -320,16 +320,6 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: raise RuntimeError( f"Original component not set for {self.name}. Call set_original_component() first." ) - # Skip non-fp params: quantized weights (bnb uint8/int8, GPTQ/AWQ int32, - # HQQ, torchao) are stored in integer dtypes and dequantized internally - # during matmul. The compute dtype must come from a fp parameter; casting - # fp inputs to an integer storage dtype destroys precision. - target_dtype = None - for p in original_component.parameters(): - if not p.dtype.is_floating_point: - continue - target_dtype = p.dtype - break input_arg_names = [ "input", "hidden_states", @@ -342,19 +332,11 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: for name in input_arg_names: if name in kwargs: hooked = self.hook_in(kwargs[name]) - if ( - target_dtype is not None - and isinstance(hooked, torch.Tensor) - and hooked.is_floating_point() - ): - hooked = hooked.to(dtype=target_dtype) kwargs[name] = hooked input_found = True break if not input_found and len(args) > 0 and isinstance(args[0], torch.Tensor): hooked_input = self.hook_in(args[0]) - if target_dtype is not None and hooked_input.is_floating_point(): - hooked_input = hooked_input.to(dtype=target_dtype) args = (hooked_input,) + args[1:] input_found = True output = original_component(*args, **kwargs) diff --git a/transformer_lens/model_bridge/generalized_components/block.py b/transformer_lens/model_bridge/generalized_components/block.py index 5806a146fa..fdb3970af6 100644 --- a/transformer_lens/model_bridge/generalized_components/block.py +++ b/transformer_lens/model_bridge/generalized_components/block.py @@ -119,12 +119,35 @@ def __init__( # blocks) when use_hook_mlp_in is set. See #1317. self.hook_mlp_in = HookPoint() + def _wire_ln1_module(self) -> None: + """Keep the raw ln1 execution reference outside the ownership tree.""" + from transformer_lens.model_bridge.generalized_components.attention import ( + AttentionBridge, + ) + + ln1 = self.submodules.get("ln1") if self.submodules else None + attn = self.submodules.get("attn") if self.submodules else None + if not isinstance(attn, AttentionBridge): + return + + ln1_module = None + if ( + ln1 is not None + and getattr(attn, "supports_split_qkv_fork", False) + and getattr(ln1, "original_component", None) is not None + ): + ln1_module = ln1.original_component + + attn._modules.pop("_ln1_module", None) + object.__setattr__(attn, "_ln1_module", ln1_module) + def _maybe_wire_capture_hooks(self) -> None: """Install the block's capture hooks (split-qkv fork, hook_mlp_in). Registered on the bridge submodule, not ``original_component`` — the manual bridge forward never calls the raw module. Idempotent. """ + self._wire_ln1_module() if self._capture_hooks_wired: return from transformer_lens.model_bridge.generalized_components.attention import ( @@ -147,7 +170,6 @@ def _capture_pre_ln1(_module: torch.nn.Module, args: tuple) -> None: handle = ln1.register_forward_pre_hook(_capture_pre_ln1) self._capture_hook_handles.append(handle) - attn._ln1_module = ln1.original_component # hook_mlp_in must capture the MLP-branch entry point: ln2's input on # pre-norm blocks, the MLP's own input on post-norm blocks (where ln2 @@ -190,6 +212,16 @@ def _read_use_hook_mlp_in(self) -> bool: return bool(cfg.use_hook_mlp_in) return self._use_hook_mlp_in + def _clear_attention_capture(self) -> None: + """Release the transient residual captured for attention input forks.""" + from transformer_lens.model_bridge.generalized_components.attention import ( + AttentionBridge, + ) + + attn = self.submodules.get("attn") if self.submodules else None + if isinstance(attn, AttentionBridge): + attn._captured_pre_ln_residual = None + def forward(self, *args: Any, **kwargs: Any) -> Any: """Forward pass through the block bridge. @@ -209,6 +241,7 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: ) self._maybe_wire_capture_hooks() + self._clear_attention_capture() self._check_stop_at_layer(*args, **kwargs) args, kwargs = self._hook_input_hidden_states(args, kwargs) @@ -216,7 +249,10 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: # This prevents errors when passing encoder-specific params to decoder-only models filtered_kwargs = self._filter_kwargs_for_forward(kwargs, len(args)) - output = self.original_component(*args, **filtered_kwargs) + try: + output = self.original_component(*args, **filtered_kwargs) + finally: + self._clear_attention_capture() force_tuple_for_bare_tensor = self._is_standalone_hidden_state_call(args, filtered_kwargs) return self._apply_output_hook( output, force_tuple_for_bare_tensor=force_tuple_for_bare_tensor @@ -272,6 +308,13 @@ def _is_standalone_hidden_state_call(args: tuple, kwargs: dict) -> bool: and isinstance(kwargs["hidden_states"], torch.Tensor) ) + def _extract_layer_idx(self) -> Optional[int]: + """Parse this block's layer index from its name (TL/GPT-2/LLaMA patterns).""" + if self.name is None: + return None + match = re.search(r"(?:^|\.)(?:blocks|h|layers)\.(\d+)", self.name) + return int(match.group(1)) if match else None + def _check_stop_at_layer(self, *args: Any, **kwargs: Any) -> None: """Check if execution should stop before this block. Raises StopAtLayerException. @@ -281,11 +324,9 @@ def _check_stop_at_layer(self, *args: Any, **kwargs: Any) -> None: if not (hasattr(self, "_stop_at_layer_idx") and self._stop_at_layer_idx is not None): return if self.name is not None: - match = ( - re.search(r"blocks\.(\d+)", self.name) - or re.search(r"\.h\.(\d+)", self.name) - or re.search(r"\.layers\.(\d+)", self.name) - ) + # Anchored alternation: native models name blocks top-level + # ("layers.0"), which the old leading-dot pattern missed entirely. + match = re.search(r"(?:^|\.)(?:blocks|h|layers)\.(\d+)", self.name) else: match = None if match: diff --git a/transformer_lens/model_bridge/generalized_components/bloom_attention.py b/transformer_lens/model_bridge/generalized_components/bloom_attention.py index cbbd290990..c8a148aed1 100644 --- a/transformer_lens/model_bridge/generalized_components/bloom_attention.py +++ b/transformer_lens/model_bridge/generalized_components/bloom_attention.py @@ -101,10 +101,15 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: # Apply input hook hooked_input = self.hook_in(hidden_states) - # Run through split Q/K/V projections (these fire hook_q, hook_k, hook_v) - q_output = self.q(hooked_input) - k_output = self.k(hooked_input) - v_output = self.v(hooked_input) + # Run through split Q/K/V projections (these fire hook_q, hook_k, hook_v), + # via the per-head fork when use_split_qkv_input / use_attn_in is set so + # those gated hooks fire here as they do on every other joint-QKV bridge. + if self._is_split_qkv_fork_active(): + q_output, k_output, v_output = self._split_forward_qkv(hooked_input) + else: + q_output = self.q(hooked_input) + k_output = self.k(hooked_input) + v_output = self.v(hooked_input) # Reconstruct attention with ALiBi (fires hook_attn_scores, hook_pattern) attn_output, attn_weights = self._reconstruct_attention( @@ -199,7 +204,18 @@ def _reconstruct_attention( attn_output = self._reshape_attn_output( attn_output, batch_size, seq_len, num_heads, head_dim ) - attn_output = self._apply_output_projection(attn_output) + if ( + bool(getattr(self.config, "use_attn_result", False)) + and hasattr(self, "o") + and self.o.original_component is not None + ): + # Fire hook_z on the flat pre-projection tensor first, so patches at + # hook_z reach the per-head computation (same order as the parent). + attn_output = self.o.hook_in(attn_output) + z_4d = attn_output.view(batch_size, seq_len, num_heads, head_dim) + attn_output = self._compute_per_head_result(z_4d, num_heads, head_dim) + else: + attn_output = self._apply_output_projection(attn_output) return (attn_output, attn_weights) diff --git a/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py b/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py index 15539b38b4..7d40d57afd 100644 --- a/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py +++ b/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py @@ -65,6 +65,9 @@ def __init__( self._activation_fn: Any = None self._register_state_dict_hook(JointGateUpMLPBridge._filter_gate_up_state_dict) + self.register_load_state_dict_pre_hook( + JointGateUpMLPBridge._restore_filtered_gate_up_state_dict + ) @staticmethod def _filter_gate_up_state_dict( @@ -79,6 +82,29 @@ def _filter_gate_up_state_dict( for k in keys_to_remove: del state_dict[k] + @staticmethod + def _restore_filtered_gate_up_state_dict( + module: torch.nn.Module, + state_dict: Dict[str, Any], + prefix: str, + local_metadata: Dict[str, Any], + strict: bool, + missing_keys: list[str], + unexpected_keys: list[str], + error_msgs: list[str], + ) -> None: + """Insert current combined weights only to satisfy strict key matching. + + Production checkpoints restore authoritative values through the unfiltered + Hugging Face ``_original_component`` path. + """ + del local_metadata, strict, missing_keys, unexpected_keys, error_msgs + gate_up = module._modules.get("gate_up") + if gate_up is None: + return + for key, value in gate_up.state_dict(prefix=f"{prefix}gate_up.").items(): + state_dict.setdefault(key, value) + def _default_split_gate_up( self, original_mlp_component: Any, diff --git a/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py b/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py index 438437474a..9e8b80d324 100644 --- a/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py +++ b/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py @@ -100,6 +100,9 @@ def __init__( # Exclude stale qkv combined weights from state_dict after splitting. self._register_state_dict_hook(JointQKVAttentionBridge._filter_qkv_state_dict) + self.register_load_state_dict_pre_hook( + JointQKVAttentionBridge._restore_filtered_qkv_state_dict + ) def __deepcopy__(self, memo): """Share split_qkv_matrix and config across clones instead of copying. @@ -143,6 +146,29 @@ def _filter_qkv_state_dict( for k in keys_to_remove: del state_dict[k] + @staticmethod + def _restore_filtered_qkv_state_dict( + module: torch.nn.Module, + state_dict: Dict[str, Any], + prefix: str, + local_metadata: Dict[str, Any], + strict: bool, + missing_keys: list[str], + unexpected_keys: list[str], + error_msgs: list[str], + ) -> None: + """Insert current combined weights only to satisfy strict key matching. + + Production checkpoints restore authoritative values through the unfiltered + Hugging Face ``_original_component`` path. + """ + del local_metadata, strict, missing_keys, unexpected_keys, error_msgs + qkv = module._modules.get("qkv") + if qkv is None: + return + for key, value in qkv.state_dict(prefix=f"{prefix}qkv.").items(): + state_dict.setdefault(key, value) + def _create_qkv_conversion_rule(self) -> BaseTensorConversion: """Create the appropriate conversion rule for the individual q, k, and v matrices. diff --git a/transformer_lens/model_bridge/generalized_components/mla_attention.py b/transformer_lens/model_bridge/generalized_components/mla_attention.py index 688312321d..c733b7731e 100644 --- a/transformer_lens/model_bridge/generalized_components/mla_attention.py +++ b/transformer_lens/model_bridge/generalized_components/mla_attention.py @@ -296,6 +296,7 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: attn_scores = torch.matmul(query_states, key_states.transpose(-2, -1)) * scaling if attention_mask is not None: + attention_mask = self._normalize_compatibility_mask_sentinel(attention_mask) attn_scores = attn_scores + attention_mask attn_scores = self.hook_attn_scores(attn_scores) diff --git a/transformer_lens/model_bridge/generalized_components/moe.py b/transformer_lens/model_bridge/generalized_components/moe.py index 334b7dc513..19ba06ae68 100644 --- a/transformer_lens/model_bridge/generalized_components/moe.py +++ b/transformer_lens/model_bridge/generalized_components/moe.py @@ -281,28 +281,11 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: raise RuntimeError( f"Original component not set for {self.name}. Call set_original_component() first." ) - target_dtype = None - try: - target_dtype = next(self.original_component.parameters()).dtype - except StopIteration: - pass if len(args) > 0: hooked = self.hook_in(args[0]) - if ( - target_dtype is not None - and isinstance(hooked, torch.Tensor) - and hooked.is_floating_point() - ): - hooked = hooked.to(dtype=target_dtype) args = (hooked,) + args[1:] elif "hidden_states" in kwargs: hooked = self.hook_in(kwargs["hidden_states"]) - if ( - target_dtype is not None - and isinstance(hooked, torch.Tensor) - and hooked.is_floating_point() - ): - hooked = hooked.to(dtype=target_dtype) kwargs = {**kwargs, "hidden_states": hooked} output = self.original_component(*args, **kwargs) if isinstance(output, tuple): diff --git a/transformer_lens/model_bridge/generalized_components/mpt_alibi_attention.py b/transformer_lens/model_bridge/generalized_components/mpt_alibi_attention.py index 6163d21c9c..6ea12b4750 100644 --- a/transformer_lens/model_bridge/generalized_components/mpt_alibi_attention.py +++ b/transformer_lens/model_bridge/generalized_components/mpt_alibi_attention.py @@ -92,9 +92,10 @@ def _reconstruct_attention( # MPT passes a bool 4D mask (True = masked), not an additive float mask. attention_mask = kwargs.get("attention_mask", None) if attention_mask is not None: - attn_scores = attn_scores.masked_fill( - attention_mask, torch.finfo(attn_scores.dtype).min + mask_value = ( + -torch.inf if self.compatibility_mode else torch.finfo(attn_scores.dtype).min ) + attn_scores = attn_scores.masked_fill(attention_mask, mask_value) attn_scores = self.hook_attn_scores(attn_scores) diff --git a/transformer_lens/model_bridge/generalized_components/position_embeddings_attention.py b/transformer_lens/model_bridge/generalized_components/position_embeddings_attention.py index f5dbbdaf2d..a9dffaf18d 100644 --- a/transformer_lens/model_bridge/generalized_components/position_embeddings_attention.py +++ b/transformer_lens/model_bridge/generalized_components/position_embeddings_attention.py @@ -348,20 +348,6 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: # Apply input hook hidden_states = self.hook_in(hidden_states) - # Match dtype of HF module. Skip non-fp params: quantized weights (bnb - # uint8/int8, GPTQ/AWQ int32, HQQ, torchao) are stored in integer dtypes - # and dequantized internally during matmul. The compute dtype must come - # from a fp parameter; casting fp inputs to an integer storage dtype - # destroys precision. - target_dtype = None - for p in hf_attn.parameters(): - if not p.dtype.is_floating_point: - continue - target_dtype = p.dtype - break - if target_dtype is not None and hidden_states.is_floating_point(): - hidden_states = hidden_states.to(dtype=target_dtype) - input_shape = hidden_states.shape[:-1] head_dim = hf_attn.head_dim hidden_shape = (*input_shape, -1, head_dim) @@ -622,6 +608,7 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: attn_weights = torch.nn.functional.softmax(attn_scores, dim=-1, dtype=torch.float32).to( query_states.dtype ) + attn_weights = self._scrub_compatibility_pattern_nans(attn_weights) # --- Dropout --- dropout_rate = getattr(hf_attn, "attention_dropout", 0.0) diff --git a/transformer_lens/model_bridge/generalized_components/unembedding.py b/transformer_lens/model_bridge/generalized_components/unembedding.py index ac7ab7e740..7c43e1bd9d 100644 --- a/transformer_lens/model_bridge/generalized_components/unembedding.py +++ b/transformer_lens/model_bridge/generalized_components/unembedding.py @@ -86,18 +86,7 @@ def forward(self, hidden_states: torch.Tensor, **kwargs: Any) -> torch.Tensor: raise RuntimeError( f"Original component not set for {self.name}. Call set_original_component() first." ) - target_dtype = None - try: - target_dtype = next(self.original_component.parameters()).dtype - except StopIteration: - pass hidden_states = self.hook_in(hidden_states) - if ( - target_dtype is not None - and isinstance(hidden_states, torch.Tensor) - and hidden_states.is_floating_point() - ): - hidden_states = hidden_states.to(dtype=target_dtype) output = self.original_component(hidden_states, **kwargs) output = self.hook_out(output) diff --git a/transformer_lens/model_bridge/get_params_util.py b/transformer_lens/model_bridge/get_params_util.py index a1a242ce7c..05330cedd8 100644 --- a/transformer_lens/model_bridge/get_params_util.py +++ b/transformer_lens/model_bridge/get_params_util.py @@ -1,48 +1,50 @@ """Utility function for getting model parameters in TransformerLens format.""" import logging -from typing import Dict +from typing import Dict, Optional import torch logger = logging.getLogger(__name__) -def _get_n_kv_heads(cfg) -> int: - """Resolve the number of key/value heads, falling back to n_heads.""" - if hasattr(cfg, "n_key_value_heads") and isinstance(cfg.n_key_value_heads, int): - return cfg.n_key_value_heads - return cfg.n_heads +def _tensor_attr(obj, *names: str) -> Optional[torch.Tensor]: + """First attribute of ``obj`` among ``names`` that is an actual tensor, else None. + NotImplementedError counts as absent: MLA attention raises it from W_Q/W_K/W_V/W_O + (compressed projections have no standard per-head form). + """ + for name in names: + try: + value = getattr(obj, name) + except (AttributeError, TypeError, NotImplementedError): + continue + if isinstance(value, torch.Tensor): + return value + return None -def _reshape_kv_weight(weight: torch.Tensor, cfg, device, dtype) -> torch.Tensor: - """Reshape a K or V weight matrix to (n_heads, d_model, d_head).""" - d_head = cfg.d_model // cfg.n_heads - if weight.shape == (cfg.d_model, cfg.d_model): - return weight.reshape(cfg.n_heads, cfg.d_model, d_head) - if weight.shape == (cfg.d_head, cfg.d_model) or weight.shape == ( - cfg.d_model // cfg.n_heads, - cfg.d_model, - ): - return weight.transpose(0, 1).unsqueeze(0).expand(cfg.n_heads, -1, -1) - if weight.numel() == cfg.n_heads * cfg.d_model * cfg.d_head: - return weight.view(cfg.n_heads, cfg.d_model, cfg.d_head) - return torch.zeros(cfg.n_heads, cfg.d_model, cfg.d_head, device=device, dtype=dtype) +def get_bridge_params(bridge) -> Dict[str, torch.Tensor]: + """Model parameters in SVDInterpreter format. -def _get_or_create_bias(bias, n_heads: int, d_head: int, device, dtype) -> torch.Tensor: - """Reshape existing bias to (n_heads, d_head), or create zeros if None.""" - if bias is not None: - return bias.reshape(n_heads, -1) - return torch.zeros(n_heads, d_head, device=device, dtype=dtype) + Reads the bridge components' TL-layout weight properties (``W_Q``, + ``W_in``, ...), which already account for layout conversion and weight + processing. For missing weights, returns zero tensors of appropriate shape + instead of raising exceptions. Skips attn keys for non-attention layers. + LayerNorm params (``blocks.{i}.ln1.w`` etc.) are included when the modules + still carry them (i.e. before folding) so consumers can detect fold state. + Returns: + dict: Dictionary of parameter tensors with TransformerLens naming convention -def get_bridge_params(bridge) -> Dict[str, torch.Tensor]: - """Model parameters in SVDInterpreter format. Skips attn keys for non-attention layers.""" - params_dict = {} + Raises: + ValueError: If configuration is inconsistent (e.g., cfg.n_layers != len(blocks)) + """ + cfg = bridge.cfg + params_dict: Dict[str, torch.Tensor] = {} def _get_device_dtype(): """Infer device/dtype from the first available model parameter.""" - device = getattr(bridge.cfg, "device", None) or torch.device("cpu") + device = getattr(cfg, "device", None) or torch.device("cpu") dtype = torch.float32 try: first_param = next(bridge.parameters()) @@ -52,24 +54,21 @@ def _get_device_dtype(): pass return (device, dtype) - try: - params_dict["embed.W_E"] = bridge.embed.weight - except AttributeError: - device, dtype = _get_device_dtype() - params_dict["embed.W_E"] = torch.zeros( - bridge.cfg.d_vocab, bridge.cfg.d_model, device=device, dtype=dtype - ) - try: - params_dict["pos_embed.W_pos"] = bridge.pos_embed.weight - except AttributeError: + def _zeros(*shape) -> torch.Tensor: device, dtype = _get_device_dtype() - params_dict["pos_embed.W_pos"] = torch.zeros( - bridge.cfg.n_ctx, bridge.cfg.d_model, device=device, dtype=dtype - ) - for layer_idx in range(bridge.cfg.n_layers): + return torch.zeros(*shape, device=device, dtype=dtype) + + embed = _tensor_attr(getattr(bridge, "embed", None), "W_E", "weight") + params_dict["embed.W_E"] = embed if embed is not None else _zeros(cfg.d_vocab, cfg.d_model) + + pos = _tensor_attr(getattr(bridge, "pos_embed", None), "W_pos", "weight") + params_dict["pos_embed.W_pos"] = pos if pos is not None else _zeros(cfg.n_ctx, cfg.d_model) + + for layer_idx in range(cfg.n_layers): if layer_idx >= len(bridge.blocks): raise ValueError( - f"Configuration mismatch: cfg.n_layers={bridge.cfg.n_layers} but only {len(bridge.blocks)} blocks found. Layer {layer_idx} does not exist." + f"Configuration mismatch: cfg.n_layers={cfg.n_layers} but only " + f"{len(bridge.blocks)} blocks found. Layer {layer_idx} does not exist." ) block = bridge.blocks[layer_idx] @@ -79,128 +78,120 @@ def _get_device_dtype(): except (TypeError, AttributeError): has_attn = hasattr(block, "attn") # Mock fallback if has_attn: - try: - w_q = block.attn.q.weight - w_k = block.attn.k.weight - w_v = block.attn.v.weight - w_o = block.attn.o.weight - if w_q.shape == (bridge.cfg.d_model, bridge.cfg.d_model): - d_head = bridge.cfg.d_model // bridge.cfg.n_heads - w_q = w_q.reshape(bridge.cfg.n_heads, bridge.cfg.d_model, d_head) - w_o = w_o.reshape(bridge.cfg.n_heads, d_head, bridge.cfg.d_model) - device, dtype = _get_device_dtype() - w_k = _reshape_kv_weight(w_k, bridge.cfg, device, dtype) - w_v = _reshape_kv_weight(w_v, bridge.cfg, device, dtype) + attn = block.attn + w_q = _tensor_attr(attn, "W_Q") + w_k = _tensor_attr(attn, "W_K") + w_v = _tensor_attr(attn, "W_V") + w_o = _tensor_attr(attn, "W_O") + if w_q is None or w_k is None or w_v is None or w_o is None: + logger.debug( + "Block %d has 'attn' but no TL-layout W_Q/W_K/W_V/W_O properties — " + "skipping attention weights for this layer", + layer_idx, + ) + else: + # GQA: expand grouped K/V (and their biases below) to n_heads so + # per-head pairings like SVDInterpreter's OV = W_V[h] @ W_O[h] + # line up — the legacy HT convention repeat_interleaved these. + n_kv_heads = w_k.shape[0] + if w_k.ndim == 3 and 0 < n_kv_heads < cfg.n_heads: + if cfg.n_heads % n_kv_heads != 0: + raise ValueError( + f"blocks.{layer_idx}.attn: n_heads ({cfg.n_heads}) is not " + f"divisible by n_kv_heads ({n_kv_heads}); cannot expand " + "grouped K/V to per-query heads." + ) + repeats = cfg.n_heads // n_kv_heads + w_k = torch.repeat_interleave(w_k, repeats, dim=0) + w_v = torch.repeat_interleave(w_v, repeats, dim=0) params_dict[f"blocks.{layer_idx}.attn.W_Q"] = w_q params_dict[f"blocks.{layer_idx}.attn.W_K"] = w_k params_dict[f"blocks.{layer_idx}.attn.W_V"] = w_v params_dict[f"blocks.{layer_idx}.attn.W_O"] = w_o - device, dtype = _get_device_dtype() - n_kv_heads = _get_n_kv_heads(bridge.cfg) - params_dict[f"blocks.{layer_idx}.attn.b_Q"] = _get_or_create_bias( - block.attn.q.bias, bridge.cfg.n_heads, bridge.cfg.d_head, device, dtype - ) - params_dict[f"blocks.{layer_idx}.attn.b_K"] = _get_or_create_bias( - block.attn.k.bias, n_kv_heads, bridge.cfg.d_head, device, dtype + for bias_name in ("b_Q", "b_K", "b_V"): + bias = _tensor_attr(attn, bias_name) + if bias is None: + bias = _zeros(cfg.n_heads, cfg.d_head) + elif bias.ndim == 2 and 0 < bias.shape[0] < cfg.n_heads: + bias = torch.repeat_interleave(bias, cfg.n_heads // bias.shape[0], dim=0) + params_dict[f"blocks.{layer_idx}.attn.{bias_name}"] = bias + b_O = _tensor_attr(attn, "b_O") + params_dict[f"blocks.{layer_idx}.attn.b_O"] = ( + b_O if b_O is not None else _zeros(cfg.d_model) ) - params_dict[f"blocks.{layer_idx}.attn.b_V"] = _get_or_create_bias( - block.attn.v.bias, n_kv_heads, bridge.cfg.d_head, device, dtype - ) - if block.attn.o.bias is not None: - params_dict[f"blocks.{layer_idx}.attn.b_O"] = block.attn.o.bias - else: - device, dtype = _get_device_dtype() - params_dict[f"blocks.{layer_idx}.attn.b_O"] = torch.zeros( - bridge.cfg.d_model, device=device, dtype=dtype - ) - except AttributeError as e: - logger.debug( - "Block %d has 'attn' in _modules but attention params could not " - "be extracted (missing q/k/v/o?): %s — skipping attention weights " - "for this layer", + + d_mlp = cfg.d_mlp if cfg.d_mlp is not None else 4 * cfg.d_model + mlp = getattr(block, "mlp", None) + w_in = _tensor_attr(mlp, "W_in") + if w_in is None: + if mlp is not None: + # Zero-filling a real MLP silently yields wrong numbers downstream + # (SVD/weight analyses decompose zeros). Say so — the fill stays for + # architectures that genuinely have no MLP under this name. + logger.warning( + "Block %d MLP weights could not be extracted — emitting ZEROS " + "for blocks.%d.mlp.W_in/W_out/b_in/b_out. Any weight-space " + "analysis of this layer will be meaningless.", + layer_idx, layer_idx, - e, - ) - try: - # Dense layers of an interleaved MoE stack keep their projections - # under dense_* — `gate` there is the sparse layers' ROUTER, so the - # standard names would either miss the weights (silently zero-filling - # a real dense MLP) or read the router as a gate projection. - # `is True`, not truthiness: auto-vivifying stand-ins (Mock blocks in - # this module's own tests) return a truthy object for any attribute - # and would take the dense branch with non-tensor projections. - if getattr(block.mlp, "bound_dense", False) is True: - mlp_in = getattr(block.mlp, "dense_in", None) - mlp_out = getattr(block.mlp, "dense_out", None) - mlp_gate = getattr(block.mlp, "dense_gate", None) - else: - mlp_in = getattr(block.mlp, "in", None) or getattr(block.mlp, "input", None) - mlp_out = getattr(block.mlp, "out", None) - mlp_gate = getattr(block.mlp, "gate", None) - if mlp_in is None: - raise AttributeError("MLP has no 'in' or 'input' attribute") - # Use normalized accessors for consistent TL orientation - params_dict[f"blocks.{layer_idx}.mlp.W_in"] = block.mlp.W_in - params_dict[f"blocks.{layer_idx}.mlp.W_out"] = block.mlp.W_out - mlp_in_bias = mlp_in.bias - if mlp_in_bias is not None: - params_dict[f"blocks.{layer_idx}.mlp.b_in"] = mlp_in_bias - else: - device, dtype = _get_device_dtype() - d_mlp = bridge.cfg.d_mlp if bridge.cfg.d_mlp is not None else 4 * bridge.cfg.d_model - params_dict[f"blocks.{layer_idx}.mlp.b_in"] = torch.zeros( - d_mlp, device=device, dtype=dtype - ) - mlp_out_bias = mlp_out.bias if mlp_out is not None else None - if mlp_out_bias is not None: - params_dict[f"blocks.{layer_idx}.mlp.b_out"] = mlp_out_bias - else: - device, dtype = _get_device_dtype() - params_dict[f"blocks.{layer_idx}.mlp.b_out"] = torch.zeros( - bridge.cfg.d_model, device=device, dtype=dtype ) - if mlp_gate is not None and hasattr(mlp_gate, "weight"): - w_gate = block.mlp.W_gate - if w_gate is not None: - params_dict[f"blocks.{layer_idx}.mlp.W_gate"] = w_gate - if getattr(mlp_gate, "bias", None) is not None: - params_dict[f"blocks.{layer_idx}.mlp.b_gate"] = mlp_gate.bias - except AttributeError as e: - # Zero-filling a real MLP silently yields wrong numbers downstream - # (SVD/weight analyses decompose zeros). Say so — the fill stays for - # architectures that genuinely have no MLP under this name. - logger.warning( - "Block %d MLP weights could not be extracted (%s) — emitting " - "ZEROS for blocks.%d.mlp.W_in/W_out/b_in/b_out. Any weight-space " - "analysis of this layer will be meaningless.", - layer_idx, - e, - layer_idx, - ) - device, dtype = _get_device_dtype() - d_mlp = bridge.cfg.d_mlp if bridge.cfg.d_mlp is not None else 4 * bridge.cfg.d_model - params_dict[f"blocks.{layer_idx}.mlp.W_in"] = torch.zeros( - bridge.cfg.d_model, d_mlp, device=device, dtype=dtype + params_dict[f"blocks.{layer_idx}.mlp.W_in"] = _zeros(cfg.d_model, d_mlp) + params_dict[f"blocks.{layer_idx}.mlp.W_out"] = _zeros(d_mlp, cfg.d_model) + params_dict[f"blocks.{layer_idx}.mlp.b_in"] = _zeros(d_mlp) + params_dict[f"blocks.{layer_idx}.mlp.b_out"] = _zeros(cfg.d_model) + else: + params_dict[f"blocks.{layer_idx}.mlp.W_in"] = w_in + w_out = _tensor_attr(mlp, "W_out") + params_dict[f"blocks.{layer_idx}.mlp.W_out"] = ( + w_out if w_out is not None else _zeros(d_mlp, cfg.d_model) ) - params_dict[f"blocks.{layer_idx}.mlp.W_out"] = torch.zeros( - d_mlp, bridge.cfg.d_model, device=device, dtype=dtype + b_in = _tensor_attr(mlp, "b_in") + params_dict[f"blocks.{layer_idx}.mlp.b_in"] = ( + b_in if b_in is not None else _zeros(d_mlp) ) - params_dict[f"blocks.{layer_idx}.mlp.b_in"] = torch.zeros( - d_mlp, device=device, dtype=dtype + b_out = _tensor_attr(mlp, "b_out") + params_dict[f"blocks.{layer_idx}.mlp.b_out"] = ( + b_out if b_out is not None else _zeros(cfg.d_model) ) - params_dict[f"blocks.{layer_idx}.mlp.b_out"] = torch.zeros( - bridge.cfg.d_model, device=device, dtype=dtype - ) - try: - params_dict["unembed.W_U"] = bridge.unembed.weight.T - except AttributeError: - device, dtype = _get_device_dtype() - params_dict["unembed.W_U"] = torch.zeros( - bridge.cfg.d_model, bridge.cfg.d_vocab, device=device, dtype=dtype - ) - try: - params_dict["unembed.b_U"] = bridge.unembed.b_U - except AttributeError: - device, dtype = _get_device_dtype() - params_dict["unembed.b_U"] = torch.zeros(bridge.cfg.d_vocab, device=device, dtype=dtype) + w_gate = _tensor_attr(mlp, "W_gate") + # Raw-attribute fallback is for plain gated MLPs only: `gate` on an + # interleaved-MoE component (anything exposing bound_dense) is the + # sparse layers' ROUTER, never a gate projection. + is_moe = getattr(type(mlp), "bound_dense", None) is not None + if w_gate is None and not is_moe: + w_gate = _tensor_attr(getattr(mlp, "gate", None), "weight") + if w_gate is not None: + params_dict[f"blocks.{layer_idx}.mlp.W_gate"] = w_gate + b_gate = _tensor_attr(mlp, "b_gate") + if b_gate is None and not is_moe: + b_gate = _tensor_attr(getattr(mlp, "gate", None), "bias") + if b_gate is not None: + params_dict[f"blocks.{layer_idx}.mlp.b_gate"] = b_gate + + # LN params (present pre-folding; folded models carry identities or none). + for ln_name in ("ln1", "ln2"): + ln = getattr(block, ln_name, None) + ln_w = _tensor_attr(ln, "w", "weight") + if ln_w is not None: + params_dict[f"blocks.{layer_idx}.{ln_name}.w"] = ln_w + ln_b = _tensor_attr(ln, "b", "bias") + if ln_b is not None: + params_dict[f"blocks.{layer_idx}.{ln_name}.b"] = ln_b + + ln_final_w = _tensor_attr(getattr(bridge, "ln_final", None), "w", "weight") + if ln_final_w is not None: + params_dict["ln_final.w"] = ln_final_w + ln_final_b = _tensor_attr(getattr(bridge, "ln_final", None), "b", "bias") + if ln_final_b is not None: + params_dict["ln_final.b"] = ln_final_b + + unembed = getattr(bridge, "unembed", None) + w_u = _tensor_attr(unembed, "W_U") + if w_u is None: + raw = _tensor_attr(unembed, "weight") + w_u = raw.T if raw is not None else _zeros(cfg.d_model, cfg.d_vocab) + params_dict["unembed.W_U"] = w_u + b_u = _tensor_attr(unembed, "b_U") + params_dict["unembed.b_U"] = b_u if b_u is not None else _zeros(cfg.d_vocab) + return params_dict diff --git a/transformer_lens/model_bridge/sources/native/init.py b/transformer_lens/model_bridge/sources/native/init.py index 229ac7e27f..bc8fdd31e1 100644 --- a/transformer_lens/model_bridge/sources/native/init.py +++ b/transformer_lens/model_bridge/sources/native/init.py @@ -29,37 +29,60 @@ ) # Residual-scaled output is gpt2-specific; other modes treat every weight the -# same. Each entry takes ``(tensor, generator)`` to thread the scoped Generator. -_NonResidualInit = Callable[[torch.Tensor, Optional[torch.Generator]], torch.Tensor] +# same. Each entry takes ``(tensor, generator, gain)`` — gain honors +# ``cfg.initializer_range`` like the legacy init did (which passed it as the +# xavier/kaiming gain); kaiming has no gain kwarg, so scale after. +_NonResidualInit = Callable[[torch.Tensor, Optional[torch.Generator], float], torch.Tensor] _NON_RESIDUAL_MODES: dict[str, _NonResidualInit] = { - "xavier_uniform": lambda t, g: nn.init.xavier_uniform_(t, generator=g), - "xavier_normal": lambda t, g: nn.init.xavier_normal_(t, generator=g), - "kaiming_uniform": lambda t, g: nn.init.kaiming_uniform_(t, nonlinearity="relu", generator=g), - "kaiming_normal": lambda t, g: nn.init.kaiming_normal_(t, nonlinearity="relu", generator=g), + "xavier_uniform": lambda t, g, gain: nn.init.xavier_uniform_(t, gain=gain, generator=g), + "xavier_normal": lambda t, g, gain: nn.init.xavier_normal_(t, gain=gain, generator=g), + "kaiming_uniform": lambda t, g, gain: nn.init.kaiming_uniform_( + t, nonlinearity="relu", generator=g + ).mul_(gain), + "kaiming_normal": lambda t, g, gain: nn.init.kaiming_normal_( + t, nonlinearity="relu", generator=g + ).mul_(gain), } _SUPPORTED_MODES = frozenset({"gpt2", *_NON_RESIDUAL_MODES}) +def _unwrap_component(module: nn.Module) -> nn.Module: + """Return the native module stored behind a bridge wrapper, if present.""" + original = getattr(module, "original_component", None) + return original if isinstance(original, nn.Module) else module + + def initialize_native_model( model: NativeModel, cfg: TransformerBridgeConfig, seed: int | None = None ) -> None: """Initialize ``model`` weights in-place. Honors ``cfg.init_mode`` and ``cfg.seed``.""" effective_seed = seed if seed is not None else cfg.seed - # Scoped generator on the model's device — None falls back to the global RNG. - try: - gen_device = next(model.parameters()).device - except StopIteration: - gen_device = torch.device("cpu") + # Always generate on CPU/fp32 and copy into the parameter: boot initializes + # before .to(device)/.to(dtype) while init_weights() runs after, and a + # generator seeded on the live parameter device produces a different stream + # — the same seed must reproduce the same weights either way. generator: Optional[torch.Generator] if effective_seed is not None: - g = torch.Generator(device=gen_device) + g = torch.Generator() g.manual_seed(effective_seed) generator = g else: generator = None + def _staged( + fn: Callable[[torch.Tensor], torch.Tensor], + ) -> Callable[[torch.Tensor], torch.Tensor]: + def apply(t: torch.Tensor) -> torch.Tensor: + staging = torch.empty(t.shape, dtype=torch.float32) + fn(staging) + with torch.no_grad(): + t.copy_(staging) + return t + + return apply + init_mode = (cfg.init_mode or "gpt2").lower() if init_mode not in _SUPPORTED_MODES: raise NotImplementedError( @@ -70,7 +93,18 @@ def initialize_native_model( weight_init: Callable[[torch.Tensor], torch.Tensor] output_init: Callable[[torch.Tensor], torch.Tensor] if init_mode == "gpt2": - std = cfg.initializer_range if cfg.initializer_range > 0 else 0.02 + # Default matches the legacy TL scheme: N(0, 0.64/d_model), i.e. + # std = 0.8/sqrt(d_model), not GPT-2's paper 0.02 — toy-model training + # dynamics (e.g. the grokking demo) depend on this scale. + std = cfg.initializer_range if cfg.initializer_range > 0 else 0.8 / math.sqrt(cfg.d_model) + + # NOTE: this residual output scaling (1/sqrt(2*n_layers), applied only + # to output projections below) is NOT present in HookedTransformer's + # _init_weights_gpt2 (see transformer_lens/HookedTransformer.py). + # Intentional delta for NativeModel: kept because it follows the + # original GPT-2 paper's residual-scaling convention and improves + # training stability at init for deeper models. Flagged in issue #1568 + # as a maintainer call; kept + documented rather than removed. residual_scale = 1.0 / math.sqrt(2 * cfg.n_layers) weight_init = lambda t: nn.init.normal_( t, mean=0.0, std=std, generator=generator @@ -80,22 +114,35 @@ def initialize_native_model( ) else: fn = _NON_RESIDUAL_MODES[init_mode] - weight_init = lambda t: fn(t, generator) # noqa: E731 + # Honor an explicitly-set initializer_range as the gain (legacy + # behavior); the sentinel/default keeps plain xavier/kaiming scaling. + gain = cfg.initializer_range if cfg.initializer_range > 0 else 1.0 + weight_init = lambda t: fn(t, generator, gain) # noqa: E731 output_init = weight_init - weight_init(model.tok_embed.weight) + weight_init = _staged(weight_init) + output_init = _staged(output_init) + + tok_embed = cast(nn.Embedding, _unwrap_component(model.tok_embed)) + weight_init(tok_embed.weight) if model.pos is not None: - weight_init(model.pos.weight) + pos = cast(nn.Embedding, _unwrap_component(model.pos)) + weight_init(pos.weight) # Rotary has only registered buffers (cos/sin), no parameters to init. for block in model.layers: - _init_block(block, weight_init=weight_init, output_init=output_init) + native_block = cast(NativeBlock, _unwrap_component(block)) + _init_block(native_block, weight_init=weight_init, output_init=output_init) _init_norm(model.ln_out) - weight_init(model.head.weight) + head = cast(nn.Linear, _unwrap_component(model.head)) + weight_init(head.weight) + if head.bias is not None: + nn.init.zeros_(head.bias) def _init_norm(norm: nn.Module) -> None: + norm = _unwrap_component(norm) if isinstance(norm, NativeRMSNorm): nn.init.ones_(norm.weight) elif isinstance(norm, nn.LayerNorm): @@ -114,13 +161,19 @@ def _init_block( output_init: Callable[[torch.Tensor], torch.Tensor], ) -> None: _init_norm(block.ln1) - _init_attention(block.attn, weight_init=weight_init, output_init=output_init) + attn = cast(NativeAttention, _unwrap_component(block.attn)) + _init_attention(attn, weight_init=weight_init, output_init=output_init) if not block.cfg.attn_only: _init_norm(block.ln2) - if isinstance(block.mlp, NativeGatedMLP): - _init_gated_mlp(block.mlp, weight_init=weight_init, output_init=output_init) + mlp = _unwrap_component(block.mlp) + if isinstance(mlp, NativeGatedMLP): + _init_gated_mlp(mlp, weight_init=weight_init, output_init=output_init) else: - _init_mlp(block.mlp, weight_init=weight_init, output_init=output_init) + _init_mlp( + cast(NativeMLP, mlp), + weight_init=weight_init, + output_init=output_init, + ) def _init_attention( @@ -129,13 +182,15 @@ def _init_attention( weight_init: Callable[[torch.Tensor], torch.Tensor], output_init: Callable[[torch.Tensor], torch.Tensor], ) -> None: - for linear in (attn.q, attn.k, attn.v): + for component in (attn.q, attn.k, attn.v): + linear = cast(nn.Linear, _unwrap_component(component)) weight_init(linear.weight) if linear.bias is not None: nn.init.zeros_(linear.bias) - output_init(attn.o.weight) - if attn.o.bias is not None: - nn.init.zeros_(attn.o.bias) + output = cast(nn.Linear, _unwrap_component(attn.o)) + output_init(output.weight) + if output.bias is not None: + nn.init.zeros_(output.bias) def _init_mlp( @@ -144,10 +199,12 @@ def _init_mlp( weight_init: Callable[[torch.Tensor], torch.Tensor], output_init: Callable[[torch.Tensor], torch.Tensor], ) -> None: - weight_init(mlp.fc_in.weight) - nn.init.zeros_(mlp.fc_in.bias) - output_init(mlp.fc_out.weight) - nn.init.zeros_(mlp.fc_out.bias) + fc_in = cast(nn.Linear, _unwrap_component(mlp.fc_in)) + fc_out = cast(nn.Linear, _unwrap_component(mlp.fc_out)) + weight_init(fc_in.weight) + nn.init.zeros_(fc_in.bias) + output_init(fc_out.weight) + nn.init.zeros_(fc_out.bias) def _init_gated_mlp( @@ -156,8 +213,10 @@ def _init_gated_mlp( weight_init: Callable[[torch.Tensor], torch.Tensor], output_init: Callable[[torch.Tensor], torch.Tensor], ) -> None: - weight_init(mlp.gate.weight) + gate = cast(nn.Linear, _unwrap_component(mlp.gate)) + weight_init(gate.weight) # ``in`` is registered via add_module; getattr resolves it from _modules. - in_proj = cast(nn.Linear, getattr(mlp, "in")) + in_proj = cast(nn.Linear, _unwrap_component(getattr(mlp, "in"))) + out_proj = cast(nn.Linear, _unwrap_component(mlp.out)) weight_init(in_proj.weight) - output_init(mlp.out.weight) + output_init(out_proj.weight) diff --git a/transformer_lens/model_bridge/sources/native/model.py b/transformer_lens/model_bridge/sources/native/model.py index 5617459e79..995a8664bd 100644 --- a/transformer_lens/model_bridge/sources/native/model.py +++ b/transformer_lens/model_bridge/sources/native/model.py @@ -281,6 +281,9 @@ def forward( scores = scores.masked_fill(block_mask, float("-inf")) pattern = F.softmax(scores, dim=-1) + # Fully masked padding queries softmax to NaN; overwrite masked entries + # so those rows contribute a zero attention update instead of poisoning later layers. + pattern = pattern.masked_fill(block_mask, 0.0) attn = torch.matmul(pattern, v).transpose(1, 2).contiguous().view(batch, seq, -1) out = self.o(attn) @@ -436,15 +439,27 @@ def __init__(self, cfg: TransformerBridgeConfig): def forward( self, - input_ids: torch.Tensor, + input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, **kwargs, ) -> torch.Tensor: """Returns logits directly.""" + if input_ids is not None and inputs_embeds is not None: + raise ValueError("Exactly one of input_ids or inputs_embeds must be provided.") + if input_ids is not None: + model_input = input_ids + hidden_states = self.tok_embed(input_ids) + elif inputs_embeds is not None: + model_input = inputs_embeds + hidden_states = inputs_embeds + else: + raise ValueError("Exactly one of input_ids or inputs_embeds must be provided.") + # Bounds check up front so both absolute and rotary paths produce a # self-explanatory error rather than IndexError / shape mismatch. - seq_len = input_ids.shape[-1] + seq_len = model_input.shape[1] if seq_len > self.cfg.n_ctx: raise ValueError( f"input length {seq_len} exceeds n_ctx={self.cfg.n_ctx}; " @@ -453,11 +468,12 @@ def forward( # Resolve position_ids before the block loop so rotary sees the caller's # positions, not the dense default. - batch, seq = input_ids.shape + batch, seq = model_input.shape[:2] if position_ids is None: - position_ids = torch.arange(seq, device=input_ids.device).unsqueeze(0).expand(batch, -1) + position_ids = ( + torch.arange(seq, device=model_input.device).unsqueeze(0).expand(batch, -1) + ) - hidden_states = self.tok_embed(input_ids) if self.pos is not None: hidden_states = hidden_states + self.pos(position_ids) diff --git a/transformer_lens/model_bridge/sources/transformers.py b/transformer_lens/model_bridge/sources/transformers.py index 40d40ca190..25fa5cb1e6 100644 --- a/transformer_lens/model_bridge/sources/transformers.py +++ b/transformer_lens/model_bridge/sources/transformers.py @@ -750,7 +750,6 @@ def boot( # resolved values. from transformer_lens.utilities.multi_gpu import ( MIXED_CPU_GPU_ERROR, - cast_floating_params_to_dtype, count_unique_devices, find_embedding_device, find_misplaced_modules, @@ -847,7 +846,11 @@ def boot( # Cast params to dtype; preserve float32 buffers (e.g., RotaryEmbedding.inv_freq). # Use module-level alignment so Accelerate can temporarily materialize offloaded # parameters before we touch them. - cast_floating_params_to_dtype(hf_model, dtype) + # Skip dtype normalization entirely when model has an active quantizer: the + # quantizer owns specific dtypes (e.g., FP8 scales) that must not be overwritten. + from transformer_lens.utilities.multi_gpu import maybe_cast_floating_params + + maybe_cast_floating_params(hf_model, dtype) # Derive cfg.device / cfg.n_devices from hf_device_map when present. This covers: # - fresh loads with a resolved device_map (set above) # - pre-loaded hf_model that the caller dispatched themselves (e.g., device_map="auto") @@ -959,10 +962,16 @@ def boot( if tokenizer is not None: # Detect BOS/EOS behavior (use non-empty string; empty is unreliable with token aliasing) encoded_test = tokenizer.encode("a") + leading_special_ids = { + token_id + for token_id in (tokenizer.bos_token_id, getattr(tokenizer, "cls_token_id", None)) + if token_id is not None + } + # CLS counts: BERT-style tokenizers prepend [CLS], which HookedTransformer + # treats as the BOS-like token; comparing only against bos_token_id (a + # fallback string on such tokenizers) concludes False and desyncs the stacks. adapter.cfg.tokenizer_prepends_bos = ( - len(encoded_test) > 1 - and tokenizer.bos_token_id is not None - and encoded_test[0] == tokenizer.bos_token_id + len(encoded_test) > 1 and encoded_test[0] in leading_special_ids ) adapter.cfg.tokenizer_appends_eos = ( len(encoded_test) > 1 diff --git a/transformer_lens/model_bridge/supported_architectures/ast.py b/transformer_lens/model_bridge/supported_architectures/ast.py index 78bde24b9d..5ecd7c7134 100644 --- a/transformer_lens/model_bridge/supported_architectures/ast.py +++ b/transformer_lens/model_bridge/supported_architectures/ast.py @@ -128,6 +128,11 @@ def prepare_model(self, hf_model: Any) -> None: and hasattr(hf_model, "classifier") and hasattr(hf_model.classifier, "dense") ): + self.component_mapping["classifier_ln"] = NormalizationBridge( + name="classifier.layernorm", + config=self.cfg, + use_native_layernorm_autograd=True, + ) self.component_mapping["unembed"] = UnembeddingBridge(name="classifier.dense") self.cfg.d_vocab = num_labels self.cfg.d_vocab_out = num_labels diff --git a/transformer_lens/model_bridge/supported_architectures/bert.py b/transformer_lens/model_bridge/supported_architectures/bert.py index f58c841711..f71a237d6a 100644 --- a/transformer_lens/model_bridge/supported_architectures/bert.py +++ b/transformer_lens/model_bridge/supported_architectures/bert.py @@ -87,7 +87,13 @@ def __init__(self, cfg: Any) -> None: # MLM defaults; prepare_model() adjusts for other task heads (e.g., NSP). self.component_mapping = { "embed": EmbeddingBridge(name="bert.embeddings.word_embeddings"), + "token_type_embed": EmbeddingBridge(name="bert.embeddings.token_type_embeddings"), "pos_embed": PosEmbedBridge(name="bert.embeddings.position_embeddings"), + "embed_ln": NormalizationBridge( + name="bert.embeddings.LayerNorm", + config=self.cfg, + use_native_layernorm_autograd=True, + ), "blocks": BlockBridge( name="bert.encoder.layer", # BERT has no single MLP module (intermediate.dense and output.dense @@ -129,6 +135,7 @@ def __init__(self, cfg: Any) -> None: ), }, ), + "mlm_head": LinearBridge(name="cls.predictions.transform.dense"), "unembed": UnembeddingBridge(name="cls.predictions.decoder"), "ln_final": NormalizationBridge( name="cls.predictions.transform.LayerNorm", @@ -144,7 +151,15 @@ def prepare_model(self, hf_model: Any) -> None: BertForNextSentencePrediction has cls.seq_relationship (NSP head) and no MLM-specific LayerNorm. """ - if hasattr(hf_model, "cls") and hasattr(hf_model.cls, "seq_relationship"): - # NSP model — swap head components + if getattr(getattr(hf_model, "bert", None), "pooler", None) is not None: + self.components["pooler"] = LinearBridge(name="bert.pooler.dense") + + has_predictions = hasattr(getattr(hf_model, "cls", None), "predictions") + has_nsp_head = hasattr(getattr(hf_model, "cls", None), "seq_relationship") + if has_nsp_head and has_predictions: + self.components["nsp_head"] = LinearBridge(name="cls.seq_relationship") + elif has_nsp_head: + # NSP-only model — swap head components. self.components["unembed"] = UnembeddingBridge(name="cls.seq_relationship") + self.components.pop("mlm_head", None) self.components.pop("ln_final", None) diff --git a/transformer_lens/model_bridge/supported_architectures/cohere.py b/transformer_lens/model_bridge/supported_architectures/cohere.py index 11d977cccc..bbc377e6bd 100644 --- a/transformer_lens/model_bridge/supported_architectures/cohere.py +++ b/transformer_lens/model_bridge/supported_architectures/cohere.py @@ -185,6 +185,9 @@ class _Cohere2AttentionBridge(PositionEmbeddingsAttentionBridge): reconstruction path. """ + # Nulls position_embeddings on NoPE layers by design. + rope_optional = True + def forward(self, *args: Any, **kwargs: Any) -> Any: """Drop position_embeddings on Cohere2 full-attention NoPE layers.""" if self._is_nope_layer(): diff --git a/transformer_lens/model_bridge/supported_architectures/dream.py b/transformer_lens/model_bridge/supported_architectures/dream.py index e11fd35050..12f444867c 100644 --- a/transformer_lens/model_bridge/supported_architectures/dream.py +++ b/transformer_lens/model_bridge/supported_architectures/dream.py @@ -100,6 +100,8 @@ class DreamArchitectureAdapter(Qwen2ArchitectureAdapter): # sampler's text (benchmarks route through diffusion_generate). applicable_phases: list[int] = [1, 2, 3, 4] supports_generation: bool = False + # Bidirectional masked-denoising objective; shifted causal CE is undefined. + supports_causal_loss: bool = False # Sampling is iterative denoising, not left-to-right; Dream ships the # schedule as a mixin method whose per-step forward goes through __call__, # so bridge hooks fire during sampling. diff --git a/transformer_lens/model_bridge/supported_architectures/gidd.py b/transformer_lens/model_bridge/supported_architectures/gidd.py index 5a269507a7..6a65b7d8d7 100644 --- a/transformer_lens/model_bridge/supported_architectures/gidd.py +++ b/transformer_lens/model_bridge/supported_architectures/gidd.py @@ -57,6 +57,8 @@ class GiddArchitectureAdapter(ArchitectureAdapter): applicable_phases: list[int] = [1, 2, 3, 4] supports_generation: bool = False + # Bidirectional masked-denoising objective; shifted causal CE is undefined. + supports_causal_loss: bool = False # Block-wise denoising with self-correction, shipped on the model class. native_sampler: str = "generate" # ScaledLinear applies a runtime weight scale; folding norms into those diff --git a/transformer_lens/model_bridge/supported_architectures/llada.py b/transformer_lens/model_bridge/supported_architectures/llada.py index 36eae41a65..5a4d867847 100644 --- a/transformer_lens/model_bridge/supported_architectures/llada.py +++ b/transformer_lens/model_bridge/supported_architectures/llada.py @@ -124,6 +124,7 @@ def forward( attn_scores = self.hook_attn_scores(attn_scores) pattern = torch.nn.functional.softmax(attn_scores, dim=-1, dtype=torch.float32).to(q.dtype) + pattern = self._scrub_compatibility_pattern_nans(pattern) dropout = float(getattr(block.config, "attention_dropout", 0.0)) if block.training and dropout > 0.0: pattern = torch.nn.functional.dropout(pattern, p=dropout, training=True) diff --git a/transformer_lens/model_bridge/supported_architectures/nanogpt.py b/transformer_lens/model_bridge/supported_architectures/nanogpt.py index 13d381d2cf..5ba52e4bf8 100644 --- a/transformer_lens/model_bridge/supported_architectures/nanogpt.py +++ b/transformer_lens/model_bridge/supported_architectures/nanogpt.py @@ -1,7 +1,5 @@ from typing import Any -import torch - from transformer_lens.conversion_utils.conversion_steps import RearrangeTensorConversion from transformer_lens.conversion_utils.param_processing_conversion import ( ParamProcessingConversion, @@ -88,16 +86,3 @@ def __init__(self, cfg: Any) -> None: ), # Final layer norm "unembed": UnembeddingBridge(name="lm_head"), } - - def convert_weights(self, remote_module: Any) -> dict[str, torch.Tensor]: - # Nanogpt models saved after torch.compile() have this unwanted prefix - # This is a simple way to remove it - unwanted_prefix = "_orig_mod." - state_dict: dict[str, torch.Tensor] = ( - remote_module.state_dict() if hasattr(remote_module, "state_dict") else remote_module - ) - for k, v in list(state_dict.items()): - if k.startswith(unwanted_prefix): - state_dict[k[len(unwanted_prefix) :]] = state_dict.pop(k) - - return super().convert_weights(remote_module) # type: ignore[misc] diff --git a/transformer_lens/model_bridge/supported_architectures/pretrain.py b/transformer_lens/model_bridge/supported_architectures/pretrain.py index 20091b65ff..5a7dfe269b 100644 --- a/transformer_lens/model_bridge/supported_architectures/pretrain.py +++ b/transformer_lens/model_bridge/supported_architectures/pretrain.py @@ -368,11 +368,10 @@ class PretrainArchitectureAdapter(ArchitectureAdapter): def __init__(self, cfg: Any) -> None: super().__init__(cfg) - self.cfg.normalization_type = "RMS" - self.cfg.positional_embedding_type = "rotary" - self.cfg.final_rms = True - self.cfg.gated_mlp = True - self.cfg.attn_only = False + # Also sets uses_rms_norm=True: norm bridges fall back to it when the + # wrapped norm's class name doesn't identify itself as RMSNorm, and a + # False fallback would mean-center RMS hook intermediates. + self._set_rms_rotary_defaults() self.component_mapping = { # "inner." because this adapter expects the source model to diff --git a/transformer_lens/model_bridge/supported_architectures/raven.py b/transformer_lens/model_bridge/supported_architectures/raven.py index 85ada2a2b1..cafdcb46ba 100644 --- a/transformer_lens/model_bridge/supported_architectures/raven.py +++ b/transformer_lens/model_bridge/supported_architectures/raven.py @@ -135,6 +135,10 @@ class RavenArchitectureAdapter(ArchitectureAdapter): # state make the phases non-meaningful. Correctness lives in the # integration tests (seed pinned before bridge and HF calls). applicable_phases: list[int] = [] + # Depth-recurrent core: HF-style past_key_values stepping cannot represent + # the re-injected recurrence, and batched left-padded stepping compounds it. + supports_kv_cache = False + supports_batched_generation = False def __init__(self, cfg: Any) -> None: """Initialize the Raven / Huginn architecture adapter.""" diff --git a/transformer_lens/model_bridge/supported_architectures/vit.py b/transformer_lens/model_bridge/supported_architectures/vit.py index 1aa5394b0a..eef75a916c 100644 --- a/transformer_lens/model_bridge/supported_architectures/vit.py +++ b/transformer_lens/model_bridge/supported_architectures/vit.py @@ -204,3 +204,5 @@ def prepare_model(self, hf_model: Any) -> None: self.component_mapping = self._build_component_mapping( prefix=prefix, with_classifier=with_classifier ) + if not with_classifier and getattr(hf_model, "pooler", None) is not None: + self.component_mapping["pooler"] = LinearBridge(name="pooler.dense") diff --git a/transformer_lens/tools/analysis/__init__.py b/transformer_lens/tools/analysis/__init__.py index e15800c5fc..a02d225a67 100644 --- a/transformer_lens/tools/analysis/__init__.py +++ b/transformer_lens/tools/analysis/__init__.py @@ -28,15 +28,21 @@ ) from transformer_lens.tools.analysis.jacobian_lens_decomposition import ( JSpaceDecomposition, + JSpaceOccupancy, + JSpaceVarianceProfile, + estimate_occupancy, get_sparse_decomposition, ) __all__ = [ "DirectLogitAttribution", "JSpaceDecomposition", + "JSpaceOccupancy", + "JSpaceVarianceProfile", "JacobianLens", "JacobianLensReadout", "direct_logit_attribution", + "estimate_occupancy", "get_act_patch_direct_path", "get_act_patch_direct_path_all_sources", "get_sparse_decomposition", diff --git a/transformer_lens/tools/analysis/jacobian_lens.py b/transformer_lens/tools/analysis/jacobian_lens.py index d4e568cb60..810f457d5b 100644 --- a/transformer_lens/tools/analysis/jacobian_lens.py +++ b/transformer_lens/tools/analysis/jacobian_lens.py @@ -73,6 +73,9 @@ from transformer_lens.tools.analysis.jacobian_lens_decomposition import ( DEFAULT_K, JSpaceDecomposition, + JSpaceOccupancy, + JSpaceVarianceProfile, + estimate_occupancy, get_sparse_decomposition, ) from transformer_lens.utilities.hf_utils import call_hf_with_retry @@ -899,6 +902,27 @@ def decompose( RuntimeError: If the default nonnegative least-squares solver cannot validate its result against the KKT conditions. """ + activation, resolved_layer = self._resolve_activation( + model, activation_or_prompt, layer, position + ) + dictionary = self.lens_vector_dictionary(model, resolved_layer) + return get_sparse_decomposition( + activation.float().to(dictionary.device), dictionary, k, algorithm=algorithm + ) + + def _resolve_activation( + self, + model: Any, + activation_or_prompt: Union[torch.Tensor, str], + layer: int, + position: Optional[int], + ) -> Tuple[torch.Tensor, int]: + """Validate the model and layer, then resolve either a raw ``[d_model]`` activation or a + prompt plus ``position`` to the activation vector to analyse. + + Returns ``(activation, resolved_layer)``. Shared by :meth:`decompose` and + :meth:`occupancy` so both accept the same input forms with identical validation. + """ self.validate_model(model) resolved_layer = _normalize_layer(layer, model.cfg.n_layers) if resolved_layer not in self.jacobians: @@ -938,10 +962,169 @@ def decompose( _, cache = model.run_with_cache(tokens, names_filter=lambda name: name == hook_name) norm_position = _normalize_positions([position], tokens.shape[1])[0] activation = cache[hook_name][0, norm_position, :] + return activation, resolved_layer + @torch.no_grad() + def occupancy( + self, + model: Any, + activation_or_prompt: Union[torch.Tensor, str], + layer: int, + *, + position: Optional[int] = None, + max_atoms: int = DEFAULT_K, + num_control_dictionaries: int = 32, + seed: int = 0, + ) -> JSpaceOccupancy: + """Estimate how many J-lens vectors are meaningfully active in an activation at ``layer``. + + Resolves ``activation_or_prompt`` (a raw ``[d_model]`` vector, or a prompt plus + ``position``) exactly as :meth:`decompose`, builds the cached full-vocabulary dictionary + via :meth:`lens_vector_dictionary`, and calls :func:`estimate_occupancy`. + + Args: + model: A raw ``TransformerBridge``. + activation_or_prompt: An activation vector, or a prompt (string / token tensor). + layer: Source layer (must be a fitted source layer). + position: Token position when a prompt is given; ``None`` for a raw activation. + max_atoms: Maximum number of J-lens vectors to consider. + num_control_dictionaries: Number of random control dictionaries to average over. + seed: Seed for the random control dictionaries (reproducibility). + + Returns: + A :class:`JSpaceOccupancy`. + """ + activation, resolved_layer = self._resolve_activation( + model, activation_or_prompt, layer, position + ) dictionary = self.lens_vector_dictionary(model, resolved_layer) - return get_sparse_decomposition( - activation.float().to(dictionary.device), dictionary, k, algorithm=algorithm + return estimate_occupancy( + activation.float().to(dictionary.device), + dictionary, + max_atoms=max_atoms, + num_control_dictionaries=num_control_dictionaries, + seed=seed, + ) + + @torch.no_grad() + def fraction_of_variance( + self, + model: Any, + prompts: Union[str, torch.Tensor, Sequence[Union[str, torch.Tensor]]], + layers: Optional[Sequence[int]] = None, + *, + k: int = DEFAULT_K, + skip_first: int = 16, + positions: Optional[Sequence[int]] = None, + show_progress: bool = False, + ) -> JSpaceVarianceProfile: + """Profile the J-space share of activation variance over a prompt corpus. + + Each prompt is run once (caching ``blocks.{layer}.hook_out`` for every requested layer). + At each sampled position the activation is decomposed and its J-space variance fraction + ``||j_space_component||^2 / ||activation||^2`` is recorded. The numerator is the + ``j_space_component`` -- the orthogonal projection of the activation onto the span of the + selected support (the paper's appendix "J-space component"), *not* the nonnegative + ``reconstruction``; the two coincide only when every selected atom stays active. Per layer + the profile reports the median of those fractions and the pooled ratio + ``sum(||j_space_component||^2) / sum(||activation||^2)`` (the paper's "fraction of total + variance"). + + A layer that samples no positions -- every prompt shorter than ``skip_first``, or only + zero-norm activations -- contributes no fractions: its ``median`` and ``pooled`` are + ``float("nan")`` and its ``per_position`` tensor is empty. + + Args: + model: A raw ``TransformerBridge``. + prompts: A prompt, or a sequence of prompts. Each token tensor must represent exactly + one prompt and have shape ``[1, seq]``. + layers: Source layers to profile; defaults to all fitted ``source_layers``. + k: Number of J-lens vectors per decomposition. + skip_first: Non-negative index before which positions are skipped (mirrors the fit's + early-position skip); not used for sampling when ``positions`` is given. + positions: Explicit positions to sample instead of ``skip_first`` onward. + show_progress: Show a tqdm progress bar over prompts. + + Returns: + A :class:`JSpaceVarianceProfile`. + + Raises: + ValueError: On an invalid model, an unfitted layer, an empty corpus, a negative + ``skip_first``, or a token tensor that does not have shape ``[1, seq]``. + """ + self.validate_model(model) + if skip_first < 0: + raise ValueError(f"skip_first must be non-negative, got {skip_first}") + if layers is None: + resolved_layers = list(self.source_layers) + else: + resolved_layers = [_normalize_layer(layer, model.cfg.n_layers) for layer in layers] + for layer in resolved_layers: + if layer not in self.jacobians: + raise ValueError( + f"layer {layer} is not in this lens's source layers; " + f"available: {self.source_layers}" + ) + prompt_list: List[Union[str, torch.Tensor]] = ( + [prompts] if isinstance(prompts, (str, torch.Tensor)) else list(prompts) + ) + if not prompt_list: + raise ValueError("prompts must be a non-empty prompt or sequence of prompts") + + hook_names = {layer: _resid_post_hook_name(layer) for layer in resolved_layers} + wanted_hooks = set(hook_names.values()) + dictionaries = { + layer: self.lens_vector_dictionary(model, layer) for layer in resolved_layers + } + fractions: Dict[int, List[float]] = {layer: [] for layer in resolved_layers} + pooled_j_space: Dict[int, float] = {layer: 0.0 for layer in resolved_layers} + pooled_total: Dict[int, float] = {layer: 0.0 for layer in resolved_layers} + + for prompt in tqdm(prompt_list, desc="J-space variance", disable=not show_progress): + tokens = model.to_tokens(prompt) if isinstance(prompt, str) else prompt + if tokens.ndim != 2 or tokens.shape[0] != 1: + raise ValueError( + "fraction_of_variance expects each tokenized prompt to have shape " + f"[1, seq], got {tuple(tokens.shape)}" + ) + _, cache = model.run_with_cache(tokens, names_filter=lambda name: name in wanted_hooks) + seq_len = tokens.shape[1] + sampled = ( + list(range(skip_first, seq_len)) + if positions is None + else _normalize_positions(positions, seq_len) + ) + for layer in resolved_layers: + dictionary = dictionaries[layer] + activations = cache[hook_names[layer]][0] # [seq, d_model] + for position in sampled: + activation = activations[position].float().to(dictionary.device) + total = float(activation @ activation) + if total <= 0.0: + continue + decomposition = get_sparse_decomposition(activation, dictionary, k) + j_space = float( + decomposition.j_space_component @ decomposition.j_space_component + ) + fractions[layer].append(j_space / total) + pooled_j_space[layer] += j_space + pooled_total[layer] += total + + median = { + layer: float(torch.tensor(fractions[layer]).median()) + if fractions[layer] + else float("nan") + for layer in resolved_layers + } + pooled = { + layer: pooled_j_space[layer] / pooled_total[layer] + if pooled_total[layer] > 0 + else float("nan") + for layer in resolved_layers + } + per_position = {layer: torch.tensor(fractions[layer]) for layer in resolved_layers} + return JSpaceVarianceProfile( + layers=resolved_layers, median=median, pooled=pooled, per_position=per_position ) # ------------------------------------------------------------------ # diff --git a/transformer_lens/tools/analysis/jacobian_lens_decomposition.py b/transformer_lens/tools/analysis/jacobian_lens_decomposition.py index 4ae46e832c..38040e037f 100644 --- a/transformer_lens/tools/analysis/jacobian_lens_decomposition.py +++ b/transformer_lens/tools/analysis/jacobian_lens_decomposition.py @@ -61,7 +61,7 @@ import math from dataclasses import dataclass -from typing import List +from typing import Dict, List, Tuple import torch @@ -447,3 +447,174 @@ def get_sparse_decomposition( j_space_component=j_space_component, non_j_space_component=non_j_space_component, ) + + +@dataclass +class JSpaceOccupancy: + """Result of a J-space occupancy estimate. + + Attributes: + occupancy: Estimated number of meaningfully-active atoms -- the step of maximum + separation between the real and random-control cumulative captured variance. + marginal_captured_variance: Per-step captured-variance gain of the real greedy selection, + shape ``[max_atoms]``. + control_captured_variance: Per-step captured-variance gain averaged over the random + control dictionaries, shape ``[max_atoms]``. + support: Greedily selected atom indices, shape ``[max_atoms]`` (token ids when the + dictionary is the vocabulary of J-lens vectors). + """ + + occupancy: int + marginal_captured_variance: torch.Tensor + control_captured_variance: torch.Tensor + support: torch.Tensor + + +def _greedy_captured_variance_gains( + atoms: torch.Tensor, atom_norms: torch.Tensor, target: torch.Tensor, max_atoms: int +) -> Tuple[torch.Tensor, torch.Tensor]: + """Greedily select exactly ``max_atoms`` atoms and return captured-variance gains. + + At each step, add the unused atom with the greatest signed, norm-normalized correlation with + the current residual. Project ``target`` orthogonally onto the full selected span using a + pseudoinverse, then set the next residual to ``target - projection``. The captured variance is + ``||Pi_S target||^2 / ||target||^2``; the returned values are its per-step increments. + + This shares the per-step correlation rule with :func:`get_sparse_decomposition`, but not its + residual recurrence: sparse decomposition uses a nonnegative coefficient-fit residual and may + stop early, while this recurrence does not stop early, so the selected supports can differ. + """ + total_variance = float(target @ target) + support: List[int] = [] + residual = target.clone() + captured_variance_gains: List[float] = [] + previous_captured_variance = 0.0 + for _ in range(max_atoms): + correlation = (atoms @ residual) / atom_norms + for chosen in support: + correlation[chosen] = float("-inf") + support.append(int(torch.argmax(correlation).item())) + active_atoms = atoms[support].T + projection = active_atoms @ (torch.linalg.pinv(active_atoms) @ target) + captured_variance = float((projection @ projection) / total_variance) + captured_variance_gains.append(captured_variance - previous_captured_variance) + previous_captured_variance = captured_variance + residual = target - projection + return torch.tensor(captured_variance_gains), torch.tensor(support, dtype=torch.long) + + +def estimate_occupancy( + x: torch.Tensor, + dictionary: torch.Tensor, + *, + max_atoms: int = DEFAULT_K, + num_control_dictionaries: int = 32, + seed: int = 0, +) -> JSpaceOccupancy: + """Estimate how many dictionary atoms are meaningfully active in ``x``. + + Runs the projection-residual recurrence described in + :func:`_greedy_captured_variance_gains` for exactly ``max_atoms`` steps and compares the real + per-step captured-variance curve against the same recurrence on ``num_control_dictionaries`` + random unit-norm dictionaries of the same size. This shares sparse decomposition's per-step + correlation rule, but uses an unconstrained span-projection residual rather than a nonnegative + coefficient-fit residual, so their supports need not match. The occupancy is the step of + maximum separation between the real and (averaged) control *cumulative* captured variance -- + the point past which further atoms add no more than random directions would. Deterministic + given ``seed`` and needs no threshold. (Captured variance is a projection, hence scale-free, + so the random control atoms are simply unit-norm.) + + Args: + x: Target vector, shape ``[d_model]``. + dictionary: Atom matrix, shape ``[num_atoms, d_model]`` (rows are atoms). + max_atoms: Number of atoms to select in the real and control recurrences. + num_control_dictionaries: Number of random control dictionaries to average over. + seed: Seed for the random control dictionaries (reproducibility). + + Returns: + An :class:`JSpaceOccupancy`. + + Raises: + ValueError: On complex inputs, a non-2-D dictionary, a target whose length does not match + ``d_model``, ``max_atoms`` outside ``[1, num_atoms]``, + ``num_control_dictionaries < 1``, a target with non-finite entries or a non-finite or + zero norm, or a dictionary with non-finite or zero-norm atoms. + """ + if dictionary.ndim != 2: + raise ValueError( + f"dictionary must be 2-D [num_atoms, d_model], got shape {tuple(dictionary.shape)}" + ) + num_atoms, d_model = dictionary.shape + if x.ndim != 1 or x.shape[0] != d_model: + raise ValueError(f"x must be 1-D of length d_model={d_model}, got shape {tuple(x.shape)}") + if not 1 <= max_atoms <= num_atoms: + raise ValueError(f"max_atoms must be between 1 and num_atoms={num_atoms}, got {max_atoms}") + if num_control_dictionaries < 1: + raise ValueError( + f"num_control_dictionaries must be at least 1, got {num_control_dictionaries}" + ) + if torch.is_complex(x) or torch.is_complex(dictionary): + raise ValueError("x and dictionary must be real-valued") + + target = x.float() + atoms = dictionary.float() + if not bool(torch.isfinite(target).all()): + raise ValueError("x contains non-finite entries") + target_squared_norm = target @ target + if not bool(torch.isfinite(target_squared_norm)): + raise ValueError("x must have finite norm") + if float(target_squared_norm) <= 0.0: + raise ValueError("x must have non-zero norm") + if not bool(torch.isfinite(atoms).all()): + raise ValueError("dictionary contains non-finite entries") + atom_norms = (atoms * atoms).sum(dim=1).sqrt() + if not bool(torch.isfinite(atom_norms).all()) or bool((atom_norms == 0).any()): + raise ValueError("dictionary contains a non-finite or zero-norm atom") + + real_captured_variance, support = _greedy_captured_variance_gains( + atoms, atom_norms, target, max_atoms + ) + + generator = torch.Generator(device=atoms.device).manual_seed(seed) + control_atom_norms = torch.ones(num_atoms, device=atoms.device) + control_variance_runs: List[torch.Tensor] = [] + for _ in range(num_control_dictionaries): + random_atoms = torch.randn( + num_atoms, d_model, generator=generator, device=atoms.device, dtype=atoms.dtype + ) + random_atoms = random_atoms / (random_atoms * random_atoms).sum(dim=1, keepdim=True).sqrt() + control_run_variance, _ = _greedy_captured_variance_gains( + random_atoms, control_atom_norms, target, max_atoms + ) + control_variance_runs.append(control_run_variance) + control_captured_variance = torch.stack(control_variance_runs).mean(dim=0) + + separation = real_captured_variance.cumsum(dim=0) - control_captured_variance.cumsum(dim=0) + occupancy = int(torch.argmax(separation).item()) + 1 + return JSpaceOccupancy( + occupancy=occupancy, + marginal_captured_variance=real_captured_variance, + control_captured_variance=control_captured_variance, + support=support, + ) + + +@dataclass +class JSpaceVarianceProfile: + """Per-layer J-space variance profile over a prompt corpus. + + Produced by :meth:`JacobianLens.fraction_of_variance`. + + Attributes: + layers: The source layers profiled, in order. + median: Per-layer median over positions of the J-space variance fraction + ``||j_space_component||^2 / ||activation||^2``. + pooled: Per-layer pooled ratio ``sum(||j_space_component||^2) / sum(||activation||^2)`` + across the corpus (the paper's "fraction of total variance"). + per_position: Per-layer 1-D tensor of the raw per-position variance fractions. + """ + + layers: List[int] + median: Dict[int, float] + pooled: Dict[int, float] + per_position: Dict[int, torch.Tensor] diff --git a/transformer_lens/tools/model_registry/AGENTS.md b/transformer_lens/tools/model_registry/AGENTS.md index 8198be7013..e3e2c1e4c1 100644 --- a/transformer_lens/tools/model_registry/AGENTS.md +++ b/transformer_lens/tools/model_registry/AGENTS.md @@ -114,7 +114,7 @@ Never edit manually. | 1 | Core forward correctness vs HuggingFace logits | | 2 | Hook firing + gradient flow | | 3 | Weight processing (compatibility mode, fold/centre) | -| 4 | Text-generation quality | +| 4 | Text-generation quality (per-model prompt profile, scored by a pinned multilingual judge) | | 7 | Multimodal (vision/text alignment) — only Llava / Gemma3-multimodal | | 8 | Audio — Hubert (waveform) and AST (spectrogram) | | 9 | Vision — ViT/DeiT pixel forward, hook/cache firing, representation stability, classification decode | @@ -132,24 +132,26 @@ SSM / recurrent families and the hybrids (Mamba-1/2, gated-delta-net, NemotronH, | 1 | **100%** | — | `STATUS_FAILED` | | 2 | 75% | `logits_equivalence`, `loss_equivalence` | `STATUS_FAILED` | | 3 | 75% | `logits_equivalence`, `loss_equivalence` | `STATUS_FAILED` | -| 4 | 50% | — | **Non-gating.** Below 50% adds `"low text quality"` to the registry `note`; never causes `STATUS_FAILED`. | +| 4 | 54.5% — the measured pass line `p4_pass_threshold()` (score of the bake-off noise floor `JUDGE_R_GOOD`) | — | **Non-gating.** Below the line adds `"text quality poor (P4=…)"` to the registry `note`; never causes `STATUS_FAILED`. | | 7 | 75% | `multimodal_forward` | `STATUS_FAILED`. NULL score (processor unavailable) also fails. | | 8 | 75% | `audio_forward` | `STATUS_FAILED`. NULL score also fails. | | 9 | 75% | `vision_forward`, `vision_cache` | `STATUS_FAILED`. NULL score also fails. | -Phase 4 is intentionally lenient — source ([`verify_models.py:554`](verify_models.py)) calls it *"a quality metric, not a correctness check."* The 50% bar asks "is the text coherent at all?" not "is this adapter clean?" +P4 prompts each model with its resolved **prompt profile** — chat template, translation, code, own-language continuation, or another task kind — via `resolve_profile()` in [`benchmarks/text_quality_profiles.py`](../../benchmarks/text_quality_profiles.py) (precedence: per-model override > architecture rule > live HF Hub signals > stored registry value > default). The resolved profile is cached sparsely on the entry as `prompt_profile` (key omitted when it's just the default). Each generation is scored against a known-good reference by one pinned multilingual judge via the perplexity ratio `PPL(generated)/PPL(reference)`, which cancels the judge's per-language handicap; the pass/fail constants are measured, not hand-picked, in [`benchmarks/text_quality.py`](../../benchmarks/text_quality.py). + +Phase 4 is a quality metric, not a correctness check. Its floor is not hand-picked: it equals the benchmark pass line, derived from the judge bake-off's fluent-vs-fluent noise floor (`p4_pass_threshold()` in [`benchmarks/text_quality_profiles.py`](../../benchmarks/text_quality_profiles.py)), so the registry note and the benchmark verdict can never disagree. **For adapter authors:** a `STATUS_VERIFIED` entry with P4 well below 100% on a small parity-test model can still indicate a real bug the system doesn't gate on (e.g. missing `preprocess_weights` fold). Investigate manually even when VERIFIED. **Reading the result:** - `status==1` + `note="Full verification completed"` → all gates passed, no quality flag. Good. -- `status==1` + `note` mentions `"low text quality"` → P4 < 50%; investigate. +- `status==1` + `note` mentions `"text quality poor"` → P4 below the pass line; investigate (`scripts/phase4_review.py` orders the candidates and separates old-scale scores). - `status==1` + P4 < 100% on a small model, no quality flag → potential weight-fold/tokenizer bug; investigate. - `status==3` (FAILED) → `note` carries the failure reason; debug from there. - `status==4` (PROVISIONAL) → structural-only pass via `--no-hf-reference`; Phase 1 was never numerically compared to HF, so it does **not** count as verified (`note` is prefixed `Structural only (no HF reference)`). Re-run without the flag for a real verification. -P1/P3 failures: [supported_architectures/AGENTS.md §When to override preprocess_weights](../../model_bridge/supported_architectures/AGENTS.md#when-to-override-preprocess_weights), [debugging_numerical_divergence.md](../../../docs/source/content/debugging_numerical_divergence.md). P4 drift: [§Tokenizer policy](../../model_bridge/supported_architectures/AGENTS.md#tokenizer-policy) (logit-scale / embedding-scale folds typically degrade P4 without crossing the 50% gate). +P1/P3 failures: [supported_architectures/AGENTS.md §When to override preprocess_weights](../../model_bridge/supported_architectures/AGENTS.md#when-to-override-preprocess_weights), [debugging_numerical_divergence.md](../../../docs/source/content/debugging_numerical_divergence.md). P4 drift: [§Tokenizer policy](../../model_bridge/supported_architectures/AGENTS.md#tokenizer-policy) (logit-scale / embedding-scale folds typically degrade P4 without crossing the pass line). --- diff --git a/transformer_lens/tools/model_registry/data/supported_models.json b/transformer_lens/tools/model_registry/data/supported_models.json index 080c7d91d2..fabaf12e6d 100644 --- a/transformer_lens/tools/model_registry/data/supported_models.json +++ b/transformer_lens/tools/model_registry/data/supported_models.json @@ -9,7 +9,7 @@ "total_architectures": 143, "total_models": 15670, "total_provisional": 7, - "total_verified": 1203, + "total_verified": 1205, "models": [ { "architecture_id": "FalconH1ForCausalLM", @@ -101,7 +101,7 @@ "status": 3, "verified_date": "2026-06-27", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.013494, mean_rel=0.006767", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.013494, mean_rel=0.006767", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -115,7 +115,7 @@ "status": 3, "verified_date": "2026-06-27", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.020484, mean_rel=0.006617", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.020484, mean_rel=0.006617", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -295,15 +295,16 @@ "architecture_id": "LlamaForCausalLM", "model_id": "01-ai/Yi-1.5-6B", "status": 1, - "verified_date": "2026-06-26", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 65.6, + "phase4_score": 75.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "LlamaForCausalLM", @@ -435,15 +436,16 @@ "architecture_id": "LlamaForCausalLM", "model_id": "01-ai/Yi-6B", "status": 1, - "verified_date": "2026-02-25", + "verified_date": "2026-08-20", "metadata": null, - "note": "Core verification completed", + "note": "Full verification completed", "phase1_score": 100.0, - "phase2_score": null, - "phase3_score": null, - "phase4_score": 67.4, + "phase2_score": 100.0, + "phase3_score": 100.0, + "phase4_score": 70.2, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "LlamaForCausalLM", @@ -479,7 +481,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -934,7 +936,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -1238,7 +1240,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=59.411900, mean_rel=0.569168", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=59.411900, mean_rel=0.569168", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -1634,7 +1636,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -1828,7 +1830,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=59.402462, mean_rel=0.441563", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=59.402462, mean_rel=0.441563", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -2931,7 +2933,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -3754,7 +3756,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=16.391922, mean_rel=3.526243", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=16.391922, mean_rel=3.526243", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 95.0, @@ -3768,7 +3770,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=18.059958, mean_rel=2.710044", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=18.059958, mean_rel=2.710044", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 95.0, @@ -4219,7 +4221,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002074, mean_rel=0.000409", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002074, mean_rel=0.000409", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -5405,15 +5407,16 @@ "architecture_id": "MixtralForCausalLM", "model_id": "BEE-spoke-data/Mixtral-GQA-400m-v2", "status": 1, - "verified_date": "2026-03-23", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 91.0, + "phase4_score": 86.1, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "LlamaForCausalLM", @@ -5836,7 +5839,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -6031,7 +6034,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=33.204865, mean_rel=0.370595", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=33.204865, mean_rel=0.370595", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -6297,7 +6300,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene \u2014 Forward pass failed: index out of range in self", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene — Forward pass failed: index out of range in self", "phase1_score": 50.0, "phase2_score": 7.7, "phase3_score": 44.4, @@ -7656,7 +7659,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7670,7 +7673,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=6.747103, mean_rel=0.054769", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=6.747103, mean_rel=0.054769", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -7684,7 +7687,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7698,7 +7701,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7712,7 +7715,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.479654, mean_rel=0.052641", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.479654, mean_rel=0.052641", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -7726,7 +7729,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7740,7 +7743,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=6.650925, mean_rel=0.050073", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=6.650925, mean_rel=0.050073", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -7754,7 +7757,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 92.3, "phase3_score": 90.0, @@ -7768,7 +7771,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.641898, mean_rel=0.054789", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.641898, mean_rel=0.054789", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -7782,7 +7785,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.808517, mean_rel=0.051374", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.808517, mean_rel=0.051374", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -11822,7 +11825,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=32.255035, mean_rel=0.318908", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=32.255035, mean_rel=0.318908", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -12810,7 +12813,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 76.9, "phase3_score": 90.0, @@ -14363,7 +14366,7 @@ "status": 3, "verified_date": "2026-07-01", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 24/51 components failed (24 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 24/51 components failed (24 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -14377,7 +14380,7 @@ "status": 3, "verified_date": "2026-07-01", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 50/99 components failed (50 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 50/99 components failed (50 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -14391,7 +14394,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 32/68 components failed (32 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 32/68 components failed (32 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -16343,7 +16346,7 @@ "architecture_id": "GPTJForCausalLM", "model_id": "EleutherAI/gpt-j-6b", "status": 1, - "verified_date": "2026-03-10", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -16351,13 +16354,14 @@ "phase3_score": 100.0, "phase4_score": 84.3, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "GPTNeoForCausalLM", "model_id": "EleutherAI/gpt-neo-1.3B", "status": 1, - "verified_date": "2026-03-10", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -16365,13 +16369,14 @@ "phase3_score": 100.0, "phase4_score": 97.6, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "GPTNeoForCausalLM", "model_id": "EleutherAI/gpt-neo-125m", "status": 1, - "verified_date": "2026-03-10", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -16379,7 +16384,8 @@ "phase3_score": 100.0, "phase4_score": 94.7, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "GPTNeoForCausalLM", @@ -16609,15 +16615,16 @@ "architecture_id": "GPTNeoXForCausalLM", "model_id": "EleutherAI/pythia-1.4b", "status": 1, - "verified_date": "2026-02-23", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 93.2, + "phase4_score": 97.3, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "GPTNeoXForCausalLM", @@ -17227,7 +17234,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=3.625000, mean_rel=0.024780", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=3.625000, mean_rel=0.024780", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -18179,7 +18186,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Can't load the model for 'EleutherAI/pythia-410m-seed1'. If you were trying to load it from 'https://hug", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Can't load the model for 'EleutherAI/pythia-410m-seed1'. If you were trying to load it from 'https://hug", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -18597,15 +18604,17 @@ "architecture_id": "GPTNeoXForCausalLM", "model_id": "EleutherAI/pythia-70m", "status": 1, - "verified_date": "2026-07-23", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 70.5, + "phase4_score": 56.6, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "GPTNeoXForCausalLM", @@ -18655,7 +18664,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 77.0/100 (avg perplexity: 327.9) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 77.0/100 (avg perplexity: 327.9) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -18669,7 +18678,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 68.8/100 (avg perplexity: 743.0) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 68.8/100 (avg perplexity: 743.0) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -19530,7 +19539,7 @@ "status": 3, "verified_date": "2026-03-11", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Ex0bit/Elbaz-Olmo-3-7B-Instruct-abliterated does not appear to have files named ('model-00001-of-00006.s", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Ex0bit/Elbaz-Olmo-3-7B-Instruct-abliterated does not appear to have files named ('model-00001-of-00006.s", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -20280,7 +20289,7 @@ "status": 3, "verified_date": "2026-03-11", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -20330,7 +20339,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=31.355835, mean_rel=0.607207", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=31.355835, mean_rel=0.607207", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -22630,7 +22639,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.011660, mean_rel=0.002703", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.011660, mean_rel=0.002703", "phase1_score": 50.0, "phase2_score": 91.7, "phase3_score": 100.0, @@ -22811,7 +22820,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 144/196 components failed (144 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 144/196 components failed (144 critical)", "phase1_score": 0.0, "phase2_score": 15.4, "phase3_score": 21.1, @@ -25324,7 +25333,7 @@ "architecture_id": "MixtralForCausalLM", "model_id": "Isotonic/TinyMixtral-4x248M-MoE", "status": 1, - "verified_date": "2026-03-23", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -25332,7 +25341,8 @@ "phase3_score": 100.0, "phase4_score": 92.5, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Qwen2ForCausalLM", @@ -26790,7 +26800,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 73.9/100 (avg perplexity: 10.2) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 73.9/100 (avg perplexity: 10.2) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -29208,7 +29218,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -31868,7 +31878,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: MachadoDeCastro/krull-micro does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: MachadoDeCastro/krull-micro does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -33388,7 +33398,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 12/87 components failed (12 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 12/87 components failed (12 critical)", "phase1_score": 0.0, "phase2_score": 15.4, "phase3_score": 50.0, @@ -33774,7 +33784,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -35084,7 +35094,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=39.516827, mean_rel=0.391392", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=39.516827, mean_rel=0.391392", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -36224,7 +36234,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=63.272919, mean_rel=0.497796", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=63.272919, mean_rel=0.497796", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -37144,7 +37154,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=34.928375, mean_rel=0.262157", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=34.928375, mean_rel=0.262157", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -37980,7 +37990,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=29.677444, mean_rel=0.238732", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=29.677444, mean_rel=0.238732", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -38495,7 +38505,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 45.4/100 (avg perplexity: 196.5) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 45.4/100 (avg perplexity: 196.5) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -39917,7 +39927,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=50.417328, mean_rel=0.326480", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=50.417328, mean_rel=0.326480", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -40398,7 +40408,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=34.771893, mean_rel=0.288610", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=34.771893, mean_rel=0.288610", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -41127,7 +41137,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=61.738430, mean_rel=0.447178", + "note": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=61.738430, mean_rel=0.447178", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -41721,7 +41731,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: extra_special_tokens must be a list/tuple of str or AddedToken, or a dict mapping names to tokens", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: extra_special_tokens must be a list/tuple of str or AddedToken, or a dict mapping names to tokens", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -43604,7 +43614,7 @@ "architecture_id": "Qwen2ForCausalLM", "model_id": "Qwen/Qwen2.5-0.5B", "status": 1, - "verified_date": "2026-07-23", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -43612,21 +43622,25 @@ "phase3_score": 100.0, "phase4_score": 96.3, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Qwen2ForCausalLM", "model_id": "Qwen/Qwen2.5-0.5B-Instruct", "status": 1, - "verified_date": "2026-03-10", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "chat", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 96.6, + "phase4_score": 97.9, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Qwen2ForCausalLM", @@ -43672,15 +43686,16 @@ "architecture_id": "Qwen2ForCausalLM", "model_id": "Qwen/Qwen2.5-1.5B", "status": 1, - "verified_date": "2026-02-24", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 97.2, + "phase4_score": 96.5, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Qwen2ForCausalLM", @@ -44596,7 +44611,7 @@ "architecture_id": "Qwen3ForCausalLM", "model_id": "Qwen/Qwen3-0.6B", "status": 1, - "verified_date": "2026-04-15", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -44604,7 +44619,8 @@ "phase3_score": 100.0, "phase4_score": 91.9, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Qwen3ForCausalLM", @@ -44637,15 +44653,16 @@ "architecture_id": "Qwen3ForCausalLM", "model_id": "Qwen/Qwen3-1.7B", "status": 1, - "verified_date": "2026-02-22", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 95.3, + "phase4_score": 97.7, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Qwen3ForCausalLM", @@ -47436,7 +47453,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 1152 (input tensor's size at dimension 0), but go", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 1152 (input tensor's size at dimension 0), but go", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -54292,7 +54309,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=nan, mean_rel=nan", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=nan, mean_rel=nan", "phase1_score": 50.0, "phase2_score": 75.0, "phase3_score": 94.1, @@ -55725,7 +55742,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: argument 'vocab': 'dict' object cannot be converted to 'Sequence'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: argument 'vocab': 'dict' object cannot be converted to 'Sequence'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -58388,7 +58405,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 130/132 components failed (125 high, 5 medium)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 130/132 components failed (125 high, 5 medium)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -59943,7 +59960,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Xenova/distilgpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Xenova/distilgpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -59985,7 +60002,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Xenova/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Xenova/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -64432,7 +64449,7 @@ "status": 3, "verified_date": "2026-04-02", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.004379, mean_rel=0.022909", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.004379, mean_rel=0.022909", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -65087,7 +65104,7 @@ "architecture_id": "Olmo2ForCausalLM", "model_id": "allenai/OLMo-2-0425-1B", "status": 1, - "verified_date": "2026-04-15", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -65095,7 +65112,8 @@ "phase3_score": 100.0, "phase4_score": 94.8, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Olmo2ForCausalLM", @@ -65616,7 +65634,7 @@ "architecture_id": "OlmoeForCausalLM", "model_id": "allenai/OLMoE-1B-7B-0924", "status": 1, - "verified_date": "2026-03-10", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -65624,7 +65642,8 @@ "phase3_score": 100.0, "phase4_score": 97.1, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "OlmoeForCausalLM", @@ -65658,15 +65677,16 @@ "architecture_id": "Olmo3ForCausalLM", "model_id": "allenai/Olmo-3-1025-7B", "status": 1, - "verified_date": "2026-02-26", + "verified_date": "2026-08-20", "metadata": null, - "note": "Core verification completed", + "note": "Full verification completed", "phase1_score": 100.0, - "phase2_score": null, - "phase3_score": null, - "phase4_score": 95.8, + "phase2_score": 100.0, + "phase3_score": 100.0, + "phase4_score": 98.8, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Olmo3ForCausalLM", @@ -67937,7 +67957,7 @@ "architecture_id": "OpenELMForCausalLM", "model_id": "apple/OpenELM-1_1B", "status": 1, - "verified_date": "2026-08-14", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -67968,7 +67988,7 @@ "architecture_id": "OpenELMForCausalLM", "model_id": "apple/OpenELM-270M", "status": 1, - "verified_date": "2026-08-14", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -68267,7 +68287,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 12/87 components failed (12 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 12/87 components failed (12 critical)", "phase1_score": 0.0, "phase2_score": 15.4, "phase3_score": 50.0, @@ -71390,7 +71410,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.005348, mean_rel=0.000007", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.005348, mean_rel=0.000007", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 95.0, @@ -72866,7 +72886,7 @@ "downloads": 89081, "total_params": null }, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -72991,7 +73011,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74348,7 +74368,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74360,15 +74380,16 @@ "architecture_id": "BloomForCausalLM", "model_id": "bigscience/bloom-1b7", "status": 1, - "verified_date": "2026-02-24", + "verified_date": "2026-08-20", "metadata": null, - "note": "Full verification completed with issues: P3=95.2% (failed: hook_functionality)", + "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, - "phase3_score": 95.2, - "phase4_score": 90.4, + "phase3_score": 100.0, + "phase4_score": 97.4, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "BloomForCausalLM", @@ -74404,7 +74425,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74416,15 +74437,16 @@ "architecture_id": "BloomForCausalLM", "model_id": "bigscience/bloom-560m", "status": 1, - "verified_date": "2026-04-07", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 75.9, + "phase4_score": 89.2, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "BloomForCausalLM", @@ -74432,7 +74454,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74460,7 +74482,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74474,7 +74496,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P2=69.2% < 75.0% (f \u2014 59/64 components failed (59 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P2=69.2% < 75.0% (f — 59/64 components failed (59 critical)", "phase1_score": 0.0, "phase2_score": 69.2, "phase3_score": 75.0, @@ -74656,7 +74678,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74670,7 +74692,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -74682,15 +74704,18 @@ "architecture_id": "MT5ForConditionalGeneration", "model_id": "bigscience/mt0-base", "status": 1, - "verified_date": "2026-06-26", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification passed, but text quality poor (P4=26.8). Needs review", + "prompt_profile": "task:instruction", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 97.8, + "phase4_score": 26.8, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MT5ForConditionalGeneration", @@ -74796,7 +74821,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=65.931717, mean_rel=2.066483", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=65.931717, mean_rel=2.066483", "phase1_score": 50.0, "phase2_score": 92.3, "phase3_score": 95.0, @@ -74810,7 +74835,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=22.915417, mean_rel=11.391559", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=22.915417, mean_rel=11.391559", "phase1_score": 50.0, "phase2_score": 92.3, "phase3_score": 95.0, @@ -76262,7 +76287,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -79416,7 +79441,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.003122, mean_rel=0.000469", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.003122, mean_rel=0.000469", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -80340,7 +80365,7 @@ "status": 3, "verified_date": "2026-05-08", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 36/282 components failed (36 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 36/282 components failed (36 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -83983,7 +84008,7 @@ "architecture_id": "GPT2LMHeadModel", "model_id": "distilbert/distilgpt2", "status": 1, - "verified_date": "2026-04-07", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -83991,7 +84016,8 @@ "phase3_score": 100.0, "phase4_score": 81.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Qwen3ForCausalLM", @@ -85218,7 +85244,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -87386,7 +87412,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -87667,7 +87693,7 @@ "architecture_id": "OPTForCausalLM", "model_id": "facebook/opt-1.3b", "status": 1, - "verified_date": "2026-03-10", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -87675,13 +87701,14 @@ "phase3_score": 100.0, "phase4_score": 96.4, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "OPTForCausalLM", "model_id": "facebook/opt-125m", "status": 1, - "verified_date": "2026-03-10", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -87689,7 +87716,8 @@ "phase3_score": 100.0, "phase4_score": 90.9, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "OPTForCausalLM", @@ -90392,7 +90420,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 68.7/100 (avg perplexity: 776.1) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 68.7/100 (avg perplexity: 776.1) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -92426,16 +92454,17 @@ { "architecture_id": "BertForMaskedLM", "model_id": "google-bert/bert-base-cased", - "status": 3, - "verified_date": "2026-06-25", + "status": 1, + "verified_date": "2026-08-20", "metadata": null, - "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", + "note": "Full verification completed", "phase1_score": 100.0, - "phase2_score": 66.7, + "phase2_score": 100.0, "phase3_score": 100.0, "phase4_score": null, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "BertForMaskedLM", @@ -92443,7 +92472,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=9.275972, mean_rel=13.166794", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=9.275972, mean_rel=13.166794", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -92541,7 +92570,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", + "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", "phase1_score": 100.0, "phase2_score": 66.7, "phase3_score": 100.0, @@ -92555,7 +92584,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", + "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", "phase1_score": 100.0, "phase2_score": 66.7, "phase3_score": 100.0, @@ -92583,7 +92612,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", + "note": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", "phase1_score": 100.0, "phase2_score": 66.7, "phase3_score": 100.0, @@ -92608,16 +92637,19 @@ { "architecture_id": "T5ForConditionalGeneration", "model_id": "google-t5/t5-base", - "status": 0, - "verified_date": null, + "status": 1, + "verified_date": "2026-08-21", "metadata": null, - "note": null, - "phase1_score": null, - "phase2_score": null, + "note": "Core verification completed", + "prompt_profile": "task:translation@en-de", + "p4_scoring_version": 2, + "phase1_score": 100.0, + "phase2_score": 100.0, "phase3_score": null, - "phase4_score": null, + "phase4_score": 100.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "T5ForConditionalGeneration", @@ -92637,15 +92669,18 @@ "architecture_id": "T5ForConditionalGeneration", "model_id": "google-t5/t5-small", "status": 1, - "verified_date": "2026-07-23", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:translation@en-de", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 97.6, + "phase4_score": 90.4, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "T5ForConditionalGeneration", @@ -92972,7 +93007,7 @@ "architecture_id": "Gemma2ForCausalLM", "model_id": "google/gemma-2-2b", "status": 1, - "verified_date": "2026-07-23", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed with issues: P3=95.5% (failed: unembed_centering)", "phase1_score": 100.0, @@ -92980,21 +93015,25 @@ "phase3_score": 95.5, "phase4_score": 98.8, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Gemma2ForCausalLM", "model_id": "google/gemma-2-2b-it", "status": 1, - "verified_date": "2026-05-19", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed with issues: P3=95.5% (failed: unembed_centering)", + "note": "Core verification completed (prior issues retained: P3=95.5%)", + "prompt_profile": "chat", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 95.5, - "phase4_score": 100.0, + "phase4_score": 97.9, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Gemma2ForCausalLM", @@ -93042,7 +93081,7 @@ "architecture_id": "GemmaForCausalLM", "model_id": "google/gemma-2b", "status": 1, - "verified_date": "2026-05-19", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -93050,7 +93089,8 @@ "phase3_score": 100.0, "phase4_score": 91.7, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "GemmaForCausalLM", @@ -93152,7 +93192,7 @@ "architecture_id": "Gemma3ForCausalLM", "model_id": "google/gemma-3-1b-it", "status": 1, - "verified_date": "2026-03-10", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -93160,7 +93200,8 @@ "phase3_score": 100.0, "phase4_score": 99.3, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Gemma3ForCausalLM", @@ -93207,7 +93248,7 @@ "architecture_id": "Gemma3ForCausalLM", "model_id": "google/gemma-3-270m", "status": 1, - "verified_date": "2026-07-23", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -93215,7 +93256,8 @@ "phase3_score": 100.0, "phase4_score": 92.7, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Gemma3ForCausalLM", @@ -98700,7 +98742,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.003898, mean_rel=0.027017", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.003898, mean_rel=0.027017", "phase1_score": 50.0, "phase2_score": 92.3, "phase3_score": 95.0, @@ -100800,7 +100842,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 4/24 components failed (4 high)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 4/24 components failed (4 high)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 85.0, @@ -101745,7 +101787,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=4.489960, mean_rel=1.238444", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=4.489960, mean_rel=1.238444", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -101871,7 +101913,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=20.307718, mean_rel=6.347236", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=20.307718, mean_rel=6.347236", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -101883,15 +101925,16 @@ "architecture_id": "GraniteForCausalLM", "model_id": "ibm-granite/granite-3.1-2b-instruct", "status": 1, - "verified_date": "2026-03-17", + "verified_date": "2026-08-20", "metadata": null, - "note": "Core verification completed", + "note": "Full verification completed", "phase1_score": 100.0, - "phase2_score": null, - "phase3_score": null, - "phase4_score": 96.5, + "phase2_score": 100.0, + "phase3_score": 100.0, + "phase4_score": 100.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "GraniteMoeForCausalLM", @@ -101969,7 +102012,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=4.789991, mean_rel=1.159405", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=4.789991, mean_rel=1.159405", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -102179,7 +102222,7 @@ "status": 3, "verified_date": "2026-04-14", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'in_proj'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'in_proj'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -102193,7 +102236,7 @@ "status": 3, "verified_date": "2026-04-15", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102207,7 +102250,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102221,7 +102264,7 @@ "status": 3, "verified_date": "2026-04-15", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102235,7 +102278,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102249,7 +102292,7 @@ "status": 3, "verified_date": "2026-04-15", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102263,7 +102306,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102305,7 +102348,7 @@ "status": 3, "verified_date": "2026-04-15", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102319,7 +102362,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -102361,7 +102404,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -103668,7 +103711,7 @@ "downloads": 5738, "total_params": null }, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'type'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'type'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -103748,7 +103791,7 @@ "architecture_id": "InternLM2ForCausalLM", "model_id": "internlm/internlm2-chat-1_8b", "status": 1, - "verified_date": "2026-07-23", + "verified_date": "2026-08-20", "metadata": { "downloads": 5069, "total_params": 1889110016 @@ -103759,7 +103802,8 @@ "phase3_score": 100.0, "phase4_score": 84.2, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "InternLM2ForCausalLM", @@ -108810,7 +108854,7 @@ "status": 3, "verified_date": "2026-04-08", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load katuni4ka/tiny-random-deepseek-v3:\n(Request ID", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load katuni4ka/tiny-random-deepseek-v3:\n(Request ID", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -108824,7 +108868,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 256 (input tensor's size at dimension 0), but got", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 256 (input tensor's size at dimension 0), but got", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -117839,7 +117883,7 @@ "status": 1, "verified_date": "2026-02-25", "metadata": null, - "note": "Below threshold: P3=81.8% but required tests failed: logits_equivalence \u2014 Scalars differ: 0.000000 vs -0.015625", + "note": "Below threshold: P3=81.8% but required tests failed: logits_equivalence — Scalars differ: 0.000000 vs -0.015625", "phase1_score": 100.0, "phase2_score": 78.6, "phase3_score": 81.8, @@ -118813,7 +118857,7 @@ "architecture_id": "Phi3ForCausalLM", "model_id": "microsoft/Phi-3-mini-4k-instruct", "status": 1, - "verified_date": "2026-04-07", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -118821,7 +118865,8 @@ "phase3_score": 100.0, "phase4_score": 98.5, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Phi3ForCausalLM", @@ -118855,7 +118900,7 @@ "architecture_id": "Phi3ForCausalLM", "model_id": "microsoft/Phi-3.5-mini-instruct", "status": 1, - "verified_date": "2026-03-30", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -118863,7 +118908,8 @@ "phase3_score": 100.0, "phase4_score": 97.4, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Phi3ForCausalLM", @@ -118995,15 +119041,16 @@ "architecture_id": "PhiForCausalLM", "model_id": "microsoft/phi-1", "status": 1, - "verified_date": "2026-02-22", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 92.3, + "phase4_score": 90.6, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "PhiForCausalLM", @@ -119023,15 +119070,16 @@ "architecture_id": "PhiForCausalLM", "model_id": "microsoft/phi-2", "status": 1, - "verified_date": "2026-03-27", + "verified_date": "2026-08-20", "metadata": null, - "note": "Full verification completed with issues: P2=92.9% (failed: backward_hooks)", + "note": "Full verification completed with issues: P2=93.3% (failed: backward_hooks)", "phase1_score": 100.0, - "phase2_score": 92.9, + "phase2_score": 93.3, "phase3_score": 100.0, "phase4_score": 95.8, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "PhiForCausalLM", @@ -120219,15 +120267,16 @@ "architecture_id": "MistralForCausalLM", "model_id": "mistralai/Mistral-7B-v0.1", "status": 1, - "verified_date": "2026-02-26", + "verified_date": "2026-08-20", "metadata": null, - "note": "Core verification completed", + "note": "Full verification completed", "phase1_score": 100.0, - "phase2_score": null, - "phase3_score": null, - "phase4_score": 96.6, + "phase2_score": 100.0, + "phase3_score": 100.0, + "phase4_score": 95.4, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MistralForCausalLM", @@ -124446,7 +124495,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -125422,7 +125471,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=75.0% but required tests failed \u2014 Tensors differ: max_diff=378.613281, mean_rel=0.057195", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=75.0% but required tests failed — Tensors differ: max_diff=378.613281, mean_rel=0.057195", "phase1_score": 50.0, "phase2_score": 76.9, "phase3_score": 75.0, @@ -126574,7 +126623,7 @@ "status": 3, "verified_date": "2026-03-11", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/gemma-3-270m-it-ONNX does not appear to have a file named pytorch_model.bin or model.safe", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/gemma-3-270m-it-ONNX does not appear to have a file named pytorch_model.bin or model.safe", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -126602,7 +126651,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-1b-ONNX-web does not appear to have a file named pytorch_model.bin or model.s", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-1b-ONNX-web does not appear to have a file named pytorch_model.bin or model.s", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -126616,7 +126665,7 @@ "status": 3, "verified_date": "2026-04-14", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-350m-ONNX-web does not appear to have a file named pytorch_model.bin or model", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-350m-ONNX-web does not appear to have a file named pytorch_model.bin or model", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -126644,7 +126693,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/tiny-random-olmo-hf does not appear to have a file named pytorch_model.bin or model.safet", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/tiny-random-olmo-hf does not appear to have a file named pytorch_model.bin or model.safet", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -127204,15 +127253,17 @@ "architecture_id": "GPT2LMHeadModel", "model_id": "openai-community/gpt2", "status": 1, - "verified_date": "2026-07-23", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 88.5, + "phase4_score": 68.2, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "GPT2LMHeadModel", @@ -127971,7 +128022,7 @@ "status": 3, "verified_date": "2026-04-08", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load optimum-intel-internal-testing/tiny-random-dee", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load optimum-intel-internal-testing/tiny-random-dee", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -128089,7 +128140,7 @@ "status": 3, "verified_date": "2026-02-23", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: optimum/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: optimum/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -129621,7 +129672,7 @@ "status": 3, "verified_date": "2026-05-08", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/190 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/190 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -134412,7 +134463,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P4=4.1% < 50.0% (failed: text_quality) \u2014 Text quality score: 4.1/100 (avg perplexity: 3.4) \u2014 generated text may be incoherent", + "note": "Below threshold: P4=4.1% < 50.0% (failed: text_quality) — Text quality score: 4.1/100 (avg perplexity: 3.4) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, @@ -136082,7 +136133,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "note": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 69.2, "phase3_score": 95.0, @@ -137686,7 +137737,7 @@ "status": 3, "verified_date": "2026-03-11", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 104/242 components failed (104 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 104/242 components failed (104 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -138281,7 +138332,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence; P4=6.8% < 50.0% (failed: te \u2014 Text quality score: 6.8/100 (avg perplexity: 372419.9) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence; P4=6.8% < 50.0% (failed: te — Text quality score: 6.8/100 (avg perplexity: 372419.9) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -138295,7 +138346,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'FalconDecoderLayer' object has no attribute 'ln_attn'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'FalconDecoderLayer' object has no attribute 'ln_attn'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -138615,15 +138666,16 @@ "architecture_id": "StableLmForCausalLM", "model_id": "stabilityai/stablelm-2-1_6b", "status": 1, - "verified_date": "2026-02-22", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 98.0, + "phase4_score": 95.3, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "StableLmForCausalLM", @@ -139088,14 +139140,15 @@ "architecture_id": "MambaForCausalLM", "model_id": "state-spaces/mamba-130m-hf", "status": 1, - "verified_date": "2026-07-23", + "verified_date": "2026-08-20", "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, "phase4_score": 95.8, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MambaForCausalLM", @@ -141778,7 +141831,7 @@ "status": 3, "verified_date": "2026-04-09", "metadata": null, - "note": "Below threshold: P3=50.0% < 75.0% (failed: process_bridge_weights, layer_norm_folding, weight_modifi \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "note": "Below threshold: P3=50.0% < 75.0% (failed: process_bridge_weights, layer_norm_folding, weight_modifi — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "phase1_score": 100.0, "phase2_score": 83.3, "phase3_score": 50.0, @@ -143238,7 +143291,7 @@ "status": 3, "verified_date": "2026-04-08", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/12 components failed (2 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/12 components failed (2 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 94.7, @@ -143252,7 +143305,7 @@ "status": 3, "verified_date": "2026-04-08", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 94.7, @@ -143280,7 +143333,7 @@ "status": 3, "verified_date": "2026-02-22", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/12 components failed (1 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/12 components failed (1 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 83.3, @@ -143521,7 +143574,7 @@ "downloads": 207171, "total_params": 2574656 }, - "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 72.2/100 (avg perplexity: 558.8) \u2014 generated text may be incoherent", + "note": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 72.2/100 (avg perplexity: 558.8) — generated text may be incoherent", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 89.5, @@ -148033,7 +148086,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -148047,7 +148100,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -148061,7 +148114,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g \u2014 144/307 components failed (144 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g — 144/307 components failed (144 critical)", "phase1_score": 0.0, "phase2_score": 7.7, "phase3_score": 22.2, @@ -149856,7 +149909,7 @@ "status": 3, "verified_date": "2026-02-24", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.004045, mean_rel=0.000066", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.004045, mean_rel=0.000066", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 100.0, @@ -154634,7 +154687,7 @@ "status": 3, "verified_date": "2026-04-09", "metadata": null, - "note": "Below threshold: P3=55.6% < 75.0% (failed: process_bridge_weights, weight_modification, hook_functio \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "note": "Below threshold: P3=55.6% < 75.0% (failed: process_bridge_weights, weight_modification, hook_functio — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "phase1_score": 100.0, "phase2_score": 83.3, "phase3_score": 55.6, @@ -156401,7 +156454,7 @@ "architecture_id": "SmolLM3ForCausalLM", "model_id": "HuggingFaceTB/SmolLM3-3B", "status": 1, - "verified_date": "2026-06-04", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -156409,7 +156462,8 @@ "phase3_score": 100.0, "phase4_score": 99.4, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "SmolLM3ForCausalLM", @@ -160071,7 +160125,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=0.009886, mean_rel=0.980186", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=0.009886, mean_rel=0.980186", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -162241,7 +162295,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "note": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "phase1_score": 100.0, "phase2_score": 69.2, "phase3_score": 95.0, @@ -168317,7 +168371,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: [Errno 2] No such file or directory: 'baichuan-inc/Baichuan2-7B-Chat-4bits/pytorch_model.bin'", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: [Errno 2] No such file or directory: 'baichuan-inc/Baichuan2-7B-Chat-4bits/pytorch_model.bin'", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -168359,7 +168413,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 70/609 components failed (70 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 70/609 components failed (70 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -168371,15 +168425,16 @@ "architecture_id": "Gemma4ForConditionalGeneration", "model_id": "google/gemma-4-E2B", "status": 1, - "verified_date": "2026-06-30", + "verified_date": "2026-08-20", "metadata": null, - "note": "Core verification completed", + "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, "phase4_score": 88.4, "phase7_score": 100.0, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Gemma4ForConditionalGeneration", @@ -172741,7 +172796,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 70/609 components failed (70 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 70/609 components failed (70 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -172755,7 +172810,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 60/534 components failed (60 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 60/534 components failed (60 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -172781,29 +172836,31 @@ "architecture_id": "ApertusForCausalLM", "model_id": "swiss-ai/Apertus-v1.1-0.5B", "status": 1, - "verified_date": "2026-06-25", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 98.9, + "phase4_score": 94.5, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "ApertusForCausalLM", "model_id": "swiss-ai/Apertus-v1.1-1.5B", "status": 1, - "verified_date": "2026-06-26", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 100.0, - "phase4_score": 94.1, + "phase4_score": 92.3, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "ApertusForCausalLM", @@ -173987,7 +174044,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -175065,7 +175122,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -175891,7 +175948,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -176563,7 +176620,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -177277,7 +177334,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -177669,7 +177726,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=38.5% < 75.0% (failed: \u2014 1/15 components failed (1 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=38.5% < 75.0% (failed: — 1/15 components failed (1 critical)", "phase1_score": 0.0, "phase2_score": 38.5, "phase3_score": 38.9, @@ -178481,7 +178538,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: LiquidAI/LFM2-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: LiquidAI/LFM2-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -178509,7 +178566,7 @@ "status": 3, "verified_date": "2026-06-26", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: LiquidAI/LFM2.5-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: LiquidAI/LFM2.5-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -178703,15 +178760,18 @@ "architecture_id": "BartForConditionalGeneration", "model_id": "facebook/bart-large-cnn", "status": 1, - "verified_date": "2026-07-08", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:summarization", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 78.7, + "phase4_score": 99.2, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "BartForConditionalGeneration", @@ -178943,7 +179003,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -178957,7 +179017,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179041,7 +179101,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 9/167 components failed (9 critical)", + "note": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 9/167 components failed (9 critical)", "phase1_score": 0.0, "phase2_score": 100.0, "phase3_score": null, @@ -179055,7 +179115,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179069,7 +179129,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179083,7 +179143,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179097,7 +179157,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179111,7 +179171,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179125,7 +179185,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179139,7 +179199,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179153,7 +179213,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179167,7 +179227,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179181,7 +179241,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179195,7 +179255,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179209,7 +179269,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179223,7 +179283,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179237,7 +179297,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179251,7 +179311,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179265,7 +179325,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -179279,7 +179339,7 @@ "status": 3, "verified_date": "2026-06-25", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "note": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -181281,7 +181341,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=11.357496, mean_rel=4.069944", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=11.357496, mean_rel=4.069944", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 90.0, @@ -182665,15 +182725,18 @@ "architecture_id": "MarianMTModel", "model_id": "Helsinki-NLP/opus-mt-nl-en", "status": 1, - "verified_date": "2026-07-24", + "verified_date": "2026-08-21", "metadata": null, "note": "Core verification completed", + "prompt_profile": "task:translation@nl-en", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 80.7, + "phase4_score": 94.2, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MarianMTModel", @@ -183071,15 +183134,18 @@ "architecture_id": "M2M100ForConditionalGeneration", "model_id": "facebook/m2m100_418M", "status": 1, - "verified_date": "2026-07-24", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed with issues, low text quality", + "note": "Core verification completed", + "prompt_profile": "task:translation@en-de", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 38.6, + "phase4_score": 100.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "M2M100ForConditionalGeneration", @@ -183239,43 +183305,52 @@ "architecture_id": "MBartForConditionalGeneration", "model_id": "facebook/mbart-large-50-many-to-many-mmt", "status": 1, - "verified_date": "2026-07-24", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:translation@en-de", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 89.4, + "phase4_score": 100.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MBartForConditionalGeneration", "model_id": "facebook/mbart-large-50", "status": 1, - "verified_date": "2026-07-07", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:denoise", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 100.0, + "phase4_score": 87.1, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MBartForConditionalGeneration", "model_id": "facebook/mbart-large-cc25", "status": 1, - "verified_date": "2026-07-07", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification passed, but text quality poor (P4=25.0). Needs review", + "prompt_profile": "task:denoise", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 92.4, + "phase4_score": 25.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "MBartForConditionalGeneration", @@ -183283,11 +183358,11 @@ "status": 1, "verified_date": "2026-07-07", "metadata": null, - "note": "Full verification completed with issues, low text quality", + "note": "P4 nulled: prior score measured under a broken MBart profile (mis-profiled + unconstrained target language); awaiting Indic denoise prompt coverage", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 32.5, + "phase4_score": null, "phase7_score": null, "phase8_score": null }, @@ -183309,15 +183384,18 @@ "architecture_id": "PegasusForConditionalGeneration", "model_id": "google/pegasus-xsum", "status": 1, - "verified_date": "2026-07-24", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed", + "note": "Core verification completed", + "prompt_profile": "task:summarization", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 100.0, + "phase4_score": 90.0, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "PegasusForConditionalGeneration", @@ -183465,7 +183543,7 @@ "status": 3, "verified_date": "2026-07-07", "metadata": null, - "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=27.217707, mean_rel=1.057937", + "note": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=27.217707, mean_rel=1.057937", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": 94.7, @@ -183645,7 +183723,7 @@ "architecture_id": "Starcoder2ForCausalLM", "model_id": "bigcode/starcoder2-3b", "status": 1, - "verified_date": "2026-07-23", + "verified_date": "2026-08-20", "metadata": null, "note": "Full verification completed", "phase1_score": 100.0, @@ -183653,7 +183731,8 @@ "phase3_score": 100.0, "phase4_score": 95.3, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "Starcoder2ForCausalLM", @@ -183841,15 +183920,18 @@ "architecture_id": "LongT5ForConditionalGeneration", "model_id": "google/long-t5-tglobal-base", "status": 1, - "verified_date": "2026-07-24", + "verified_date": "2026-08-21", "metadata": null, - "note": "Full verification completed with issues, low text quality", + "note": "Core verification passed, but text quality poor (P4=48.4). Needs review", + "prompt_profile": "task:denoise", + "p4_scoring_version": 2, "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, - "phase4_score": 49.2, + "phase4_score": 48.4, "phase7_score": null, - "phase8_score": null + "phase8_score": null, + "phase9_score": null }, { "architecture_id": "LongT5ForConditionalGeneration", @@ -183857,7 +183939,7 @@ "status": 3, "verified_date": "2026-07-07", "metadata": null, - "note": "Checkpoint ships no lm_head.weight with tie_word_embeddings=false, so HF randomly re-initializes the LM head on every load (Google released it as a pretraining artifact requiring fine-tuning); the nondeterministic unembed is the only P1 failure \u2014 all 233 other components including the local-attention encoder pass. Not an adapter bug.", + "note": "Checkpoint ships no lm_head.weight with tie_word_embeddings=false, so HF randomly re-initializes the LM head on every load (Google released it as a pretraining artifact requiring fine-tuning); the nondeterministic unembed is the only P1 failure — all 233 other components including the local-attention encoder pass. Not an adapter bug.", "phase1_score": 50.0, "phase2_score": 100.0, "phase3_score": null, @@ -184025,7 +184107,7 @@ "status": 1, "verified_date": "2026-07-07", "metadata": null, - "note": "Verified on a local snapshot of yujiepan/llama-4-tiny-random with text_config.attn_temperature_tuning coerced to bool \u2014 the upstream config declares it as int 4, which transformers 5.x strict config validation rejects. P7 skipped: the tiny ships no processor files. P3=95 (attention_output_centering worst_mean=0.099 on random weights). Official Scout/Maverick checkpoints registered for big-hardware verification.", + "note": "Verified on a local snapshot of yujiepan/llama-4-tiny-random with text_config.attn_temperature_tuning coerced to bool — the upstream config declares it as int 4, which transformers 5.x strict config validation rejects. P7 skipped: the tiny ships no processor files. P3=95 (attention_output_centering worst_mean=0.099 on random weights). Official Scout/Maverick checkpoints registered for big-hardware verification.", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": 95.0, @@ -186545,7 +186627,7 @@ "status": 3, "verified_date": "2026-07-07", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -187343,7 +187425,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -187441,7 +187523,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -187497,7 +187579,7 @@ "status": 3, "verified_date": "2026-07-21", "metadata": null, - "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "note": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "phase1_score": 0.0, "phase2_score": null, "phase3_score": null, @@ -216267,7 +216349,7 @@ "status": 1, "verified_date": "2026-07-23", "metadata": null, - "note": "Full verification completed (P1 in fp32, P2/P4 in bf16 for memory). bf16 P1 is precision-bound: max_diff=0.375, mean_rel=0.022 at bf16, but the same comparison in fp32 matches within tolerance \u2014 measured, not assumed. Required a reconstruction dtype fix: FlexOlmoRotaryEmbedding returns fp32 cos/sin without casting to the input dtype, promoting the attention output to fp32 against bf16 projection weights.", + "note": "Full verification completed (P1 in fp32, P2/P4 in bf16 for memory). bf16 P1 is precision-bound: max_diff=0.375, mean_rel=0.022 at bf16, but the same comparison in fp32 matches within tolerance — measured, not assumed. Required a reconstruction dtype fix: FlexOlmoRotaryEmbedding returns fp32 cos/sin without casting to the input dtype, promoting the attention output to fp32 against bf16 projection weights.", "phase1_score": 100.0, "phase2_score": 100.0, "phase3_score": null, @@ -216505,7 +216587,7 @@ "status": 1, "verified_date": "2026-07-23", "metadata": null, - "note": "Forward parity PROVEN: bridge byte-identical to raw HF in fp32 on identical ids (max \u0394logit 0, max \u0394log_softmax 0, 100% argmax). P4=55.9 is genuine model behavior (one hard NL prompt, diffusion sampling, weak GPT-2 judge), not a bridge defect. HF end-to-end capture requires a 4D (b,1,s,s) block attention mask.", + "note": "Forward parity PROVEN: bridge byte-identical to raw HF in fp32 on identical ids (max Δlogit 0, max Δlog_softmax 0, 100% argmax). P4=55.9 is genuine model behavior (one hard NL prompt, diffusion sampling, weak GPT-2 judge), not a bridge defect. HF end-to-end capture requires a 4D (b,1,s,s) block attention mask.", "phase1_score": 100.0, "phase2_score": null, "phase3_score": null, diff --git a/transformer_lens/tools/model_registry/data/verification_history.json b/transformer_lens/tools/model_registry/data/verification_history.json index 8bbd370ca0..24dd53444e 100644 --- a/transformer_lens/tools/model_registry/data/verification_history.json +++ b/transformer_lens/tools/model_registry/data/verification_history.json @@ -1,5 +1,5 @@ { - "last_updated": "2026-08-14T15:51:08.675104", + "last_updated": "2026-08-21T12:51:05.468344", "records": [ { "model_id": "Macropodus/macbert4mdcspell_v1", @@ -2287,7 +2287,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% \u2014 No tokenizer files on HuggingFace (ValueError: Couldn't instantiate the backend tokenizer)", + "notes": "Below threshold: P1=0.0% < 100.0% — No tokenizer files on HuggingFace (ValueError: Couldn't instantiate the backend tokenizer)", "invalidated": false, "invalidation_reason": null }, @@ -2297,7 +2297,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% \u2014 Requires bitsandbytes 8-bit quantization (ImportError: pip install -U bitsandbytes>=0.46.1)", + "notes": "Below threshold: P1=0.0% < 100.0% — Requires bitsandbytes 8-bit quantization (ImportError: pip install -U bitsandbytes>=0.46.1)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2397,7 +2397,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 47/292 components failed (14 critical, 33 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 47/292 components failed (14 critical, 33 high)", "invalidated": false, "invalidation_reason": null }, @@ -2407,7 +2407,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.125000, mean_rel=0.033691", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.125000, mean_rel=0.033691", "invalidated": false, "invalidation_reason": null }, @@ -2417,7 +2417,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.281250, mean_rel=0.051025", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.281250, mean_rel=0.051025", "invalidated": false, "invalidation_reason": null }, @@ -2447,7 +2447,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2457,7 +2457,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", "invalidated": false, "invalidation_reason": null }, @@ -2467,7 +2467,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2477,7 +2477,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2487,7 +2487,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 31/196 components failed (31 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 31/196 components failed (31 critical)", "invalidated": false, "invalidation_reason": null }, @@ -2497,7 +2497,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 48/76 components failed (48 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 48/76 components failed (48 critical)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2507,7 +2507,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.250000, mean_rel=0.045166", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.250000, mean_rel=0.045166", "invalidated": false, "invalidation_reason": null }, @@ -2677,7 +2677,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: '<' not supported between instances of 'NoneType' and 'int'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: '<' not supported between instances of 'NoneType' and 'int'", "invalidated": false, "invalidation_reason": null }, @@ -2697,7 +2697,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: '<' not supported between instances of 'NoneType' and 'int'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: '<' not supported between instances of 'NoneType' and 'int'", "invalidated": false, "invalidation_reason": null }, @@ -2777,7 +2777,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/12 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/12 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -2787,7 +2787,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, critical_forward_ho \u2014 Tensors differ: max_diff=0.083040, mean_rel=0.006218", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, critical_forward_ho — Tensors differ: max_diff=0.083040, mean_rel=0.006218", "invalidated": false, "invalidation_reason": null }, @@ -2837,7 +2837,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2847,7 +2847,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=0.097573, mean_rel=0.008319", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=0.097573, mean_rel=0.008319", "invalidated": false, "invalidation_reason": null }, @@ -2857,7 +2857,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=0.129729, mean_rel=0.023225", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=0.129729, mean_rel=0.023225", "invalidated": false, "invalidation_reason": null }, @@ -2867,7 +2867,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: logits_equivalence, hook_functionality, critical_forward_ \u2014 Tensors differ: max_diff=0.286175, mean_rel=0.028925", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: logits_equivalence, hook_functionality, critical_forward_ — Tensors differ: max_diff=0.286175, mean_rel=0.028925", "invalidated": false, "invalidation_reason": null }, @@ -2887,7 +2887,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/gemma-3-1b-it-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/gemma-3-1b-it-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2897,7 +2897,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -2937,7 +2937,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, critical_forward_ho \u2014 Tensors differ: max_diff=0.705528, mean_rel=0.011718", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, critical_forward_ho — Tensors differ: max_diff=0.705528, mean_rel=0.011718", "invalidated": false, "invalidation_reason": null }, @@ -2967,7 +2967,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3037,7 +3037,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3107,7 +3107,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=8.3% < 75.0% (failed: g \u2014 2/148 components failed (2 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=8.3% < 75.0% (failed: g — 2/148 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -3147,7 +3147,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: ", "invalidated": false, "invalidation_reason": null }, @@ -3227,7 +3227,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3327,7 +3327,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=1.562500, mean_rel=0.753906", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=1.562500, mean_rel=0.753906", "invalidated": false, "invalidation_reason": null }, @@ -3357,7 +3357,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.250000, mean_rel=0.014526", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.250000, mean_rel=0.014526", "invalidated": false, "invalidation_reason": null }, @@ -3477,7 +3477,7 @@ "verified_date": "2026-02-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3677,7 +3677,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3727,7 +3727,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -3907,7 +3907,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/147 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/147 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -3947,7 +3947,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: ", "invalidated": false, "invalidation_reason": null }, @@ -4077,7 +4077,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Xenova/distilgpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Xenova/distilgpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -4087,7 +4087,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Xenova/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Xenova/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -4117,7 +4117,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: optimum/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: optimum/gpt2 does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -4157,7 +4157,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4167,7 +4167,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: 'NoneType' object has no attribute 'from_pretrained'", "invalidated": false, "invalidation_reason": null }, @@ -4177,7 +4177,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4187,7 +4187,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4207,7 +4207,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 48/76 components failed (48 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 48/76 components failed (48 critical)", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4247,7 +4247,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002074, mean_rel=0.000409", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002074, mean_rel=0.000409", "invalidated": false, "invalidation_reason": null }, @@ -4537,7 +4537,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 8-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": false, "invalidation_reason": null }, @@ -4637,7 +4637,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4717,7 +4717,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4807,7 +4807,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4847,7 +4847,7 @@ "verified_date": "2026-02-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -4867,7 +4867,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading a GPTQ quantized model requires optimum (`pip install optimum`)", "invalidated": false, "invalidation_reason": null }, @@ -5027,7 +5027,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5047,7 +5047,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/Llama-3.2-1B-Instruct-GGUF does not appear to have a file named pytorch_model.bin or model.safet", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/Llama-3.2-1B-Instruct-GGUF does not appear to have a file named pytorch_model.bin or model.safet", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5077,7 +5077,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5087,7 +5087,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5117,7 +5117,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5127,7 +5127,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5177,7 +5177,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5197,7 +5197,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5257,7 +5257,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5327,7 +5327,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5357,7 +5357,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5377,7 +5377,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/DeepSeek-R1-Distill-Qwen-1.5B-GGUF does not appear to have a file named pytorch_model.bin or mod", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/DeepSeek-R1-Distill-Qwen-1.5B-GGUF does not appear to have a file named pytorch_model.bin or mod", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5407,7 +5407,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5447,7 +5447,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 130/132 components failed (125 high, 5 medium)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 130/132 components failed (125 high, 5 medium)", "invalidated": false, "invalidation_reason": null }, @@ -5457,7 +5457,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002012, mean_rel=0.000401", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002012, mean_rel=0.000401", "invalidated": false, "invalidation_reason": null }, @@ -5597,7 +5597,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: extra_special_tokens must be a list/tuple of str or AddedToken, or a dict mapping names to tokens", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: extra_special_tokens must be a list/tuple of str or AddedToken, or a dict mapping names to tokens", "invalidated": false, "invalidation_reason": null }, @@ -5607,7 +5607,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.004045, mean_rel=0.000066", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.004045, mean_rel=0.000066", "invalidated": false, "invalidation_reason": null }, @@ -5617,7 +5617,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.003122, mean_rel=0.000469", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.003122, mean_rel=0.000469", "invalidated": false, "invalidation_reason": null }, @@ -5687,7 +5687,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.011660, mean_rel=0.002703", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.011660, mean_rel=0.002703", "invalidated": false, "invalidation_reason": null }, @@ -5797,7 +5797,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5827,7 +5827,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5857,7 +5857,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5867,7 +5867,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5877,7 +5877,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5917,7 +5917,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/Qwen3-0.6B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/Qwen3-0.6B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5937,7 +5937,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5947,7 +5947,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5957,7 +5957,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/Qwen3-4B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/Qwen3-4B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5967,7 +5967,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5977,7 +5977,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5987,7 +5987,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -5997,7 +5997,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: unsloth/Qwen3-1.7B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: unsloth/Qwen3-1.7B-GGUF does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6007,7 +6007,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6017,7 +6017,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6027,7 +6027,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6037,7 +6037,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6047,7 +6047,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6057,7 +6057,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6067,7 +6067,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6077,7 +6077,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6087,7 +6087,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6097,7 +6097,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: No module named 'triton'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: No module named 'triton'", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6117,7 +6117,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6127,7 +6127,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6137,7 +6137,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6147,7 +6147,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6157,7 +6157,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6177,7 +6177,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6197,7 +6197,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6207,7 +6207,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6217,7 +6217,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br \u2014 Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed, load_bridge_unprocessed, load_br — Failed to load unprocessed TransformerBridge: compressed_tensors is not installed and is required for compressed-tensors quantization. Please install ", "invalidated": true, "invalidation_reason": "TransformerLens does not support quantized models at this time" }, @@ -6227,7 +6227,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 1/130 components failed (1 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 1/130 components failed (1 high)", "invalidated": false, "invalidation_reason": null }, @@ -6237,7 +6237,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 1/130 components failed (1 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 1/130 components failed (1 high)", "invalidated": false, "invalidation_reason": null }, @@ -6247,7 +6247,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 1/130 components failed (1 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 1/130 components failed (1 high)", "invalidated": false, "invalidation_reason": null }, @@ -6317,7 +6317,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook \u2014 Bridge is missing 56 hooks from reference model", + "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook — Bridge is missing 56 hooks from reference model", "invalidated": false, "invalidation_reason": null }, @@ -6347,7 +6347,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook \u2014 Bridge is missing 56 hooks from reference model", + "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook — Bridge is missing 56 hooks from reference model", "invalidated": false, "invalidation_reason": null }, @@ -6707,7 +6707,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=85.0% but required tests failed: logits_equivalence \u2014 Found 1 significant mismatches in critical hooks", + "notes": "Below threshold: P3=85.0% but required tests failed: logits_equivalence — Found 1 significant mismatches in critical hooks", "invalidated": false, "invalidation_reason": null }, @@ -6767,7 +6767,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=nan, mean_rel=nan", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=nan, mean_rel=nan", "invalidated": false, "invalidation_reason": null }, @@ -6807,7 +6807,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810717, mean_rel=73.159515", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810717, mean_rel=73.159515", "invalidated": false, "invalidation_reason": null }, @@ -7077,7 +7077,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810717, mean_rel=73.159515", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810717, mean_rel=73.159515", "invalidated": false, "invalidation_reason": null }, @@ -7217,7 +7217,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=81.8% but required tests failed: logits_equivalence \u2014 Scalars differ: 0.000000 vs -0.015625", + "notes": "Below threshold: P3=81.8% but required tests failed: logits_equivalence — Scalars differ: 0.000000 vs -0.015625", "invalidated": false, "invalidation_reason": null }, @@ -7237,7 +7237,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho \u2014 Scalars differ: 5.875000 vs 5.812500", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho — Scalars differ: 5.875000 vs 5.812500", "invalidated": false, "invalidation_reason": null }, @@ -7247,7 +7247,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho \u2014 Scalars differ: 5.875000 vs 5.812500", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho — Scalars differ: 5.875000 vs 5.812500", "invalidated": false, "invalidation_reason": null }, @@ -7267,7 +7267,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho \u2014 Scalars differ: 5.968750 vs 5.875000", + "notes": "Below threshold: P2=71.4% < 75.0% (failed: loss_equivalence, hook_functionality, critical_forward_ho — Scalars differ: 5.968750 vs 5.875000", "invalidated": false, "invalidation_reason": null }, @@ -7287,7 +7287,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P4=0.0% < 50.0% (failed: text_quality) \u2014 Text quality score: 77.3/100 (avg perplexity: 30.4) \u2014 generated text may be incoherent", + "notes": "Below threshold: P4=0.0% < 50.0% (failed: text_quality) — Text quality score: 77.3/100 (avg perplexity: 30.4) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7297,7 +7297,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P4=0.0% < 50.0% (failed: text_quality) \u2014 Text quality score: 77.3/100 (avg perplexity: 30.4) \u2014 generated text may be incoherent", + "notes": "Below threshold: P4=0.0% < 50.0% (failed: text_quality) — Text quality score: 77.3/100 (avg perplexity: 30.4) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7317,7 +7317,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P4=4.1% < 50.0% (failed: text_quality) \u2014 Text quality score: 4.1/100 (avg perplexity: 3.4) \u2014 generated text may be incoherent", + "notes": "Below threshold: P4=4.1% < 50.0% (failed: text_quality) — Text quality score: 4.1/100 (avg perplexity: 3.4) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7337,7 +7337,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 68.7/100 (avg perplexity: 776.1) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 68.7/100 (avg perplexity: 776.1) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7347,7 +7347,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 68.7/100 (avg perplexity: 776.1) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 68.7/100 (avg perplexity: 776.1) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7357,7 +7357,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence \u2014 Text quality score: 68.7/100 (avg perplexity: 776.1) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence — Text quality score: 68.7/100 (avg perplexity: 776.1) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7367,7 +7367,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence; P4=6.8% < 50.0% (failed: te \u2014 Text quality score: 6.8/100 (avg perplexity: 372419.9) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence; P4=6.8% < 50.0% (failed: te — Text quality score: 6.8/100 (avg perplexity: 372419.9) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7377,7 +7377,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 77.0/100 (avg perplexity: 327.9) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 77.0/100 (avg perplexity: 327.9) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7387,7 +7387,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 68.8/100 (avg perplexity: 743.0) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 68.8/100 (avg perplexity: 743.0) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7397,7 +7397,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=3.625000, mean_rel=0.024780", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=3.625000, mean_rel=0.024780", "invalidated": false, "invalidation_reason": null }, @@ -7407,7 +7407,7 @@ "verified_date": "2026-02-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Can't load the model for 'EleutherAI/pythia-410m-seed1'. If you were trying to load it from 'https://hug", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Can't load the model for 'EleutherAI/pythia-410m-seed1'. If you were trying to load it from 'https://hug", "invalidated": false, "invalidation_reason": null }, @@ -7437,7 +7437,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002956, mean_rel=0.000962", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002956, mean_rel=0.000962", "invalidated": false, "invalidation_reason": null }, @@ -7457,7 +7457,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002956, mean_rel=0.000962", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002956, mean_rel=0.000962", "invalidated": false, "invalidation_reason": null }, @@ -7577,7 +7577,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=13.293901, mean_rel=32.253456", + "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=13.293901, mean_rel=32.253456", "invalidated": false, "invalidation_reason": null }, @@ -7627,7 +7627,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence \u2014 Found 1 significant mismatches in critical hooks", + "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence — Found 1 significant mismatches in critical hooks", "invalidated": false, "invalidation_reason": null }, @@ -7637,7 +7637,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence \u2014 Text quality score: 77.5/100 (avg perplexity: 372.2) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence — Text quality score: 77.5/100 (avg perplexity: 372.2) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -7647,7 +7647,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log \u2014 2/10 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log — 2/10 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -7697,7 +7697,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=12.128962, mean_rel=0.271985", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=12.128962, mean_rel=0.271985", "invalidated": false, "invalidation_reason": null }, @@ -7737,7 +7737,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810719, mean_rel=60.400272", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810719, mean_rel=60.400272", "invalidated": false, "invalidation_reason": null }, @@ -7767,7 +7767,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=19.425457, mean_rel=11.940315", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=19.425457, mean_rel=11.940315", "invalidated": false, "invalidation_reason": null }, @@ -7787,7 +7787,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "invalidated": false, "invalidation_reason": null }, @@ -7797,7 +7797,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=25.509125, mean_rel=0.521523", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=25.509125, mean_rel=0.521523", "invalidated": false, "invalidation_reason": null }, @@ -7807,7 +7807,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=21.740696, mean_rel=13.788611", + "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=21.740696, mean_rel=13.788611", "invalidated": false, "invalidation_reason": null }, @@ -7847,7 +7847,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002039, mean_rel=0.000401", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002039, mean_rel=0.000401", "invalidated": false, "invalidation_reason": null }, @@ -7947,7 +7947,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=12.128962, mean_rel=0.271985", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=12.128962, mean_rel=0.271985", "invalidated": false, "invalidation_reason": null }, @@ -8007,7 +8007,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence \u2014 Text quality score: 77.5/100 (avg perplexity: 372.2) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence — Text quality score: 77.5/100 (avg perplexity: 372.2) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -8027,7 +8027,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002039, mean_rel=0.000401", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002039, mean_rel=0.000401", "invalidated": false, "invalidation_reason": null }, @@ -8047,7 +8047,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "invalidated": false, "invalidation_reason": null }, @@ -8067,7 +8067,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002956, mean_rel=0.000962", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002956, mean_rel=0.000962", "invalidated": false, "invalidation_reason": null }, @@ -8077,7 +8077,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.002956, mean_rel=0.000962", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.002956, mean_rel=0.000962", "invalidated": false, "invalidation_reason": null }, @@ -8287,7 +8287,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=13.293901, mean_rel=32.253456", + "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=13.293901, mean_rel=32.253456", "invalidated": false, "invalidation_reason": null }, @@ -8337,7 +8337,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence \u2014 Found 1 significant mismatches in critical hooks", + "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence — Found 1 significant mismatches in critical hooks", "invalidated": false, "invalidation_reason": null }, @@ -8357,7 +8357,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log \u2014 2/10 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log — 2/10 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -8407,7 +8407,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks \u2014 Backward hooks check failed: 'tuple' object has no attribute 'clone'", + "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks — Backward hooks check failed: 'tuple' object has no attribute 'clone'", "invalidated": false, "invalidation_reason": null }, @@ -8417,7 +8417,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks \u2014 Backward hooks check failed: 'tuple' object has no attribute 'clone'", + "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks — Backward hooks check failed: 'tuple' object has no attribute 'clone'", "invalidated": false, "invalidation_reason": null }, @@ -8447,7 +8447,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810719, mean_rel=60.400272", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810719, mean_rel=60.400272", "invalidated": false, "invalidation_reason": null }, @@ -8477,7 +8477,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=19.425457, mean_rel=11.940315", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=19.425457, mean_rel=11.940315", "invalidated": false, "invalidation_reason": null }, @@ -8507,7 +8507,7 @@ "verified_date": "2026-03-09", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks \u2014 Backward hooks check failed: 'tuple' object has no attribute 'clone'", + "notes": "Below threshold: P3=50.0% < 75.0% (failed: hook_functionality, critical_forward_hooks, forward_hooks — Backward hooks check failed: 'tuple' object has no attribute 'clone'", "invalidated": false, "invalidation_reason": null }, @@ -8687,7 +8687,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log \u2014 2/10 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=94.7% but required tests failed: log — 2/10 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -8707,7 +8707,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 1/149 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 1/149 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -8727,7 +8727,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=18.423609, mean_rel=0.259477", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=18.423609, mean_rel=0.259477", "invalidated": false, "invalidation_reason": null }, @@ -8927,7 +8927,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=13.293901, mean_rel=32.253456", + "notes": "Below threshold: P3=76.2% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=13.293901, mean_rel=32.253456", "invalidated": false, "invalidation_reason": null }, @@ -8977,7 +8977,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence \u2014 Found 1 significant mismatches in critical hooks", + "notes": "Below threshold: P3=90.5% but required tests failed: logits_equivalence — Found 1 significant mismatches in critical hooks", "invalidated": false, "invalidation_reason": null }, @@ -9087,7 +9087,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=28.810719, mean_rel=60.400272", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence — Tensors differ: max_diff=28.810719, mean_rel=60.400272", "invalidated": false, "invalidation_reason": null }, @@ -9117,7 +9117,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Tensors differ: max_diff=19.425457, mean_rel=11.940315", + "notes": "Below threshold: P2=64.3% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=19.425457, mean_rel=11.940315", "invalidated": false, "invalidation_reason": null }, @@ -9137,7 +9137,7 @@ "verified_date": "2026-03-10", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, \u2014 Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", + "notes": "Below threshold: P3=58.8% < 75.0% (failed: weight_modification, hook_functionality, run_with_cache, — Critical backward hooks check failed: Output 0 of BackwardHookFunctionBackward is a view and is being modified inplace. This view was created inside a", "invalidated": false, "invalidation_reason": null }, @@ -9627,7 +9627,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "invalidated": false, "invalidation_reason": null }, @@ -9637,7 +9637,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/gemma-3-270m-it-ONNX does not appear to have a file named pytorch_model.bin or model.safe", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/gemma-3-270m-it-ONNX does not appear to have a file named pytorch_model.bin or model.safe", "invalidated": false, "invalidation_reason": null }, @@ -9787,7 +9787,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 104/242 components failed (104 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 104/242 components failed (104 critical)", "invalidated": false, "invalidation_reason": null }, @@ -9997,7 +9997,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`", "invalidated": false, "invalidation_reason": null }, @@ -10137,7 +10137,7 @@ "verified_date": "2026-03-11", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Ex0bit/Elbaz-Olmo-3-7B-Instruct-abliterated does not appear to have files named ('model-00001-of-00006.s", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Ex0bit/Elbaz-Olmo-3-7B-Instruct-abliterated does not appear to have files named ('model-00001-of-00006.s", "invalidated": false, "invalidation_reason": null }, @@ -10367,7 +10367,7 @@ "verified_date": "2026-03-19", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 3/197 components failed (3 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 3/197 components failed (3 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10567,7 +10567,7 @@ "verified_date": "2026-03-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.1% < 75.0% (failed: generation, gene \u2014 Forward pass failed: 'PhiAttention' object has no attribute 'o_proj'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.1% < 75.0% (failed: generation, gene — Forward pass failed: 'PhiAttention' object has no attribute 'o_proj'", "invalidated": false, "invalidation_reason": null }, @@ -10577,7 +10577,7 @@ "verified_date": "2026-03-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.1% < 75.0% (failed: generation, gene \u2014 Forward pass failed: 'PhiAttention' object has no attribute 'o_proj'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.1% < 75.0% (failed: generation, gene — Forward pass failed: 'PhiAttention' object has no attribute 'o_proj'", "invalidated": false, "invalidation_reason": null }, @@ -10607,7 +10607,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10617,7 +10617,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10627,7 +10627,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10637,7 +10637,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10647,7 +10647,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gener \u2014 Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gener — Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", "invalidated": false, "invalidation_reason": null }, @@ -10697,7 +10697,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", "invalidated": false, "invalidation_reason": null }, @@ -10717,7 +10717,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 64/163 components failed (64 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 64/163 components failed (64 critical)", "invalidated": false, "invalidation_reason": null }, @@ -10737,7 +10737,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: shape '[1, 28, 24, 71]' is invalid for input of size 47796", "invalidated": false, "invalidation_reason": null }, @@ -10747,7 +10747,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene \u2014 Forward pass failed: shape '[1, 28, 24, 128]' is invalid for input of size 28672", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=8.3% < 75.0% (failed: generation, gene — Forward pass failed: shape '[1, 28, 24, 128]' is invalid for input of size 28672", "invalidated": false, "invalidation_reason": null }, @@ -10767,7 +10767,7 @@ "verified_date": "2026-03-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: unsupported operand type(s) for *: 'NoneType' and 'int'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: unsupported operand type(s) for *: 'NoneType' and 'int'", "invalidated": false, "invalidation_reason": null }, @@ -11067,7 +11067,7 @@ "verified_date": "2026-04-02", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.004379, mean_rel=0.022909", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.004379, mean_rel=0.022909", "invalidated": false, "invalidation_reason": null }, @@ -11147,7 +11147,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 6/32 components failed (6 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 6/32 components failed (6 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11157,7 +11157,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 6/32 components failed (6 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 6/32 components failed (6 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11167,7 +11167,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 5/32 components failed (4 critical, 1 medium)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 5/32 components failed (4 critical, 1 medium)", "invalidated": false, "invalidation_reason": null }, @@ -11177,7 +11177,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 4/32 components failed (4 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 4/32 components failed (4 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11217,7 +11217,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11227,7 +11227,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load trl-internal-testing/tiny-DeepseekV3ForCausalL", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load trl-internal-testing/tiny-DeepseekV3ForCausalL", "invalidated": false, "invalidation_reason": null }, @@ -11237,7 +11237,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load katuni4ka/tiny-random-deepseek-v3:\n(Request ID", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load katuni4ka/tiny-random-deepseek-v3:\n(Request ID", "invalidated": false, "invalidation_reason": null }, @@ -11247,7 +11247,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load optimum-intel-internal-testing/tiny-random-dee", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: There was a specific connection error when trying to load optimum-intel-internal-testing/tiny-random-dee", "invalidated": false, "invalidation_reason": null }, @@ -11257,7 +11257,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11267,7 +11267,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11277,7 +11277,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11287,7 +11287,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11297,7 +11297,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 18/32 components failed (18 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 18/32 components failed (18 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11317,7 +11317,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 10/28 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 10/28 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11337,7 +11337,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 4/22 components failed (4 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 4/22 components failed (4 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11347,7 +11347,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 4/22 components failed (4 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 4/22 components failed (4 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11357,7 +11357,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 10/24 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 10/24 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11367,7 +11367,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 10/24 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 10/24 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11377,7 +11377,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 10/24 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 10/24 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11397,7 +11397,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/16 components failed (1 medium)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/16 components failed (1 medium)", "invalidated": false, "invalidation_reason": null }, @@ -11437,7 +11437,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/16 components failed (1 medium)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/16 components failed (1 medium)", "invalidated": false, "invalidation_reason": null }, @@ -11457,7 +11457,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/12 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/12 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11467,7 +11467,7 @@ "verified_date": "2026-04-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/12 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/12 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11737,7 +11737,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", "invalidated": false, "invalidation_reason": null }, @@ -11747,7 +11747,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: This modeling file requires the following packages that were not found in your environment: bitsandbytes", "invalidated": false, "invalidation_reason": null }, @@ -11757,7 +11757,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=74.608353, mean_rel=1.619285", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=74.608353, mean_rel=1.619285", "invalidated": false, "invalidation_reason": null }, @@ -11767,7 +11767,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=78.619270, mean_rel=1.866265", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=78.619270, mean_rel=1.866265", "invalidated": false, "invalidation_reason": null }, @@ -11777,7 +11777,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=nan, mean_rel=nan", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=nan, mean_rel=nan", "invalidated": false, "invalidation_reason": null }, @@ -11787,7 +11787,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=33.073044, mean_rel=0.316714", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=33.073044, mean_rel=0.316714", "invalidated": false, "invalidation_reason": null }, @@ -11797,7 +11797,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=33.073044, mean_rel=0.316714", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=33.073044, mean_rel=0.316714", "invalidated": false, "invalidation_reason": null }, @@ -11807,7 +11807,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generation failed: 'NoneType' object is not subscriptable", + "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generation failed: 'NoneType' object is not subscriptable", "invalidated": false, "invalidation_reason": null }, @@ -11827,7 +11827,7 @@ "verified_date": "2026-04-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene \u2014 Forward pass failed: Cannot copy out of meta tensor; no data!", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene — Forward pass failed: Cannot copy out of meta tensor; no data!", "invalidated": false, "invalidation_reason": null }, @@ -11857,7 +11857,7 @@ "verified_date": "2026-05-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/190 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/190 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11867,7 +11867,7 @@ "verified_date": "2026-05-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/190 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/190 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -11877,7 +11877,7 @@ "verified_date": "2026-05-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/558 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/558 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12277,7 +12277,7 @@ "verified_date": "2026-06-04", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12307,7 +12307,7 @@ "verified_date": "2026-06-04", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12337,7 +12337,7 @@ "verified_date": "2026-06-05", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Weight magnitude issues: 1 too large", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Weight magnitude issues: 1 too large", "invalidated": false, "invalidation_reason": null }, @@ -12347,7 +12347,7 @@ "verified_date": "2026-06-05", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Weight magnitude issues: 1 too large", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Weight magnitude issues: 1 too large", "invalidated": false, "invalidation_reason": null }, @@ -12447,7 +12447,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/211 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/211 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12467,7 +12467,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 9/167 components failed (9 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 9/167 components failed (9 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12487,7 +12487,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12497,7 +12497,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12507,7 +12507,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12517,7 +12517,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12527,7 +12527,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12537,7 +12537,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12547,7 +12547,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12557,7 +12557,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12567,7 +12567,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12577,7 +12577,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12587,7 +12587,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12597,7 +12597,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: I/O error: IO Error: No space left on device (os error 28)", "invalidated": false, "invalidation_reason": null }, @@ -12657,7 +12657,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12667,7 +12667,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12677,7 +12677,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 8/167 components failed (8 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 8/167 components failed (8 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12687,7 +12687,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12697,7 +12697,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12707,7 +12707,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/247 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/247 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12717,7 +12717,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12727,7 +12727,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12737,7 +12737,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12747,7 +12747,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12757,7 +12757,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12767,7 +12767,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12777,7 +12777,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12787,7 +12787,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12797,7 +12797,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12807,7 +12807,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12817,7 +12817,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/487 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/487 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12827,7 +12827,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12837,7 +12837,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 26/527 components failed (26 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 26/527 components failed (26 critical)", "invalidated": false, "invalidation_reason": null }, @@ -12877,7 +12877,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: MachadoDeCastro/krull-micro does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: MachadoDeCastro/krull-micro does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -12907,7 +12907,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", + "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", "invalidated": false, "invalidation_reason": null }, @@ -12917,7 +12917,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=9.275972, mean_rel=13.166794", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=9.275972, mean_rel=13.166794", "invalidated": false, "invalidation_reason": null }, @@ -13057,7 +13057,7 @@ "verified_date": "2026-06-25", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13197,7 +13197,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: [Errno 2] No such file or directory: 'baichuan-inc/Baichuan2-7B-Chat-4bits/pytorch_model.bin'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: [Errno 2] No such file or directory: 'baichuan-inc/Baichuan2-7B-Chat-4bits/pytorch_model.bin'", "invalidated": false, "invalidation_reason": null }, @@ -13217,7 +13217,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", + "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", "invalidated": false, "invalidation_reason": null }, @@ -13227,7 +13227,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", + "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 40, 28996]) vs torch.Size([1, 32, 28996])", "invalidated": false, "invalidation_reason": null }, @@ -13247,7 +13247,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, \u2014 Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", + "notes": "Below threshold: P2=66.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Shape mismatch: torch.Size([1, 38, 30522]) vs torch.Size([1, 30, 30522])", "invalidated": false, "invalidation_reason": null }, @@ -13277,7 +13277,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13287,7 +13287,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13297,7 +13297,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13307,7 +13307,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13317,7 +13317,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P2=69.2% < 75.0% (f \u2014 59/64 components failed (59 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P2=69.2% < 75.0% (f — 59/64 components failed (59 critical)", "invalidated": false, "invalidation_reason": null }, @@ -13407,7 +13407,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=0.009886, mean_rel=0.980186", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=0.009886, mean_rel=0.980186", "invalidated": false, "invalidation_reason": null }, @@ -13417,7 +13417,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13427,7 +13427,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -13447,7 +13447,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=65.931717, mean_rel=2.066483", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=65.931717, mean_rel=2.066483", "invalidated": false, "invalidation_reason": null }, @@ -13457,7 +13457,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=22.915417, mean_rel=11.391559", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=22.915417, mean_rel=11.391559", "invalidated": false, "invalidation_reason": null }, @@ -13477,7 +13477,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.003898, mean_rel=0.027017", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.003898, mean_rel=0.027017", "invalidated": false, "invalidation_reason": null }, @@ -13577,7 +13577,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 4/24 components failed (4 high)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 4/24 components failed (4 high)", "invalidated": false, "invalidation_reason": null }, @@ -13667,7 +13667,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "invalidated": false, "invalidation_reason": null }, @@ -13817,7 +13817,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -13837,7 +13837,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -13847,7 +13847,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=38.5% < 75.0% (failed: \u2014 1/15 components failed (1 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=38.5% < 75.0% (failed: — 1/15 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -13867,7 +13867,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 1152 (input tensor's size at dimension 0), but go", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 1152 (input tensor's size at dimension 0), but go", "invalidated": false, "invalidation_reason": null }, @@ -14077,7 +14077,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 12/87 components failed (12 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 12/87 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14107,7 +14107,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 12/87 components failed (12 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 12/87 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14157,7 +14157,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: \u2014 144/196 components failed (144 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=15.4% < 75.0% (failed: — 144/196 components failed (144 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14287,7 +14287,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=59.411900, mean_rel=0.569168", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=59.411900, mean_rel=0.569168", "invalidated": false, "invalidation_reason": null }, @@ -14297,7 +14297,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=59.402462, mean_rel=0.441563", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=59.402462, mean_rel=0.441563", "invalidated": false, "invalidation_reason": null }, @@ -14307,7 +14307,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=33.204865, mean_rel=0.370595", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=33.204865, mean_rel=0.370595", "invalidated": false, "invalidation_reason": null }, @@ -14317,7 +14317,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=32.255035, mean_rel=0.318908", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=32.255035, mean_rel=0.318908", "invalidated": false, "invalidation_reason": null }, @@ -14327,7 +14327,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -14337,7 +14337,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=31.355835, mean_rel=0.607207", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=31.355835, mean_rel=0.607207", "invalidated": false, "invalidation_reason": null }, @@ -14347,7 +14347,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 73.9/100 (avg perplexity: 10.2) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 73.9/100 (avg perplexity: 10.2) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -14357,7 +14357,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", "invalidated": false, "invalidation_reason": null }, @@ -14367,7 +14367,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 70/609 components failed (70 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 70/609 components failed (70 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14377,7 +14377,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 70/609 components failed (70 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 70/609 components failed (70 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14387,7 +14387,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 60/534 components failed (60 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 60/534 components failed (60 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14467,7 +14467,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "invalidated": false, "invalidation_reason": null }, @@ -14477,7 +14477,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=16.391922, mean_rel=3.526243", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=16.391922, mean_rel=3.526243", "invalidated": false, "invalidation_reason": null }, @@ -14487,7 +14487,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=18.059958, mean_rel=2.710044", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=18.059958, mean_rel=2.710044", "invalidated": false, "invalidation_reason": null }, @@ -14517,7 +14517,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=4.489960, mean_rel=1.238444", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=4.489960, mean_rel=1.238444", "invalidated": false, "invalidation_reason": null }, @@ -14527,7 +14527,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=20.307718, mean_rel=6.347236", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=20.307718, mean_rel=6.347236", "invalidated": false, "invalidation_reason": null }, @@ -14567,7 +14567,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14597,7 +14597,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14607,7 +14607,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14617,7 +14617,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14627,7 +14627,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14637,7 +14637,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/347 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/347 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -14647,7 +14647,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'type'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'type'", "invalidated": false, "invalidation_reason": null }, @@ -14697,7 +14697,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -14787,7 +14787,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: argument 'vocab': 'dict' object cannot be converted to 'Sequence'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: argument 'vocab': 'dict' object cannot be converted to 'Sequence'", "invalidated": false, "invalidation_reason": null }, @@ -14857,7 +14857,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -14877,7 +14877,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 256 (input tensor's size at dimension 0), but got", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: split_with_sizes expects split_sizes to sum exactly to 256 (input tensor's size at dimension 0), but got", "invalidated": false, "invalidation_reason": null }, @@ -14997,7 +14997,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.005348, mean_rel=0.000007", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.005348, mean_rel=0.000007", "invalidated": false, "invalidation_reason": null }, @@ -15067,7 +15067,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=75.0% but required tests failed \u2014 Tensors differ: max_diff=378.613281, mean_rel=0.057195", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=75.0% but required tests failed — Tensors differ: max_diff=378.613281, mean_rel=0.057195", "invalidated": false, "invalidation_reason": null }, @@ -15227,7 +15227,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 83.9/100 (below 85.0, avg perplexity: 125.9)", "invalidated": false, "invalidation_reason": null }, @@ -15237,7 +15237,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=39.516827, mean_rel=0.391392", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=39.516827, mean_rel=0.391392", "invalidated": false, "invalidation_reason": null }, @@ -15247,7 +15247,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=63.272919, mean_rel=0.497796", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=63.272919, mean_rel=0.497796", "invalidated": false, "invalidation_reason": null }, @@ -15257,7 +15257,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=34.928375, mean_rel=0.262157", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=34.928375, mean_rel=0.262157", "invalidated": false, "invalidation_reason": null }, @@ -15267,7 +15267,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=29.677444, mean_rel=0.238732", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=29.677444, mean_rel=0.238732", "invalidated": false, "invalidation_reason": null }, @@ -15277,7 +15277,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 45.4/100 (avg perplexity: 196.5) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 45.4/100 (avg perplexity: 196.5) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -15287,7 +15287,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=50.417328, mean_rel=0.326480", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=50.417328, mean_rel=0.326480", "invalidated": false, "invalidation_reason": null }, @@ -15297,7 +15297,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=34.771893, mean_rel=0.288610", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=34.771893, mean_rel=0.288610", "invalidated": false, "invalidation_reason": null }, @@ -15307,7 +15307,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=61.738430, mean_rel=0.447178", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=61.738430, mean_rel=0.447178", "invalidated": false, "invalidation_reason": null }, @@ -15407,7 +15407,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "invalidated": false, "invalidation_reason": null }, @@ -15427,7 +15427,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=4.789991, mean_rel=1.159405", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=4.789991, mean_rel=1.159405", "invalidated": false, "invalidation_reason": null }, @@ -15487,7 +15487,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-1b-ONNX-web does not appear to have a file named pytorch_model.bin or model.s", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/granite-4.0-1b-ONNX-web does not appear to have a file named pytorch_model.bin or model.s", "invalidated": false, "invalidation_reason": null }, @@ -15517,7 +15517,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 56/243 components failed (56 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 56/243 components failed (56 critical)", "invalidated": false, "invalidation_reason": null }, @@ -15527,7 +15527,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 72/307 components failed (72 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 72/307 components failed (72 critical)", "invalidated": false, "invalidation_reason": null }, @@ -15537,7 +15537,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g \u2014 144/307 components failed (144 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g — 144/307 components failed (144 critical)", "invalidated": false, "invalidation_reason": null }, @@ -15567,7 +15567,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -15577,7 +15577,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: You set `ignore_mismatched_sizes` to `False`, thus raising an error. For details look at the above repor", "invalidated": false, "invalidation_reason": null }, @@ -15587,7 +15587,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: LiquidAI/LFM2-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: LiquidAI/LFM2-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -15597,7 +15597,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: LiquidAI/LFM2.5-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: LiquidAI/LFM2.5-8B-A1B-ONNX does not appear to have a file named pytorch_model.bin or model.safetensors.", "invalidated": false, "invalidation_reason": null }, @@ -15617,7 +15617,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Loading an AWQ quantized model requires gptqmodel. Please install it with `pip install gptqmodel`", "invalidated": false, "invalidation_reason": null }, @@ -15957,7 +15957,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", "invalidated": false, "invalidation_reason": null }, @@ -15967,7 +15967,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'MixtralDecoderLayer' object has no attribute 'block_sparse_moe'", "invalidated": false, "invalidation_reason": null }, @@ -15977,7 +15977,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -15987,7 +15987,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=6.747103, mean_rel=0.054769", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=6.747103, mean_rel=0.054769", "invalidated": false, "invalidation_reason": null }, @@ -15997,7 +15997,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -16007,7 +16007,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -16017,7 +16017,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.479654, mean_rel=0.052641", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.479654, mean_rel=0.052641", "invalidated": false, "invalidation_reason": null }, @@ -16027,7 +16027,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -16037,7 +16037,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=6.650925, mean_rel=0.050073", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=6.650925, mean_rel=0.050073", "invalidated": false, "invalidation_reason": null }, @@ -16047,7 +16047,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Generated text has no new tokens", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -16057,7 +16057,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.641898, mean_rel=0.054789", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.641898, mean_rel=0.054789", "invalidated": false, "invalidation_reason": null }, @@ -16067,7 +16067,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=5.808517, mean_rel=0.051374", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=5.808517, mean_rel=0.051374", "invalidated": false, "invalidation_reason": null }, @@ -16307,7 +16307,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: onnx-community/tiny-random-olmo-hf does not appear to have a file named pytorch_model.bin or model.safet", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: onnx-community/tiny-random-olmo-hf does not appear to have a file named pytorch_model.bin or model.safet", "invalidated": false, "invalidation_reason": null }, @@ -16327,7 +16327,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 32/68 components failed (32 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 32/68 components failed (32 critical)", "invalidated": false, "invalidation_reason": null }, @@ -16337,7 +16337,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "invalidated": false, "invalidation_reason": null }, @@ -16347,7 +16347,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: self_attn: q_norm declared but HF module has none.", "invalidated": false, "invalidation_reason": null }, @@ -16357,7 +16357,7 @@ "verified_date": "2026-06-26", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.013494, mean_rel=0.006767", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.013494, mean_rel=0.006767", "invalidated": false, "invalidation_reason": null }, @@ -16367,7 +16367,7 @@ "verified_date": "2026-06-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed \u2014 Tensors differ: max_diff=0.437500, mean_rel=0.223633", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed — Tensors differ: max_diff=0.437500, mean_rel=0.223633", "invalidated": false, "invalidation_reason": null }, @@ -16377,7 +16377,7 @@ "verified_date": "2026-06-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.013494, mean_rel=0.006767", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.013494, mean_rel=0.006767", "invalidated": false, "invalidation_reason": null }, @@ -16387,7 +16387,7 @@ "verified_date": "2026-06-27", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=0.020484, mean_rel=0.006617", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=0.020484, mean_rel=0.006617", "invalidated": false, "invalidation_reason": null }, @@ -16397,7 +16397,7 @@ "verified_date": "2026-07-01", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 96/171 components failed (96 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 96/171 components failed (96 critical)", "invalidated": true, "invalidation_reason": "Superseded by the clean re-run (P1=100) after the component-benchmark fix that skips SSM mixer-internal submodules; the 96/171 component failures were the isolated harness feeding d_model-shaped inputs to SSM-internal projections, not a real divergence." }, @@ -16427,7 +16427,7 @@ "verified_date": "2026-07-01", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence \u2014 Tensors differ: max_diff=0.375000, mean_rel=0.002045", + "notes": "Below threshold: P3=94.7% but required tests failed: logits_equivalence — Tensors differ: max_diff=0.375000, mean_rel=0.002045", "invalidated": true, "invalidation_reason": "bf16 precision of compatibility-mode center_unembed (root-caused by toggle: off=0.000 both dtypes, fp32=4.2e-5); superseded by the clean fp32 run (P3=100). Not an algorithmic bug." }, @@ -16547,7 +16547,7 @@ "verified_date": "2026-07-01", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 24/51 components failed (24 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 24/51 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -16577,7 +16577,7 @@ "verified_date": "2026-07-01", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) \u2014 50/99 components failed (50 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits) — 50/99 components failed (50 critical)", "invalidated": false, "invalidation_reason": null }, @@ -16897,7 +16897,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/16 components failed (2 high)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/16 components failed (2 high)", "invalidated": false, "invalidation_reason": null }, @@ -16907,7 +16907,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/16 components failed (2 high)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/16 components failed (2 high)", "invalidated": false, "invalidation_reason": null }, @@ -16977,7 +16977,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass) \u2014 Forward pass failed: Could not infer dtype of NoneType", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass) — Forward pass failed: Could not infer dtype of NoneType", "invalidated": false, "invalidation_reason": null }, @@ -16987,7 +16987,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass) \u2014 Forward pass failed: Could not infer dtype of NoneType", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass) — Forward pass failed: Could not infer dtype of NoneType", "invalidated": false, "invalidation_reason": null }, @@ -17007,7 +17007,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/75 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/75 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17017,7 +17017,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/75 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/75 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17027,7 +17027,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/75 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/75 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17037,7 +17037,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/75 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/75 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17067,7 +17067,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Cannot build Piece from string \":0\"", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Cannot build Piece from string \":0\"", "invalidated": false, "invalidation_reason": null }, @@ -17077,7 +17077,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed \u2014 Tensors differ: max_diff=0.084499, mean_rel=1.582682", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed — Tensors differ: max_diff=0.084499, mean_rel=1.582682", "invalidated": false, "invalidation_reason": null }, @@ -17087,7 +17087,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed \u2014 Tensors differ: max_diff=0.096940, mean_rel=2.925628", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=95.0% but required tests failed — Tensors differ: max_diff=0.096940, mean_rel=2.925628", "invalidated": false, "invalidation_reason": null }, @@ -17097,7 +17097,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=0.079056, mean_rel=0.949212", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=0.079056, mean_rel=0.949212", "invalidated": false, "invalidation_reason": null }, @@ -17117,7 +17117,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=30.8% < 75.0% (failed: \u2014 5/12 components failed (5 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=30.8% < 75.0% (failed: — 5/12 components failed (5 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17127,7 +17127,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=30.8% < 75.0% (failed: \u2014 5/12 components failed (5 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=30.8% < 75.0% (failed: — 5/12 components failed (5 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17147,7 +17147,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/16 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/16 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17157,7 +17157,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/16 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/16 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17177,7 +17177,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=13.567083, mean_rel=3.490963", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=13.567083, mean_rel=3.490963", "invalidated": false, "invalidation_reason": null }, @@ -17187,7 +17187,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=13.567083, mean_rel=3.490963", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=13.567083, mean_rel=3.490963", "invalidated": false, "invalidation_reason": null }, @@ -17227,7 +17227,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=78.9% but requir \u2014 30/184 components failed (30 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=78.9% but requir — 30/184 components failed (30 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17237,7 +17237,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=78.9% but requir \u2014 30/184 components failed (30 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=78.9% but requir — 30/184 components failed (30 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17247,7 +17247,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=78.9% but required tests failed \u2014 Tensors differ: max_diff=27.805214, mean_rel=17.542879", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=78.9% but required tests failed — Tensors differ: max_diff=27.805214, mean_rel=17.542879", "invalidated": false, "invalidation_reason": null }, @@ -17257,7 +17257,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Attention output weights not well-centered (worst_mean=0.061788)", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Attention output weights not well-centered (worst_mean=0.061788)", "invalidated": false, "invalidation_reason": null }, @@ -17267,7 +17267,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Attention output weights not well-centered (worst_mean=0.061788)", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Attention output weights not well-centered (worst_mean=0.061788)", "invalidated": false, "invalidation_reason": null }, @@ -17277,7 +17277,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence \u2014 Attention output weights not well-centered (worst_mean=0.061788)", + "notes": "Below threshold: P3=78.9% but required tests failed: logits_equivalence, loss_equivalence — Attention output weights not well-centered (worst_mean=0.061788)", "invalidated": false, "invalidation_reason": null }, @@ -17307,7 +17307,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 Error running comprehensive component benchmark: index 2 is out of range", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — Error running comprehensive component benchmark: index 2 is out of range", "invalidated": false, "invalidation_reason": null }, @@ -17367,7 +17367,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Checkpoint ships no lm_head.weight with tie_word_embeddings=false, so HF randomly re-initializes the LM head on every load (Google released it as a pretraining artifact requiring fine-tuning); the nondeterministic unembed is the only P1 failure \u2014 all 233 other components including the local-attention encoder pass. Not an adapter bug.", + "notes": "Checkpoint ships no lm_head.weight with tie_word_embeddings=false, so HF randomly re-initializes the LM head on every load (Google released it as a pretraining artifact requiring fine-tuning); the nondeterministic unembed is the only P1 failure — all 233 other components including the local-attention encoder pass. Not an adapter bug.", "invalidated": false, "invalidation_reason": null }, @@ -17437,7 +17437,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Verified on a local snapshot of yujiepan/llama-4-tiny-random with text_config.attn_temperature_tuning coerced to bool \u2014 the upstream config declares it as int 4, which transformers 5.x strict config validation rejects. P7 skipped: the tiny ships no processor files. P3=95 (attention_output_centering worst_mean=0.099 on random weights). Official Scout/Maverick checkpoints registered for big-hardware verification.", + "notes": "Verified on a local snapshot of yujiepan/llama-4-tiny-random with text_config.attn_temperature_tuning coerced to bool — the upstream config declares it as int 4, which transformers 5.x strict config validation rejects. P7 skipped: the tiny ships no processor files. P3=95 (attention_output_centering worst_mean=0.099 on random weights). Official Scout/Maverick checkpoints registered for big-hardware verification.", "invalidated": false, "invalidation_reason": null }, @@ -17517,7 +17517,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=30.078743, mean_rel=0.607381", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=30.078743, mean_rel=0.607381", "invalidated": false, "invalidation_reason": null }, @@ -17537,7 +17537,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=30.078743, mean_rel=0.607381", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=30.078743, mean_rel=0.607381", "invalidated": false, "invalidation_reason": null }, @@ -17567,7 +17567,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=27.217707, mean_rel=1.057937", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=27.217707, mean_rel=1.057937", "invalidated": false, "invalidation_reason": null }, @@ -17597,7 +17597,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g \u2014 90/154 components failed (90 critical)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass); P2=7.7% < 75.0% (failed: g — 90/154 components failed (90 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17777,7 +17777,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -17797,7 +17797,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/462 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/462 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -17907,7 +17907,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation \u2014 Generated text has no new tokens", + "notes": "Below threshold: P2=69.2% < 75.0% (failed: generation, generation_with_kv_cache, multiple_generation — Generated text has no new tokens", "invalidated": false, "invalidation_reason": null }, @@ -17927,7 +17927,7 @@ "verified_date": "2026-07-07", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=30.8% < 75.0% (failed: hook_functional \u2014 Forward pass failed: index out of range in self", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=30.8% < 75.0% (failed: hook_functional — Forward pass failed: index out of range in self", "invalidated": false, "invalidation_reason": null }, @@ -18027,7 +18027,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Input must be a List[Union[str, AddedToken]]", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Input must be a List[Union[str, AddedToken]]", "invalidated": false, "invalidation_reason": null }, @@ -18097,7 +18097,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: DreamGenerationConfig.validate() got an unexpected keyword argument 'user_set_attributes'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: DreamGenerationConfig.validate() got an unexpected keyword argument 'user_set_attributes'", "invalidated": false, "invalidation_reason": null }, @@ -18187,7 +18187,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Text quality score: 76.4/100 (avg perplexity: 16.6) \u2014 generated text may be incoherent", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Text quality score: 76.4/100 (avg perplexity: 16.6) — generated text may be incoherent", "invalidated": false, "invalidation_reason": null }, @@ -18197,7 +18197,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=0.207234, mean_rel=0.000825", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=0.207234, mean_rel=0.000825", "invalidated": false, "invalidation_reason": null }, @@ -18237,7 +18237,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=19.557707, mean_rel=0.281903", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=19.557707, mean_rel=0.281903", "invalidated": false, "invalidation_reason": null }, @@ -18257,7 +18257,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=19.557709, mean_rel=0.281903", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=19.557709, mean_rel=0.281903", "invalidated": false, "invalidation_reason": null }, @@ -18267,7 +18267,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=19.557707, mean_rel=0.281903", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=19.557707, mean_rel=0.281903", "invalidated": false, "invalidation_reason": null }, @@ -18297,7 +18297,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'GiddForDiffusionLM' object has no attribute 'all_tied_weights_keys'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'GiddForDiffusionLM' object has no attribute 'all_tied_weights_keys'", "invalidated": false, "invalidation_reason": null }, @@ -18307,7 +18307,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'GiddModel' object has no attribute 'weight'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'GiddModel' object has no attribute 'weight'", "invalidated": false, "invalidation_reason": null }, @@ -18317,7 +18317,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=26.121746, mean_rel=4.879314", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=26.121746, mean_rel=4.879314", "invalidated": false, "invalidation_reason": null }, @@ -18327,7 +18327,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/136 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/136 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18337,7 +18337,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/268 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/268 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18347,7 +18347,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/136 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/136 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18357,7 +18357,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/268 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/268 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18387,7 +18387,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", "invalidated": false, "invalidation_reason": null }, @@ -18397,7 +18397,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 24/100 components failed (24 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 24/100 components failed (24 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18417,7 +18417,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=42.9% < 75.0% (failed: hook_functional \u2014 Forward pass failed: 'tuple' object has no attribute 'dtype'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=42.9% < 75.0% (failed: hook_functional — Forward pass failed: 'tuple' object has no attribute 'dtype'", "invalidated": false, "invalidation_reason": null }, @@ -18447,7 +18447,7 @@ "verified_date": "2026-07-08", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 36/475 components failed (36 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 36/475 components failed (36 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18467,7 +18467,7 @@ "verified_date": "2026-07-12", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log \u2014 8/98 components failed (8 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log — 8/98 components failed (8 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18477,7 +18477,7 @@ "verified_date": "2026-07-12", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18487,7 +18487,7 @@ "verified_date": "2026-07-12", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=90.0% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18497,7 +18497,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18507,7 +18507,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18517,7 +18517,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 10/114 components failed (10 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 10/114 components failed (10 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18527,7 +18527,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log \u2014 20/114 components failed (20 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P3=89.5% but required tests failed: log — 20/114 components failed (20 critical)", "invalidated": false, "invalidation_reason": null }, @@ -18537,7 +18537,7 @@ "verified_date": "2026-07-14", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=23.349285, mean_rel=0.575189", + "notes": "Below threshold: P3=89.5% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=23.349285, mean_rel=0.575189", "invalidated": false, "invalidation_reason": null }, @@ -18627,7 +18627,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -18637,7 +18637,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -18647,7 +18647,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: The model's quantization config from the arguments has no `quant_method` attribute. Make sure that the m", "invalidated": false, "invalidation_reason": null }, @@ -18787,7 +18787,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed \u2014 Tensors differ: max_diff=11.357496, mean_rel=4.069944", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits); P3=90.0% but required tests failed — Tensors differ: max_diff=11.357496, mean_rel=4.069944", "invalidated": false, "invalidation_reason": null }, @@ -18837,7 +18837,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene \u2014 Forward pass failed: create_causal_mask() got an unexpected keyword argument 'input_embeds'", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene — Forward pass failed: create_causal_mask() got an unexpected keyword argument 'input_embeds'", "invalidated": false, "invalidation_reason": null }, @@ -18847,7 +18847,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: 'FalconDecoderLayer' object has no attribute 'ln_attn'", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: 'FalconDecoderLayer' object has no attribute 'ln_attn'", "invalidated": false, "invalidation_reason": null }, @@ -18917,7 +18917,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene \u2014 Forward pass failed: index out of range in self", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=7.7% < 75.0% (failed: generation, gene — Forward pass failed: index out of range in self", "invalidated": false, "invalidation_reason": null }, @@ -18947,7 +18947,7 @@ "verified_date": "2026-07-21", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, ", "invalidated": false, "invalidation_reason": null }, @@ -19257,7 +19257,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 Error running comprehensive component benchmark: Component attn not found in blocks.0 components", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — Error running comprehensive component benchmark: Component attn not found in blocks.0 components", "invalidated": false, "invalidation_reason": null }, @@ -19297,7 +19297,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence \u2014 Tensors differ: max_diff=32.690491, mean_rel=0.409154", + "notes": "Below threshold: P3=90.0% but required tests failed: logits_equivalence, loss_equivalence — Tensors differ: max_diff=32.690491, mean_rel=0.409154", "invalidated": false, "invalidation_reason": null }, @@ -19327,7 +19327,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P2=31.2% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook \u2014 Logits computation failed: Invalid positional_embedding_type passed in relative_positional_bias", + "notes": "Below threshold: P2=31.2% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_registry, hook — Logits computation failed: Invalid positional_embedding_type passed in relative_positional_bias", "invalidated": false, "invalidation_reason": null }, @@ -19347,7 +19347,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 12/126 components failed (12 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 12/126 components failed (12 critical)", "invalidated": false, "invalidation_reason": null }, @@ -19457,7 +19457,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", "invalidated": false, "invalidation_reason": null }, @@ -19467,7 +19467,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=2.599856, mean_rel=0.102672", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=2.599856, mean_rel=0.102672", "invalidated": false, "invalidation_reason": null }, @@ -19517,7 +19517,7 @@ "verified_date": "2026-07-22", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P3=83.3% but required tests failed: logits_equivalence, loss_equivalence \u2014 Unembed matrix not well-centered (mean=0.075038)", + "notes": "Below threshold: P3=83.3% but required tests failed: logits_equivalence, loss_equivalence — Unembed matrix not well-centered (mean=0.075038)", "invalidated": false, "invalidation_reason": null }, @@ -19927,7 +19927,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=17.494274, mean_rel=8.712387", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=17.494274, mean_rel=8.712387", "invalidated": false, "invalidation_reason": null }, @@ -19937,7 +19937,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=18.492882, mean_rel=11.615888", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=18.492882, mean_rel=11.615888", "invalidated": false, "invalidation_reason": null }, @@ -19947,7 +19947,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=17.494274, mean_rel=8.712387", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=17.494274, mean_rel=8.712387", "invalidated": false, "invalidation_reason": null }, @@ -19957,7 +19957,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) \u2014 Tensors differ: max_diff=18.492882, mean_rel=11.615888", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass_logits) — Tensors differ: max_diff=18.492882, mean_rel=11.615888", "invalidated": false, "invalidation_reason": null }, @@ -20187,7 +20187,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=30.8% < 75.0% (failed: hook_functional \u2014 Forward pass failed: index out of range in self", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: forward_pass); P2=30.8% < 75.0% (failed: hook_functional — Forward pass failed: index out of range in self", "invalidated": false, "invalidation_reason": null }, @@ -20387,7 +20387,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/23 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/23 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -20397,7 +20397,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=90.0% but requir \u2014 1/15 components failed (1 low)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=90.0% but requir — 1/15 components failed (1 low)", "invalidated": false, "invalidation_reason": null }, @@ -20417,7 +20417,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 2/23 components failed (2 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 2/23 components failed (2 critical)", "invalidated": false, "invalidation_reason": null }, @@ -20427,7 +20427,7 @@ "verified_date": "2026-07-23", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=90.0% but requir \u2014 1/15 components failed (1 low)", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: all_components, forward_pass_logits); P3=90.0% but requir — 1/15 components failed (1 low)", "invalidated": false, "invalidation_reason": null }, @@ -20607,7 +20607,7 @@ "verified_date": "2026-07-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) \u2014 1/107 components failed (1 critical)", + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components) — 1/107 components failed (1 critical)", "invalidated": false, "invalidation_reason": null }, @@ -20777,7 +20777,7 @@ "verified_date": "2026-07-24", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P8=0.0% < 75.0% (failed: audio_text_forward) \u2014 Audio-conditioned forward failed: 'NoneType' object has no attribute '_attn_implementation'", + "notes": "Below threshold: P8=0.0% < 75.0% (failed: audio_text_forward) — Audio-conditioned forward failed: 'NoneType' object has no attribute '_attn_implementation'", "invalidated": false, "invalidation_reason": null }, @@ -20907,7 +20907,7 @@ "verified_date": "2026-07-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P8=0.0% < 75.0% (failed: audio_forward, audio_cache, audio_representation_stability \u2014 Audio forward pass failed: Dimension out of range (expected to be in range of [-3, 2], but got 3)", + "notes": "Below threshold: P8=0.0% < 75.0% (failed: audio_forward, audio_cache, audio_representation_stability — Audio forward pass failed: Dimension out of range (expected to be in range of [-3, 2], but got 3)", "invalidated": false, "invalidation_reason": null }, @@ -20957,7 +20957,7 @@ "verified_date": "2026-07-30", "verified_by": "verify_models", "transformerlens_version": null, - "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) \u2014 Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", + "notes": "Below threshold: P1=0.0% < 100.0% (failed: load_bridge_unprocessed) — Failed to load unprocessed TransformerBridge: Could not determine supported architecture from config. Available architectures: ['AfmoeForCausalLM', 'A", "invalidated": false, "invalidation_reason": null }, @@ -21280,6 +21280,1766 @@ "notes": "Full verification completed", "invalidated": false, "invalidation_reason": null + }, + { + "model_id": "google-t5/t5-small", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-t5/t5-small", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-t5/t5-base", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "openai-community/gpt2", + "architecture_id": "GPT2LMHeadModel", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "distilbert/distilgpt2", + "architecture_id": "GPT2LMHeadModel", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "swiss-ai/Apertus-v1.1-0.5B", + "architecture_id": "ApertusForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "swiss-ai/Apertus-v1.1-1.5B", + "architecture_id": "ApertusForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "apple/OpenELM-270M", + "architecture_id": "OpenELMForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "apple/OpenELM-1_1B", + "architecture_id": "OpenELMForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Isotonic/TinyMixtral-4x248M-MoE", + "architecture_id": "MixtralForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "BEE-spoke-data/Mixtral-GQA-400m-v2", + "architecture_id": "MixtralForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-4-E2B", + "architecture_id": "Gemma4ForConditionalGeneration", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-2-2b", + "architecture_id": "Gemma2ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=93.3% (failed: backward_hooks); P3=90.9% (failed: unembed_centering, backward_hooks)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-2-2b-it", + "architecture_id": "Gemma2ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P3=95.5% (failed: unembed_centering)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/Phi-3-mini-4k-instruct", + "architecture_id": "Phi3ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/Phi-3.5-mini-instruct", + "architecture_id": "Phi3ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "01-ai/Yi-6B", + "architecture_id": "LlamaForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "01-ai/Yi-1.5-6B", + "architecture_id": "LlamaForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen2.5-0.5B", + "architecture_id": "Qwen2ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen2.5-1.5B", + "architecture_id": "Qwen2ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen3-0.6B", + "architecture_id": "Qwen3ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=93.3% (failed: backward_hooks)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen3-1.7B", + "architecture_id": "Qwen3ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/pythia-70m", + "architecture_id": "GPTNeoXForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/pythia-1.4b", + "architecture_id": "GPTNeoXForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/gpt-neo-125m", + "architecture_id": "GPTNeoForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/gpt-neo-1.3B", + "architecture_id": "GPTNeoForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/opt-125m", + "architecture_id": "OPTForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/opt-1.3b", + "architecture_id": "OPTForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigscience/bloom-560m", + "architecture_id": "BloomForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=93.3% (failed: gated_hooks_fire); P3=95.5% (failed: gated_hooks_fire)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigscience/bloom-1b7", + "architecture_id": "BloomForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=93.3% (failed: gated_hooks_fire); P3=95.5% (failed: gated_hooks_fire)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-2b", + "architecture_id": "GemmaForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-3-270m", + "architecture_id": "Gemma3ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-3-1b-it", + "architecture_id": "Gemma3ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/phi-1", + "architecture_id": "PhiForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/phi-2", + "architecture_id": "PhiForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=93.3% (failed: backward_hooks)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "stabilityai/stablelm-2-1_6b", + "architecture_id": "StableLmForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "state-spaces/mamba-130m-hf", + "architecture_id": "MambaForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "ibm-granite/granite-3.1-2b-instruct", + "architecture_id": "GraniteForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "HuggingFaceTB/SmolLM3-3B", + "architecture_id": "SmolLM3ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/OLMo-2-0425-1B", + "architecture_id": "Olmo2ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/Olmo-3-1025-7B", + "architecture_id": "Olmo3ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Below threshold: P3=71.4% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=23.143579, mean_rel=9.020543", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/OLMoE-1B-7B-0924", + "architecture_id": "OlmoeForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Below threshold: P3=72.7% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=26.260244, mean_rel=3.783418", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/gpt-j-6b", + "architecture_id": "GPTJForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "mistralai/Mistral-7B-v0.1", + "architecture_id": "MistralForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/bart-large-cnn", + "architecture_id": "BartForConditionalGeneration", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigscience/mt0-base", + "architecture_id": "MT5ForConditionalGeneration", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/long-t5-tglobal-base", + "architecture_id": "LongT5ForConditionalGeneration", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/pegasus-xsum", + "architecture_id": "PegasusForConditionalGeneration", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/m2m100_418M", + "architecture_id": "M2M100ForConditionalGeneration", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigcode/starcoder2-3b", + "architecture_id": "Starcoder2ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "internlm/internlm2-chat-1_8b", + "architecture_id": "InternLM2ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/Olmo-3-1025-7B", + "architecture_id": "Olmo3ForCausalLM", + "verified_date": "2026-08-19", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-t5/t5-small", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=92.9% (failed: hook_functionality)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-t5/t5-base", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality: P2=92.9% (failed: hook_functionality)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "openai-community/gpt2", + "architecture_id": "GPT2LMHeadModel", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "distilbert/distilgpt2", + "architecture_id": "GPT2LMHeadModel", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "swiss-ai/Apertus-v1.1-0.5B", + "architecture_id": "ApertusForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "swiss-ai/Apertus-v1.1-1.5B", + "architecture_id": "ApertusForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "apple/OpenELM-270M", + "architecture_id": "OpenELMForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "apple/OpenELM-1_1B", + "architecture_id": "OpenELMForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Isotonic/TinyMixtral-4x248M-MoE", + "architecture_id": "MixtralForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "BEE-spoke-data/Mixtral-GQA-400m-v2", + "architecture_id": "MixtralForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-4-E2B", + "architecture_id": "Gemma4ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-2-2b", + "architecture_id": "Gemma2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P3=95.5% (failed: unembed_centering)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-2-2b-it", + "architecture_id": "Gemma2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P3=95.5% (failed: unembed_centering)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/Phi-3-mini-4k-instruct", + "architecture_id": "Phi3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/Phi-3.5-mini-instruct", + "architecture_id": "Phi3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "01-ai/Yi-6B", + "architecture_id": "LlamaForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "01-ai/Yi-1.5-6B", + "architecture_id": "LlamaForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen2.5-0.5B", + "architecture_id": "Qwen2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen2.5-1.5B", + "architecture_id": "Qwen2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen3-0.6B", + "architecture_id": "Qwen3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen3-1.7B", + "architecture_id": "Qwen3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/pythia-70m", + "architecture_id": "GPTNeoXForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/pythia-1.4b", + "architecture_id": "GPTNeoXForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/gpt-neo-125m", + "architecture_id": "GPTNeoForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/gpt-neo-1.3B", + "architecture_id": "GPTNeoForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/opt-125m", + "architecture_id": "OPTForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/opt-1.3b", + "architecture_id": "OPTForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigscience/bloom-560m", + "architecture_id": "BloomForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigscience/bloom-1b7", + "architecture_id": "BloomForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-2b", + "architecture_id": "GemmaForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-3-270m", + "architecture_id": "Gemma3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-3-1b-it", + "architecture_id": "Gemma3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/phi-1", + "architecture_id": "PhiForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/phi-2", + "architecture_id": "PhiForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=93.3% (failed: backward_hooks)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "stabilityai/stablelm-2-1_6b", + "architecture_id": "StableLmForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "state-spaces/mamba-130m-hf", + "architecture_id": "MambaForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "ibm-granite/granite-3.1-2b-instruct", + "architecture_id": "GraniteForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "HuggingFaceTB/SmolLM3-3B", + "architecture_id": "SmolLM3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/OLMo-2-0425-1B", + "architecture_id": "Olmo2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/Olmo-3-1025-7B", + "architecture_id": "Olmo3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/OLMoE-1B-7B-0924", + "architecture_id": "OlmoeForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P3=95.5% (failed: forward_hooks)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/gpt-j-6b", + "architecture_id": "GPTJForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "mistralai/Mistral-7B-v0.1", + "architecture_id": "MistralForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/bart-large-cnn", + "architecture_id": "BartForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=92.9% (failed: hook_functionality)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigscience/mt0-base", + "architecture_id": "MT5ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality: P2=92.9% (failed: hook_functionality)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/long-t5-tglobal-base", + "architecture_id": "LongT5ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality: P2=92.9% (failed: hook_functionality)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/pegasus-xsum", + "architecture_id": "PegasusForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=92.9% (failed: hook_functionality)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/m2m100_418M", + "architecture_id": "M2M100ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality: P2=92.9% (failed: hook_functionality)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigcode/starcoder2-3b", + "architecture_id": "Starcoder2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "internlm/internlm2-chat-1_8b", + "architecture_id": "InternLM2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-t5/t5-small", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-t5/t5-base", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "openai-community/gpt2", + "architecture_id": "GPT2LMHeadModel", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "distilbert/distilgpt2", + "architecture_id": "GPT2LMHeadModel", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "swiss-ai/Apertus-v1.1-0.5B", + "architecture_id": "ApertusForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "swiss-ai/Apertus-v1.1-1.5B", + "architecture_id": "ApertusForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "apple/OpenELM-270M", + "architecture_id": "OpenELMForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "apple/OpenELM-1_1B", + "architecture_id": "OpenELMForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Isotonic/TinyMixtral-4x248M-MoE", + "architecture_id": "MixtralForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "BEE-spoke-data/Mixtral-GQA-400m-v2", + "architecture_id": "MixtralForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-4-E2B", + "architecture_id": "Gemma4ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-2-2b", + "architecture_id": "Gemma2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P3=95.5% (failed: unembed_centering)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-2-2b-it", + "architecture_id": "Gemma2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P3=95.5% (failed: unembed_centering)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/Phi-3-mini-4k-instruct", + "architecture_id": "Phi3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/Phi-3.5-mini-instruct", + "architecture_id": "Phi3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "01-ai/Yi-6B", + "architecture_id": "LlamaForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "01-ai/Yi-1.5-6B", + "architecture_id": "LlamaForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen2.5-0.5B", + "architecture_id": "Qwen2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen2.5-1.5B", + "architecture_id": "Qwen2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen3-0.6B", + "architecture_id": "Qwen3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "Qwen/Qwen3-1.7B", + "architecture_id": "Qwen3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/pythia-70m", + "architecture_id": "GPTNeoXForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/pythia-1.4b", + "architecture_id": "GPTNeoXForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/gpt-neo-125m", + "architecture_id": "GPTNeoForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/gpt-neo-1.3B", + "architecture_id": "GPTNeoForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/opt-125m", + "architecture_id": "OPTForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/opt-1.3b", + "architecture_id": "OPTForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigscience/bloom-560m", + "architecture_id": "BloomForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigscience/bloom-1b7", + "architecture_id": "BloomForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-2b", + "architecture_id": "GemmaForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-3-270m", + "architecture_id": "Gemma3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/gemma-3-1b-it", + "architecture_id": "Gemma3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/phi-1", + "architecture_id": "PhiForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "microsoft/phi-2", + "architecture_id": "PhiForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues: P2=93.3% (failed: backward_hooks)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "stabilityai/stablelm-2-1_6b", + "architecture_id": "StableLmForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "state-spaces/mamba-130m-hf", + "architecture_id": "MambaForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "ibm-granite/granite-3.1-2b-instruct", + "architecture_id": "GraniteForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "HuggingFaceTB/SmolLM3-3B", + "architecture_id": "SmolLM3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/OLMo-2-0425-1B", + "architecture_id": "Olmo2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/Olmo-3-1025-7B", + "architecture_id": "Olmo3ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "allenai/OLMoE-1B-7B-0924", + "architecture_id": "OlmoeForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "EleutherAI/gpt-j-6b", + "architecture_id": "GPTJForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "mistralai/Mistral-7B-v0.1", + "architecture_id": "MistralForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/bart-large-cnn", + "architecture_id": "BartForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigscience/mt0-base", + "architecture_id": "MT5ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/long-t5-tglobal-base", + "architecture_id": "LongT5ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google/pegasus-xsum", + "architecture_id": "PegasusForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "facebook/m2m100_418M", + "architecture_id": "M2M100ForConditionalGeneration", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed with issues, low text quality", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "bigcode/starcoder2-3b", + "architecture_id": "Starcoder2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "internlm/internlm2-chat-1_8b", + "architecture_id": "InternLM2ForCausalLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "openai-community/gpt2", + "architecture_id": "GPT2LMHeadModel", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "continuation" + }, + { + "model_id": "EleutherAI/pythia-70m", + "architecture_id": "GPTNeoXForCausalLM", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "continuation" + }, + { + "model_id": "Helsinki-NLP/opus-mt-nl-en", + "architecture_id": "MarianMTModel", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@nl-en" + }, + { + "model_id": "google-t5/t5-small", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@en-de" + }, + { + "model_id": "google-t5/t5-base", + "architecture_id": "T5ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@en-de" + }, + { + "model_id": "facebook/bart-large-cnn", + "architecture_id": "BartForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:summarization" + }, + { + "model_id": "bigscience/mt0-base", + "architecture_id": "MT5ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification passed, but text quality poor (P4=26.8). Needs review", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:instruction" + }, + { + "model_id": "google/long-t5-tglobal-base", + "architecture_id": "LongT5ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification passed, but text quality poor (P4=48.4). Needs review", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:denoise" + }, + { + "model_id": "google/pegasus-xsum", + "architecture_id": "PegasusForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:summarization" + }, + { + "model_id": "facebook/m2m100_418M", + "architecture_id": "M2M100ForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@en-de" + }, + { + "model_id": "Qwen/Qwen2.5-0.5B-Instruct", + "architecture_id": "Qwen2ForCausalLM", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "chat" + }, + { + "model_id": "google/gemma-2-2b-it", + "architecture_id": "Gemma2ForCausalLM", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "chat" + }, + { + "model_id": "facebook/mbart-large-50-many-to-many-mmt", + "architecture_id": "MBartForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:translation@en-de" + }, + { + "model_id": "facebook/mbart-large-50", + "architecture_id": "MBartForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification completed", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:denoise" + }, + { + "model_id": "facebook/mbart-large-cc25", + "architecture_id": "MBartForConditionalGeneration", + "verified_date": "2026-08-21", + "verified_by": "verify_models", + "transformerlens_version": "0.0.0", + "notes": "Core verification passed, but text quality poor (P4=25.0). Needs review", + "invalidated": false, + "invalidation_reason": null, + "p4_scoring_version": 2, + "prompt_profile": "task:denoise" + }, + { + "model_id": "google-bert/bert-base-cased", + "architecture_id": "BertForMaskedLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva — 1/79 components failed (1 critical)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-bert/bert-base-cased", + "architecture_id": "BertForMaskedLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva — 1/79 components failed (1 critical)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-bert/bert-base-cased", + "architecture_id": "BertForMaskedLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva — 2/79 components failed (2 critical)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-bert/bert-base-cased", + "architecture_id": "BertForMaskedLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Below threshold: P1=50.0% < 100.0% (failed: all_components); P2=68.8% < 75.0% (failed: logits_equiva — 1/79 components failed (1 critical)", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-bert/bert-base-cased", + "architecture_id": "BertForMaskedLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Below threshold: P2=68.8% < 75.0% (failed: logits_equivalence, loss_equivalence, hook_functionality, — Tensors differ: max_diff=29.631586, mean_rel=1.923156", + "invalidated": false, + "invalidation_reason": null + }, + { + "model_id": "google-bert/bert-base-cased", + "architecture_id": "BertForMaskedLM", + "verified_date": "2026-08-20", + "verified_by": "verify_models", + "transformerlens_version": null, + "notes": "Full verification completed", + "invalidated": false, + "invalidation_reason": null } ] } diff --git a/transformer_lens/tools/model_registry/hf_scraper.py b/transformer_lens/tools/model_registry/hf_scraper.py index b34f470346..b48042af72 100644 --- a/transformer_lens/tools/model_registry/hf_scraper.py +++ b/transformer_lens/tools/model_registry/hf_scraper.py @@ -36,6 +36,16 @@ from pathlib import Path from typing import Optional +from transformer_lens.benchmarks.text_quality_profiles import ( + ARCHITECTURE_PROFILE_KINDS, + MODEL_PROFILE_OVERRIDES, + HFSignals, + extract_languages, + is_default_profile, + profile_from_hf_signals, + resolve_profile, +) + from . import HF_SUPPORTED_ARCHITECTURES from .registry_io import is_quantized_model @@ -158,9 +168,33 @@ def _load_existing_gaps(output_dir: Path) -> dict[str, dict]: return by_arch -def _build_model_entry(model_id: str, architecture_id: str) -> dict: - """Build a model entry dict matching the ModelEntry schema.""" - return { +def _extract_profile_signals(model_info) -> HFSignals: # type: ignore[no-untyped-def] + """Distill pipeline_tag/tags/cardData off a listing payload (no extra request). + + Args: + model_info: ModelInfo object from list_models(expand=[..., 'pipeline_tag', + 'tags', 'cardData']) + """ + pipeline_tag = getattr(model_info, "pipeline_tag", None) + tags = tuple(getattr(model_info, "tags", None) or []) + card_data = getattr(model_info, "card_data", None) + card_language = getattr(card_data, "language", None) if card_data is not None else None + if card_language is None and card_data is not None and hasattr(card_data, "get"): + card_language = card_data.get("language") + languages = extract_languages(card_language, tags) + return HFSignals(pipeline_tag=pipeline_tag, languages=languages, tags=tags) + + +def _build_model_entry( + model_id: str, architecture_id: str, signals: Optional[HFSignals] = None +) -> dict: + """Build a model entry dict matching the ModelEntry schema. + + ``signals``, when given, resolves and stores a sparse ``prompt_profile`` key + (omitted when it's just the default) and warns on tag/curation disagreement + — the warning is how curation gaps (missing override/architecture rule) surface. + """ + entry = { "architecture_id": architecture_id, "model_id": model_id, "status": 0, @@ -175,6 +209,25 @@ def _build_model_entry(model_id: str, architecture_id: str) -> dict: "phase8_score": None, "phase9_score": None, } + if signals is not None: + hinted = profile_from_hf_signals(model_id, architecture_id, signals) + resolved = resolve_profile(model_id, architecture_id, signals=signals) + deliberately_curated = ( + model_id in MODEL_PROFILE_OVERRIDES or architecture_id in ARCHITECTURE_PROFILE_KINDS + ) + if hinted is not None and hinted.kind != resolved.kind and not deliberately_curated: + # A disagreement nothing deliberate explains is a curation gap. + logger.warning( + f"Profile mismatch for {model_id} ({architecture_id}): Hub tags say " + f"{hinted.kind!r}, curation resolves {resolved.kind!r}" + ) + if not is_default_profile(resolved): + # Keep key position consistent with ModelEntry.to_dict (after note). + items = list(entry.items()) + items.insert([k for k, _ in items].index("note") + 1, ("prompt_profile", str(resolved))) + entry.clear() + entry.update(items) + return entry def _canonical_author_sweep( @@ -182,6 +235,7 @@ def _canonical_author_sweep( supported_models: list[dict], seen_models: set[str], architecture: Optional[str] = None, + refresh_profiles: bool = False, ) -> int: """Admit canonical-org supported-arch models regardless of downloads. Returns count added. @@ -201,15 +255,31 @@ def _canonical_author_sweep( if architecture is not None and architecture not in expected_archs: continue try: - models_iter = api.list_models(author=author, expand=["config", "safetensors"]) + models_iter = api.list_models( + author=author, + expand=["config", "safetensors", "pipeline_tag", "tags", "cardData"], + ) except Exception as exc: # pragma: no cover — network/transient logger.warning(f"Canonical sweep: list_models(author={author!r}) failed: {exc}") continue # Iterate paginated results; a single timeout shouldn't lose every prior author. + existing_by_id = {m["model_id"]: m for m in supported_models} if refresh_profiles else {} try: for model in models_iter: if model.id in seen_models: + # Below-threshold canonical models are reachable only here; + # the main scan's backfill never sees them. + if refresh_profiles: + existing_entry = existing_by_id.get(model.id) + if existing_entry is not None and "prompt_profile" not in existing_entry: + resolved = resolve_profile( + model.id, + existing_entry.get("architecture_id"), + signals=_extract_profile_signals(model), + ) + if not is_default_profile(resolved): + existing_entry["prompt_profile"] = str(resolved) continue if is_quantized_model(model.id): continue @@ -221,7 +291,8 @@ def _canonical_author_sweep( # Reject e.g. mistralai's non-Mistral checkpoints. if model_arch not in expected_archs: continue - supported_models.append(_build_model_entry(model.id, model_arch)) + signals = _extract_profile_signals(model) + supported_models.append(_build_model_entry(model.id, model_arch, signals)) seen_models.add(model.id) added += 1 logger.info(f"Canonical sweep added: {model.id} ({model_arch})") @@ -242,6 +313,7 @@ def scrape_all_models( min_downloads: int = 500, canonical_sweep: bool = True, architecture: Optional[str] = None, + refresh_profiles: bool = False, ) -> tuple[dict, dict]: """Scrape ALL models from HuggingFace and categorize by architecture. @@ -269,6 +341,9 @@ def scrape_all_models( this class (e.g. ``"LlamaForCausalLM"``). Applies to both the main scan and the canonical-author sweep. Useful for populating the registry after adding a single new adapter without rescanning every architecture. + refresh_profiles: If True, backfill a missing ``prompt_profile`` key onto + already-seen registry entries using the listing payload already in hand — no + extra requests (default: False). Returns: Tuple of (supported_models_dict, architecture_gaps_dict) @@ -292,6 +367,9 @@ def scrape_all_models( # Track all models by architecture (start with existing models) supported_models: list[dict] = list(existing_models) # Preserve existing + # Same dict objects as supported_models — mutating via this index (--refresh-profiles) + # is reflected in the final write. + existing_by_id: dict[str, dict] = {m["model_id"]: m for m in supported_models} unsupported_arch_counts: dict[str, int] = {} # arch -> count unsupported_arch_samples: dict[str, list[str]] = {} # arch -> top model IDs unsupported_arch_downloads: dict[str, int] = {} # arch -> total downloads @@ -361,19 +439,22 @@ def scrape_all_models( logger.info("Will scan ALL new models (this may take a while)") try: - # Use expand=['config', 'safetensors'] to get architecture and parameter - # count data inline with the listing, avoiding per-model API calls. - # With ~1000 models per page, a full scan of 200K+ models needs only - # ~200 paginated requests (well within the 1000 req / 5 min limit). - # Use ``filter`` rather than ``pipeline_tag`` so encoder-decoder models - # are discoverable: HF assigns T5/mT5 a primary pipeline_tag of - # "translation" (or None for mT5) and only lists "text2text-generation" - # in the broader tag list. ``filter`` matches against tags, ``pipeline_tag`` - # only against the canonical primary tag. + # Use expand=['config', 'safetensors', 'pipeline_tag', 'tags', 'cardData'] to get + # architecture, parameter count, and prompt-profile signals inline with the + # listing, avoiding per-model API calls. With ~1000 models per page, a full + # scan of 200K+ models needs only ~200 paginated requests (well within the + # 1000 req / 5 min limit). + # Use ``filter`` rather than ``pipeline_tag`` (the query param) so + # encoder-decoder models are discoverable: HF assigns T5/mT5 a primary + # pipeline_tag of "translation" (or None for mT5) and only lists + # "text2text-generation" in the broader tag list. ``filter`` matches against + # tags, ``pipeline_tag`` only against the canonical primary tag. The + # expanded ``pipeline_tag`` *field* below is a different thing — it's per-model + # metadata fed to profile_from_hf_signals, not a query filter. list_kwargs: dict = { "filter": task, "sort": "downloads", - "expand": ["config", "safetensors"], + "expand": ["config", "safetensors", "pipeline_tag", "tags", "cardData"], } if max_models is not None: list_kwargs["limit"] = max_models + len(seen_models) @@ -393,6 +474,19 @@ def scrape_all_models( # Skip if already in our JSON or processed in this run if model.id in seen_models: skipped += 1 + if refresh_profiles: + existing_entry = existing_by_id.get(model.id) + if ( + existing_entry is not None + and "prompt_profile" not in existing_entry + ): + resolved = resolve_profile( + model.id, + existing_entry.get("architecture_id"), + signals=_extract_profile_signals(model), + ) + if not is_default_profile(resolved): + existing_entry["prompt_profile"] = str(resolved) continue # Filter by minimum download count. Since results are sorted @@ -430,7 +524,8 @@ def scrape_all_models( if arch is None: errors += 1 elif arch in HF_SUPPORTED_ARCHITECTURES: - supported_models.append(_build_model_entry(model.id, arch)) + signals = _extract_profile_signals(model) + supported_models.append(_build_model_entry(model.id, arch, signals)) new_supported += 1 else: unsupported_arch_counts[arch] = unsupported_arch_counts.get(arch, 0) + 1 @@ -537,7 +632,11 @@ def scrape_all_models( # Don't lose the main-scan registry on a sweep-time failure. try: canonical_added = _canonical_author_sweep( - api, supported_models, seen_models, architecture=architecture + api, + supported_models, + seen_models, + architecture=architecture, + refresh_profiles=refresh_profiles, ) new_supported += canonical_added logger.info(f"Canonical sweep added {canonical_added} models.") @@ -830,6 +929,12 @@ def main(): "(e.g. 'LlamaForCausalLM'). Use after adding a new adapter to populate the " "registry with that architecture's models without rescanning everything.", ) + parser.add_argument( + "--refresh-profiles", + action="store_true", + help="Backfill a missing prompt_profile key onto already-seen registry entries " + "from the listing payload already in hand (no extra requests).", + ) args = parser.parse_args() @@ -843,6 +948,7 @@ def main(): min_downloads=args.min_downloads, canonical_sweep=not args.no_canonical_sweep, architecture=args.architecture, + refresh_profiles=args.refresh_profiles, ) diff --git a/transformer_lens/tools/model_registry/registry_io.py b/transformer_lens/tools/model_registry/registry_io.py index dfe075e6a3..d52d87a6eb 100644 --- a/transformer_lens/tools/model_registry/registry_io.py +++ b/transformer_lens/tools/model_registry/registry_io.py @@ -11,6 +11,11 @@ from pathlib import Path from typing import Callable, Optional +from transformer_lens.benchmarks.text_quality_profiles import ( + P4_SCORING_VERSION, + is_default_profile, +) + from .verification import VerificationHistory, VerificationRecord logger = logging.getLogger(__name__) @@ -174,7 +179,14 @@ def _get_tl_version() -> Optional[str]: try: import transformer_lens - return getattr(transformer_lens, "__version__", None) + version = getattr(transformer_lens, "__version__", None) + if version: + return str(version) + # The package exports no __version__; installed-distribution + # metadata is the fallback (dev installs record 0.0.0). + from importlib.metadata import version as dist_version + + return dist_version("transformer-lens") except Exception: return None @@ -186,6 +198,7 @@ def update_model_status( note: Optional[str] = None, phase_scores: Optional[dict[int, Optional[float]]] = None, sanitize_fn: Optional[Callable[[Optional[str]], Optional[str]]] = None, + prompt_profile: Optional[str] = None, ) -> bool: """Update a single model entry in supported_models.json. @@ -202,6 +215,10 @@ def update_model_status( note: Optional note for skip/fail reason phase_scores: Phase score dict {1: float, 2: float, 3: float, 4: float} sanitize_fn: Optional callable to sanitize note strings + prompt_profile: Phase-4 prompt profile actually used (e.g. + "task:translation@en-de"). Sparse: the default "continuation" + removes the key (clearing a stale non-default value), None (no + Phase-4 result) leaves it untouched. Returns: True if entry was found/created and updated @@ -235,6 +252,12 @@ def update_model_status( entry[key] = phase_scores[phase_num] elif key not in entry: entry[key] = None + if prompt_profile is not None and is_default_profile(prompt_profile): + entry.pop("prompt_profile", None) + elif prompt_profile is not None: + entry["prompt_profile"] = prompt_profile + if 4 in phase_scores: + entry["p4_scoring_version"] = P4_SCORING_VERSION # Reorder keys so phase scores are always in numerical order _KEY_ORDER = [ "architecture_id", @@ -243,6 +266,8 @@ def update_model_status( "verified_date", "metadata", "note", + "prompt_profile", + "p4_scoring_version", "phase1_score", "phase2_score", "phase3_score", @@ -281,6 +306,20 @@ def update_model_status( "phase9_score": phase_scores.get(9), } ) + new_entry = data["models"][-1] + extras: list[tuple[str, object]] = [] + if prompt_profile is not None and not is_default_profile(prompt_profile): + extras.append(("prompt_profile", prompt_profile)) + if phase_scores.get(4) is not None: + extras.append(("p4_scoring_version", P4_SCORING_VERSION)) + if extras: + # Keep key position consistent with _KEY_ORDER (after "note"). + items = list(new_entry.items()) + idx = [k for k, _ in items].index("note") + 1 + for offset, pair in enumerate(extras): + items.insert(idx + offset, pair) + new_entry.clear() + new_entry.update(items) updated = True if updated: @@ -296,12 +335,28 @@ def update_model_status( return updated +def registry_prompt_profile(model_id: str) -> Optional[str]: + """Stored prompt_profile for a model, or None. Uncached read: the sweep + rewrites the registry between models.""" + try: + data = load_supported_models_raw() + except Exception: + return None + for entry in data.get("models", []): + if entry.get("model_id") == model_id: + profile = entry.get("prompt_profile") + return profile if isinstance(profile, str) else None + return None + + def add_verification_record( model_id: str, arch_id: str, notes: Optional[str] = None, verified_by: str = "verify_models", sanitize_fn: Optional[Callable[[Optional[str]], Optional[str]]] = None, + prompt_profile: Optional[str] = None, + p4_scoring_version: Optional[int] = None, ) -> None: """Append a VerificationRecord to verification_history.json. @@ -325,6 +380,8 @@ def add_verification_record( verified_by=verified_by, transformerlens_version=_get_tl_version(), notes=notes, + prompt_profile=prompt_profile, + p4_scoring_version=p4_scoring_version, ) history = load_verification_history() diff --git a/transformer_lens/tools/model_registry/schemas.py b/transformer_lens/tools/model_registry/schemas.py index 6cd941c154..8b0b00c0c5 100644 --- a/transformer_lens/tools/model_registry/schemas.py +++ b/transformer_lens/tools/model_registry/schemas.py @@ -64,6 +64,8 @@ class ModelEntry: verified_date: Date when verification was performed metadata: Optional metadata from HuggingFace note: Optional note (skip/fail reason, e.g. "Estimated 48 GB exceeds 16 GB limit") + prompt_profile: Phase-4 prompt profile used (e.g. "task:translation@en-de"); + omitted from JSON for the default continuation profile phase1_score: Benchmark Phase 1 score (HF vs Bridge), 0-100 or None phase2_score: Benchmark Phase 2 score (Bridge vs HT unprocessed), 0-100 or None phase3_score: Benchmark Phase 3 score (Bridge vs HT processed), 0-100 or None @@ -79,6 +81,8 @@ class ModelEntry: verified_date: Optional[date] = None metadata: Optional[ModelMetadata] = None note: Optional[str] = None + prompt_profile: Optional[str] = None + p4_scoring_version: Optional[int] = None phase1_score: Optional[float] = None phase2_score: Optional[float] = None phase3_score: Optional[float] = None @@ -88,8 +92,9 @@ class ModelEntry: phase9_score: Optional[float] = None def to_dict(self) -> dict: - """Convert to a JSON-serializable dictionary.""" - return { + """Convert to a JSON-serializable dictionary. prompt_profile is sparse: + omitted when None so default-profile entries carry no key.""" + result = { "architecture_id": self.architecture_id, "model_id": self.model_id, "status": self.status, @@ -104,6 +109,18 @@ def to_dict(self) -> dict: "phase8_score": self.phase8_score, "phase9_score": self.phase9_score, } + extras: list[tuple[str, object]] = [] + if self.prompt_profile is not None: + extras.append(("prompt_profile", self.prompt_profile)) + if self.p4_scoring_version is not None: + extras.append(("p4_scoring_version", self.p4_scoring_version)) + if extras: + note_index = list(result).index("note") + 1 + items = list(result.items()) + for offset, pair in enumerate(extras): + items.insert(note_index + offset, pair) + result = dict(items) + return result @classmethod def from_dict(cls, data: dict) -> "ModelEntry": @@ -128,6 +145,8 @@ def from_dict(cls, data: dict) -> "ModelEntry": verified_date=verified_date, metadata=metadata, note=data.get("note"), + prompt_profile=data.get("prompt_profile"), + p4_scoring_version=data.get("p4_scoring_version"), phase1_score=data.get("phase1_score"), phase2_score=data.get("phase2_score"), phase3_score=data.get("phase3_score"), diff --git a/transformer_lens/tools/model_registry/validate.py b/transformer_lens/tools/model_registry/validate.py index 17c5049d5d..d070303ea9 100644 --- a/transformer_lens/tools/model_registry/validate.py +++ b/transformer_lens/tools/model_registry/validate.py @@ -290,6 +290,31 @@ def _validate_model_entry(data: dict, path: str) -> list[ValidationError]: if "note" in data and data["note"] is not None: errors.extend(_validate_string(data["note"], f"{path}.note", min_length=1)) + # p4_scoring_version (optional sparse int; absent = old GPT-2 scale) + if "p4_scoring_version" in data and data["p4_scoring_version"] is not None: + version = data["p4_scoring_version"] + if not isinstance(version, int) or isinstance(version, bool) or version < 2: + errors.append( + ValidationError(f"{path}.p4_scoring_version", "must be an int >= 2", version) + ) + + # prompt_profile (optional sparse string; must parse as a profile spec) + if "prompt_profile" in data and data["prompt_profile"] is not None: + errors.extend( + _validate_string(data["prompt_profile"], f"{path}.prompt_profile", min_length=1) + ) + if isinstance(data["prompt_profile"], str): + try: + from transformer_lens.benchmarks.text_quality_profiles import ( + ProfileSpec, + ) + + ProfileSpec.parse(data["prompt_profile"]) + except ValueError as e: + errors.append( + ValidationError(f"{path}.prompt_profile", str(e), data["prompt_profile"]) + ) + # verified_date (optional date string) if "verified_date" in data and data["verified_date"] is not None: errors.extend( @@ -310,7 +335,15 @@ def _validate_model_entry(data: dict, path: str) -> list[ValidationError]: errors.extend(_validate_model_metadata(data["metadata"], f"{path}.metadata")) # phase scores (optional floats, 0-100 or None) - for phase_field in ("phase1_score", "phase2_score", "phase3_score"): + for phase_field in ( + "phase1_score", + "phase2_score", + "phase3_score", + "phase4_score", + "phase7_score", + "phase8_score", + "phase9_score", + ): if phase_field in data and data[phase_field] is not None: val = data[phase_field] if not isinstance(val, (int, float)) or isinstance(val, bool): diff --git a/transformer_lens/tools/model_registry/verification.py b/transformer_lens/tools/model_registry/verification.py index f402aef435..f6734cf4a6 100644 --- a/transformer_lens/tools/model_registry/verification.py +++ b/transformer_lens/tools/model_registry/verification.py @@ -29,6 +29,10 @@ class VerificationRecord: architecture_id: str = "Unknown" verified_by: Optional[str] = None transformerlens_version: Optional[str] = None + # P4 verdict flips are undiagnosable without knowing which profile and + # scoring scale produced the record. + prompt_profile: Optional[str] = None + p4_scoring_version: Optional[int] = None notes: Optional[str] = None invalidated: bool = False invalidation_reason: Optional[str] = None @@ -41,6 +45,8 @@ def to_dict(self) -> dict: "verified_date": self.verified_date.isoformat(), "verified_by": self.verified_by, "transformerlens_version": self.transformerlens_version, + "prompt_profile": self.prompt_profile, + "p4_scoring_version": self.p4_scoring_version, "notes": self.notes, "invalidated": self.invalidated, "invalidation_reason": self.invalidation_reason, @@ -55,6 +61,8 @@ def from_dict(cls, data: dict) -> "VerificationRecord": verified_date=date.fromisoformat(data["verified_date"]), verified_by=data.get("verified_by"), transformerlens_version=data.get("transformerlens_version"), + prompt_profile=data.get("prompt_profile"), + p4_scoring_version=data.get("p4_scoring_version"), notes=data.get("notes"), invalidated=data.get("invalidated", False), invalidation_reason=data.get("invalidation_reason"), diff --git a/transformer_lens/tools/model_registry/verify_models.py b/transformer_lens/tools/model_registry/verify_models.py index 66ebcca592..df549bf8a1 100644 --- a/transformer_lens/tools/model_registry/verify_models.py +++ b/transformer_lens/tools/model_registry/verify_models.py @@ -35,6 +35,10 @@ from pathlib import Path from typing import Optional +from transformer_lens.benchmarks.text_quality_profiles import ( + P4_SCORING_VERSION, + p4_pass_threshold, +) from transformer_lens.utilities.heterogeneous_config import het_safe_view # Exit code used for graceful interrupts (Ctrl+C). The wrapper script @@ -394,6 +398,7 @@ def estimate_benchmark_memory_gb( dtype: str = "float32", phases: Optional[list[int]] = None, use_hf_reference: bool = True, + device: str = "cpu", ) -> float: """Estimate peak memory needed for benchmark suite. @@ -420,8 +425,12 @@ def estimate_benchmark_memory_gb( bpp = bytes_per_param.get(dtype, 4) model_size_gb = n_params * bpp / (1024**3) - # GPT-2 scorer overhead (loaded during Phase 4) - gpt2_overhead_gb = 0.5 + # Phase-4 judge overhead: measured 2.33 GB RSS loading Qwen2.5-0.5B fp32 + # on CPU (494M params). Kept slightly above the measurement; over-counting + # is the safe direction. + # The CPU-pinned judge never occupies accelerator memory; charging it to + # a cuda budget produces spurious VRAM skips. + judge_overhead_gb = 2.5 if device == "cpu" else 0.0 # Activation/framework overhead as a fraction of model size overhead_fraction = 0.2 @@ -441,8 +450,8 @@ def estimate_benchmark_memory_gb( # Bridge + HookedTransformer = 2 copies phase_peaks.append(model_size_gb * 2.0 * (1 + overhead_fraction)) elif p == 4: - # Bridge + GPT-2 scorer - phase_peaks.append(model_size_gb * (1 + overhead_fraction) + gpt2_overhead_gb) + # Bridge + judge + phase_peaks.append(model_size_gb * (1 + overhead_fraction) + judge_overhead_gb) return max(phase_peaks) if phase_peaks else model_size_gb @@ -595,6 +604,18 @@ def _extract_phase_scores(results: list) -> dict[int, Optional[float]]: return scores +def _extract_prompt_profile(results: list) -> Optional[str]: + """Effective Phase-4 prompt profile from the benchmark details, or None + when no Phase-4 result exists. The default "continuation" is reported so + the registry write can clear a stale non-default key.""" + for result in results: + if result.phase == 4 and result.details: + profile = result.details.get("prompt_profile") + if isinstance(profile, str): + return profile + return None + + # Per-phase minimum score thresholds (0-100). # Phase 1: Core correctness (bridge vs HF) — must pass everything. # Phase 2: Hook/cache/gradient tests — most should pass. @@ -604,7 +625,9 @@ def _extract_phase_scores(results: list) -> dict[int, Optional[float]]: 1: 100.0, 2: 75.0, 3: 75.0, - 4: 50.0, + # Phase 4 floor == the benchmark pass line; a gap between them lets a + # failing score carry a clean "completed" note. + 4: p4_pass_threshold(), 7: 75.0, 8: 75.0, 9: 75.0, @@ -760,15 +783,89 @@ def _build_verified_note( else: issue_parts.append(f"P{phase}={score}%") + p4_uncovered = next( + ( + r.message + for r in all_results + if r.phase == 4 + and r.severity == BenchmarkSeverity.SKIPPED + and r.message.startswith("P4 skipped:") + ), + None, + ) + suffix = "" + if p4_uncovered: + # Keep the gap visible in the registry until prompt coverage is added. + reason = p4_uncovered.split("—")[0].replace("P4 skipped:", "").strip() + suffix = f"; P4 skipped (uncovered: {reason} — file a coverage issue)" + if issue_parts and low_text_quality: return ( f"Full verification completed with issues, low text quality: {'; '.join(issue_parts)}" + + suffix ) if issue_parts: - return f"Full verification completed with issues: {'; '.join(issue_parts)}" + return f"Full verification completed with issues: {'; '.join(issue_parts)}" + suffix if low_text_quality: - return "Full verification completed with issues, low text quality" - return "Full verification completed" + return "Full verification completed with issues, low text quality" + suffix + return "Full verification completed" + suffix + + +def _preserved_issue_suffix(model_id: str, eff_phases) -> str: + """Sub-100 scores from phases not re-run this pass stay visible in the + note; a partial pass must not overwrite tracked residue.""" + from transformer_lens.tools.model_registry.registry_io import ( + load_supported_models_raw, + ) + + try: + entry = next( + ( + m + for m in load_supported_models_raw().get("models", []) + if m.get("model_id") == model_id + ), + None, + ) + except OSError: + return "" + if entry is None: + return "" + residue = [] + for phase in (2, 3, 7, 8, 9): + if phase in (eff_phases or []): + continue + score = entry.get(f"phase{phase}_score") + if score is not None and score < 100.0: + residue.append(f"P{phase}={score}%") + if not residue: + return "" + return f" (prior issues retained: {', '.join(residue)})" + + +def _p1_only_core_note(p4_score, all_results: list) -> str: + """Note for a core run where P1 passed but P4 did not contribute a pass. + + A skipped P4 is a coverage gap, not a quality failure — the stale + (possibly old-scale) score must not be relabeled "poor".""" + from transformer_lens.benchmarks.utils import BenchmarkSeverity + + p4_skip_msg = next( + ( + r.message + for r in all_results + if r.phase == 4 + and r.severity == BenchmarkSeverity.SKIPPED + and r.message.startswith("P4 skipped:") + ), + None, + ) + if p4_skip_msg is not None: + reason = p4_skip_msg.split("—")[0].replace("P4 skipped:", "").strip() + return f"Core verification passed; P4 skipped ({reason})" + if p4_score is None: + return "Core verification passed, but text quality benchmark errored. Needs review" + return f"Core verification passed, but text quality poor (P4={p4_score}). Needs review" def _clear_hf_cache(quiet: bool = False) -> None: @@ -779,8 +876,16 @@ def _clear_hf_cache(quiet: bool = False) -> None: if not cache_dir.exists(): return + from transformer_lens.benchmarks.text_quality import JUDGE_MODEL_ID + + # The pinned Phase-4 judge is needed by every run; deleting it here would + # force a re-download per family. + judge_dir = "models--" + JUDGE_MODEL_ID.replace("/", "--") + freed = 0 for blobs_dir in cache_dir.glob("models--*/blobs"): + if blobs_dir.parent.name == judge_dir: + continue for blob in blobs_dir.iterdir(): try: size = blob.stat().st_size @@ -867,21 +972,25 @@ def verify_models( # phases stays None = full verification for the model. - # Pre-load the GPT-2 scoring model for Phase 4 so it persists across all - # models in the batch instead of being loaded and destroyed for each one. - _scoring_model = None - _scoring_tokenizer = None + # Pre-load the Phase-4 judge so it persists across all models in the batch + # instead of being loaded and destroyed for each one. + _judge_model = None + _judge_tokenizer = None if phases is None or 4 in phases: try: - from transformer_lens.benchmarks.text_quality import _load_scoring_model + from transformer_lens.benchmarks.text_quality import ( + JUDGE_MODEL_ID, + JUDGE_REVISION, + load_judge, + ) - _scoring_model, _scoring_tokenizer = _load_scoring_model("gpt2", device) + _judge_model, _judge_tokenizer = load_judge() if not quiet: - print("Pre-loaded GPT-2 scoring model for Phase 4") + print(f"Pre-loaded Phase 4 judge {JUDGE_MODEL_ID}@{JUDGE_REVISION[:8]}") except Exception as e: if not quiet: - print(f"Warning: Could not pre-load GPT-2 scorer: {e}") - print(" Phase 4 will load its own scorer per model.") + print(f"Warning: Could not pre-load Phase 4 judge: {e}") + print(" Phase 4 will load its own judge per model.") total = len(candidates) for i, candidate in enumerate(candidates, 1): @@ -953,7 +1062,7 @@ def verify_models( # Step 2: Check memory estimated_mem = estimate_benchmark_memory_gb( - n_params, dtype, phases=phases_to_run, use_hf_reference=use_hf_reference + n_params, dtype, phases=phases_to_run, use_hf_reference=use_hf_reference, device=device ) candidate.estimated_memory_gb = estimated_mem if not quiet: @@ -985,7 +1094,14 @@ def verify_models( } torch_dtype = _dtype_map[dtype] + from transformer_lens.benchmarks.text_quality_profiles import resolve_profile + from transformer_lens.tools.model_registry.registry_io import ( + registry_prompt_profile, + ) + + resolved_profile = str(resolve_profile(model_id, arch, registry_prompt_profile(model_id))) if not quiet: + print(f" Prompt profile: {resolved_profile}") print(f" Running phases {phases} in a single benchmark call...") try: all_results = run_benchmark_suite( @@ -997,8 +1113,9 @@ def verify_models( verbose=not quiet, phases=phases_to_run, trust_remote_code=needs_remote_code, - scoring_model=_scoring_model, - scoring_tokenizer=_scoring_tokenizer, + judge_model=_judge_model, + judge_tokenizer=_judge_tokenizer, + prompt_profile=resolved_profile, ) except Exception as e: error_msg = str(e) @@ -1095,7 +1212,9 @@ def verify_models( if p1_pass and p4_pass and p7_pass and p8_pass: partial_status = STATUS_VERIFIED - partial_note = "Core verification completed" + partial_note = "Core verification completed" + _preserved_issue_suffix( + model_id, eff_phases + ) elif p1_pass and p4_pass and not p7_pass: p7_score = filtered_scores.get(7) if p7_score is None: @@ -1112,9 +1231,7 @@ def verify_models( ) elif p1_pass: partial_status = STATUS_VERIFIED - partial_note = ( - "Core verification passed, but text quality poor. Needs review" - ) + partial_note = _p1_only_core_note(p4, all_results) else: # P1 failed — build a descriptive failure note partial_status = STATUS_FAILED @@ -1151,6 +1268,7 @@ def verify_models( status=partial_status, phase_scores=filtered_scores, note=partial_note, + prompt_profile=_extract_prompt_profile(all_results), ) # A provisional run was not numerically verified; do not write a # verification-history record (VerificationHistory.is_verified() @@ -1161,6 +1279,8 @@ def verify_models( arch, notes=partial_note, sanitize_fn=_sanitize_note, + prompt_profile=_extract_prompt_profile(all_results), + p4_scoring_version=(P4_SCORING_VERSION if 4 in filtered_scores else None), ) if partial_status == STATUS_FAILED: progress.failed.append(model_id) @@ -1205,6 +1325,7 @@ def verify_models( written_status, phase_scores=phase_scores, note=note, + prompt_profile=_extract_prompt_profile(all_results), ) # Provisional runs are not numerically verified — no history record # (is_verified() would otherwise report them as verified). @@ -1213,6 +1334,8 @@ def verify_models( model_id, arch, notes=note, + prompt_profile=_extract_prompt_profile(all_results), + p4_scoring_version=(P4_SCORING_VERSION if 4 in phase_scores else None), ) if is_provisional: progress.provisional.append(model_id) @@ -1235,12 +1358,15 @@ def verify_models( note=note, phase_scores=phase_scores, sanitize_fn=_sanitize_note, + prompt_profile=_extract_prompt_profile(all_results), ) add_verification_record( model_id, arch, notes=note, sanitize_fn=_sanitize_note, + prompt_profile=_extract_prompt_profile(all_results), + p4_scoring_version=(P4_SCORING_VERSION if 4 in phase_scores else None), ) progress.failed.append(model_id) @@ -1275,9 +1401,9 @@ def verify_models( _save_checkpoint(progress) # Clean up pre-loaded scoring model - if _scoring_model is not None: - del _scoring_model - del _scoring_tokenizer + if _judge_model is not None: + del _judge_model + del _judge_tokenizer gc.collect() return progress @@ -1289,6 +1415,7 @@ def _print_dry_run( max_memory_gb: float, phases: Optional[list[int]] = None, use_hf_reference: bool = True, + device: str = "cpu", ) -> None: """Print what would be tested in a dry run.""" print(f"\nDry run: {len(candidates)} models would be tested") @@ -1314,7 +1441,11 @@ def _print_dry_run( try: n_params = estimate_model_params(c.model_id) mem = estimate_benchmark_memory_gb( - n_params, dtype, phases=phases_to_run, use_hf_reference=use_hf_reference + n_params, + dtype, + phases=phases_to_run, + use_hf_reference=use_hf_reference, + device=device, ) status = "OK" if mem <= max_memory_gb else "SKIP (too large)" if mem > max_memory_gb: @@ -1562,6 +1693,7 @@ def main() -> None: max_memory_gb, phases=args.phases, use_hf_reference=not args.no_hf_reference, + device=args.device, ) return diff --git a/transformer_lens/train.py b/transformer_lens/train.py index db6f5d7697..13964e0923 100644 --- a/transformer_lens/train.py +++ b/transformer_lens/train.py @@ -4,6 +4,7 @@ modeling tasks. """ +import dataclasses from dataclasses import dataclass from typing import Optional, Union @@ -75,6 +76,10 @@ def train( The trained model """ + # Work on a copy: mutating the caller's config (wandb_project_name/device + # defaults below) was a silent side effect the caller never asked for. + config = dataclasses.replace(config) + torch.manual_seed(config.seed) model.train() diff --git a/transformer_lens/utilities/architectures.py b/transformer_lens/utilities/architectures.py index cf9558ae52..a1339a82a2 100644 --- a/transformer_lens/utilities/architectures.py +++ b/transformer_lens/utilities/architectures.py @@ -25,6 +25,14 @@ "SwitchTransformersForConditionalGeneration", } +# Post-norm decoders: ln1/ln2 normalize each sublayer's OUTPUT before the residual +# add, so LN folding and writing-weight centering (which assume the gain sits on a +# sublayer's INPUT) are not valid algebra for them. +POST_NORM_ARCHITECTURES: set[str] = { + "Olmo2ForCausalLM", + "Olmo3ForCausalLM", +} + # Masked language models (BERT-style, no text generation) MASKED_LM_ARCHITECTURES: set[str] = { "BertForMaskedLM", diff --git a/transformer_lens/utilities/lm_utils.py b/transformer_lens/utilities/lm_utils.py index a3d4f79326..79d47c975e 100644 --- a/transformer_lens/utilities/lm_utils.py +++ b/transformer_lens/utilities/lm_utils.py @@ -9,22 +9,24 @@ import torch import torch.nn.functional as F -from jaxtyping import Float, Int +from jaxtyping import Bool, Float, Int def lm_cross_entropy_loss( logits: Float[torch.Tensor, "batch pos d_vocab"], tokens: Int[torch.Tensor, "batch pos"], - attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, + attention_mask: Optional[ + Union[Bool[torch.Tensor, "batch pos"], Int[torch.Tensor, "batch pos"]] + ] = None, per_token: bool = False, -) -> Union[Float[torch.Tensor, ""], Float[torch.Tensor, "batch pos"]]: +) -> Union[Float[torch.Tensor, ""], Float[torch.Tensor, "batch pos-1"]]: """Cross entropy loss for the language model, gives the loss for predicting the NEXT token. Args: logits (torch.Tensor): Logits. Shape [batch, pos, d_vocab] tokens (torch.Tensor[int64]): Input tokens. Shape [batch, pos] - attention_mask (torch.Tensor[int64], optional): Attention mask. Shape [batch, pos]. Used to - mask out padding tokens. Defaults to None. + attention_mask (torch.Tensor[int64 or bool], optional): Attention mask. Shape [batch, pos]. + Used to mask out padding tokens. Defaults to None. per_token (bool, optional): Whether to return the log probs predicted for the correct token, or the loss (ie mean of the predicted log probs). Note that the returned array has shape [batch, seq-1] as we cannot predict the first token (alternately, we ignore the final logit). Defaults to False. """ log_probs = F.log_softmax(logits, dim=-1) @@ -34,10 +36,14 @@ def lm_cross_entropy_loss( predicted_log_probs = log_probs[..., :-1, :].gather(dim=-1, index=tokens[..., 1:, None])[..., 0] if attention_mask is not None: + assert attention_mask.shape == tokens.shape, ( + "attention_mask must have the same shape as tokens, " + f"got {tuple(attention_mask.shape)} and {tuple(tokens.shape)}" + ) # Ignore token positions which are masked out or where the next token is masked out # (generally padding tokens) next_token_mask = torch.logical_and(attention_mask[:, :-1], attention_mask[:, 1:]) - predicted_log_probs *= next_token_mask + predicted_log_probs = predicted_log_probs.masked_fill(~next_token_mask, 0.0) n_tokens = next_token_mask.sum().item() else: n_tokens = predicted_log_probs.numel() @@ -51,7 +57,7 @@ def lm_accuracy( logits: Float[torch.Tensor, "batch pos d_vocab"], tokens: Int[torch.Tensor, "batch pos"], per_token: bool = False, -) -> Union[Float[torch.Tensor, ""], Float[torch.Tensor, "batch pos"]]: +) -> Union[Float[torch.Tensor, ""], Bool[torch.Tensor, "batch pos-1"]]: """Cross-Entropy Accuracy for Language Modelling. We measure the accuracy on the logits for predicting the NEXT token. If per_token is True, returns the boolean for top 1 accuracy for each token in the batch. Note that this has size [batch, seq_len-1], as we cannot predict the first token. diff --git a/transformer_lens/utilities/multi_gpu.py b/transformer_lens/utilities/multi_gpu.py index e29023aedf..a4915088e6 100644 --- a/transformer_lens/utilities/multi_gpu.py +++ b/transformer_lens/utilities/multi_gpu.py @@ -249,7 +249,11 @@ def is_mixed_cpu_gpu(values: Any) -> bool: def cast_floating_params_to_dtype(model: nn.Module, dtype: torch.dtype) -> None: - """Cast materialized floating parameters while preserving Accelerate offload hooks.""" + """Cast materialized floating parameters while preserving Accelerate offload hooks. + + Skips one-byte floats (FP8 dtypes like float8_e8m0fnu) which are quantizer-owned + scale parameters — casting them corrupts the quantization format. + """ from accelerate.utils import align_module_device for module in model.modules(): @@ -259,9 +263,28 @@ def cast_floating_params_to_dtype(model: nn.Module, dtype: torch.dtype) -> None: continue if param.device.type == "meta": continue + # Skip one-byte floats (FP8 scale tensors): they are quantizer-owned + # and casting them breaks the weight/scale pair relationship. + if param.dtype.itemsize < 2: + continue param.data = param.data.to(dtype=dtype) +def maybe_cast_floating_params(model: nn.Module, dtype: torch.dtype) -> None: + """Cast floating params to dtype, skipping models with active quantization. + + When a model has an active quantization_config, the quantizer owns specific + dtypes (e.g., FP8 scales) that must not be overwritten. This helper wraps + the cast with that check. + + See: https://github.com/TransformerLensOrg/TransformerLens/issues/1713 + """ + from transformer_lens.utilities.quantization import quantization_method + + if quantization_method(getattr(model, "config", None)) is None: + cast_floating_params_to_dtype(model, dtype) + + def find_embedding_device(hf_model: Any) -> Optional[torch.device]: """Return the device that input tokens should be placed on for a dispatched HF model. diff --git a/transformer_lens/utilities/tokenize_utils.py b/transformer_lens/utilities/tokenize_utils.py index beedfd7b62..d5310c7649 100644 --- a/transformer_lens/utilities/tokenize_utils.py +++ b/transformer_lens/utilities/tokenize_utils.py @@ -7,7 +7,7 @@ import os from copy import deepcopy -from typing import Any +from typing import Any, Optional import einops import numpy as np @@ -182,18 +182,27 @@ def get_tokenizer_with_bos(tokenizer: PreTrainedTokenizerBase) -> PreTrainedToke def get_input_with_manually_prepended_bos( - bos_token: str, input: str | list[str] + bos_token: Optional[str], input: str | list[str] ) -> str | list[str]: """ Manually prepends the bos token to the input. Args: - bos_token (str): The BOS token to prepend. + bos_token (Optional[str]): The BOS token to prepend, or None for a tokenizer + that has none (e.g. BERT, T5). input (str | list[str]): The input to prepend the bos token to. Returns: - str | list[str]: The input with the bos token manually prepended. + str | list[str]: The input with the bos token manually prepended, or unchanged + when there is no BOS token to prepend. """ + if bos_token is None: + # Nothing to prepend. Callers reach this when prepend_bos is asked for and + # cfg.tokenizer_prepends_bos is False — correctly so for a BOS-less tokenizer, + # since detect_tokenizer_bos_eos() requires a bos_token_id. Concatenating + # would raise a TypeError naming neither the tokenizer nor the flag. + return input + if isinstance(input, str): input = bos_token + input else: @@ -202,7 +211,9 @@ def get_input_with_manually_prepended_bos( def get_tokens_with_bos_removed( - tokenizer: PreTrainedTokenizerBase, tokens: torch.Tensor + tokenizer: PreTrainedTokenizerBase, + tokens: torch.Tensor, + padding_side: str | None = None, ) -> torch.Tensor: """ Removes the bos token from the beginning of each sequence in `tokens`. @@ -211,11 +222,23 @@ def get_tokens_with_bos_removed( Args: tokenizer (PreTrainedTokenizerBase): The tokenizer used to tokenize the input. tokens (torch.Tensor): The tokenized input. + padding_side: The side used to pad ``tokens``. Defaults to the tokenizer setting. Returns: torch.Tensor: The tokenized input with the bos token removed. """ - if tokenizer.padding_side == "right": + if tokenizer.bos_token_id is None: + # Nothing to remove (#1628). Callers reach this when cfg.tokenizer_prepends_bos + # says the tokenizer prepends a BOS but the tokenizer has none — a stale + # flag, since detect_tokenizer_bos_eos() requires a bos_token_id. Trusting + # it here would drop a real first token under right padding ([CLS] for a + # BERT tokenizer), and compare tokens against None under left padding. + return tokens + + if padding_side is None: + padding_side = tokenizer.padding_side + + if padding_side == "right": return tokens[..., 1:] else: @@ -234,7 +257,10 @@ def get_tokens_with_bos_removed( def get_attention_mask( - tokenizer: PreTrainedTokenizerBase, tokens: torch.Tensor, prepend_bos: bool + tokenizer: PreTrainedTokenizerBase, + tokens: torch.Tensor, + prepend_bos: bool, + padding_side: str | None = None, ) -> torch.Tensor: """ Computes the attention mask for the tokenized input. @@ -246,6 +272,7 @@ def get_attention_mask( tokenizer (PreTrainedTokenizerBase): The tokenizer used for tokenization. tokens (torch.Tensor): The tokenized input. prepend_bos (bool): If True, a BOS token is prepended to the input. + padding_side: The side used to pad ``tokens``. Defaults to the tokenizer setting. Returns: torch.Tensor: The attention mask for the input. @@ -255,9 +282,11 @@ def get_attention_mask( attention_mask = torch.ones_like(tokens) if tokenizer is None: return attention_mask + if padding_side is None: + padding_side = tokenizer.padding_side is_not_pad_token = tokens.ne(tokenizer.pad_token_id) - if tokenizer.padding_side == "right": + if padding_side == "right": # Zero-out the rightmost trailing pad tokens is_trailing_pad = get_cumsum_along_dim(is_not_pad_token, -1, reverse=True) == 0 attention_mask[is_trailing_pad] = 0 diff --git a/transformer_lens/weight_processing.py b/transformer_lens/weight_processing.py index e4b8ccabb9..13c07760da 100644 --- a/transformer_lens/weight_processing.py +++ b/transformer_lens/weight_processing.py @@ -16,6 +16,7 @@ from transformer_lens.FactoredMatrix import FactoredMatrix from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter from transformer_lens.utilities import filter_dict_by_prefix +from transformer_lens.utilities.architectures import POST_NORM_ARCHITECTURES class ProcessWeights: @@ -1130,9 +1131,11 @@ def center_writing_weights( Returns: Dict[str, torch.Tensor]: Modified state dict with centered writing weights. """ - # Skip centering for Olmo2 models - input of attn of 1st layer is not normed - if getattr(cfg, "original_architecture", None) == "Olmo2ForCausalLM": - print("Not centering embedding weights for Olmo2ForCausalLM") + # Post-norm models leave the first attention's input un-normed, so centering + # the embedding would shift a residual stream nothing re-normalizes. + architecture = getattr(cfg, "original_architecture", None) + if architecture in POST_NORM_ARCHITECTURES: + print(f"Not centering embedding weights for {architecture}") else: # Make a deep copy to avoid modifying the original embed_W_E_key = ProcessWeights._get_param_key("embed.W_E", adapter)