Release v3.8.0

.github/workflows/checks.yml

@@ -277,22 +277,28 @@ jobs:
   coverage-test:
     name: Full Code Coverage Test
     runs-on: ubuntu-latest
-    # Suite runs under pytest-xdist on a 2-vCPU runner (~45 min with swap);
+    # Suite runs under pytest-xdist on a 2-vCPU runner (~35 min with swap);
     # 75 catches a real hang well below the old serial 90.
     timeout-minutes: 75
     steps:
       - uses: actions/checkout@v4
       - name: Add swap space
         # The 2-vCPU runner has ~7GB RAM; two xdist workers each holding
         # torch + resident models overflow it. Swap absorbs the model-load
-        # spikes (mirrors the notebook-checks job).
+        # spikes. 8G left the suite's heavy tail one xdist scheduling roll
+        # from the runner killing the step at ~95% (identical SHAs pass and
+        # fail), so 16G — on the ~65G /mnt temp disk, keeping the OS disk
+        # free for the uv env and model caches.
         run: |
           sudo swapoff /swapfile 2>/dev/null || true
           sudo rm -f /swapfile
-          sudo fallocate -l 8G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
+          sudo swapoff /mnt/swapfile 2>/dev/null || true
+          sudo rm -f /mnt/swapfile
+          sudo fallocate -l 16G /mnt/swapfile
+          sudo chmod 600 /mnt/swapfile
+          sudo mkswap /mnt/swapfile
+          sudo swapon /mnt/swapfile
+          free -h && df -h / /mnt
       - name: Install uv
         uses: astral-sh/setup-uv@v7
         with:
@@ -358,6 +364,15 @@ jobs:
           # Worker count knob (this runner is 2-vCPU, so -n auto = 2). If swap
           # still can't hold peak memory, drop to "-n 1" or grow the swapfile.
           XDIST_ARGS: "-n auto --dist loadscope"
+      - name: Memory post-mortem
+        # The tail-of-suite kills leave no cause in the step log; dmesg names
+        # the killer (kernel OOM, systemd-oomd, or neither) for the next one.
+        if: failure()
+        run: |
+          sudo dmesg | tail -n 80 || true
+          free -h || true
+          swapon --show || true
+          df -h / /mnt || true
       - name: Build check
         run: uv build
       - name: Upload Coverage Report Artifact

.gitignore

@@ -29,3 +29,4 @@ docs/source/generated
 !.claude/commands/
 .adapter-progress.json
 transformer_lens/tools/model_registry/data/verification_checkpoint.json
+venv/

demos/BERT.ipynb

@@ -307,7 +307,7 @@
      "output_type": "stream",
      "text": [
       "Prompt: ['The [MASK] is bright today.', 'She [MASK] to the store.', 'The dog [MASK] the ball.']\n",
-      "Prediction: \"['Prediction 0: sun', 'Prediction 1: returned', 'Prediction 2: has']\"\n"
+      "Prediction: \"['Prediction 0: sun', 'Prediction 1: went', 'Prediction 2: caught']\"\n"
      ]
     }
    ],
@@ -499,7 +499,13 @@
     "\n",
     "inputs = tokenizer(sentence_a, sentence_b, return_tensors=\"pt\")\n",
     "device = next(nsp.parameters()).device\n",
-    "predictions = nsp(inputs[\"input_ids\"].to(device), return_type=\"predictions\")\n",
+    "# token_type_ids mark where sentence A ends and B begins — without them the NSP\n",
+    "# head sees one undifferentiated span and can return the wrong verdict.\n",
+    "predictions = nsp(\n",
+    "    inputs[\"input_ids\"].to(device),\n",
+    "    token_type_ids=inputs[\"token_type_ids\"].to(device),\n",
+    "    return_type=\"predictions\",\n",
+    ")\n",
     "\n",
     "print(f\"Sentence A: {sentence_a}\")\n",
     "print(f\"Sentence B: {sentence_b}\")\n",

demos/Grokking_Demo.ipynb

@@ -784,7 +784,9 @@
     "model_checkpoints = []\n",
     "checkpoint_epochs = []\n",
     "if TRAIN_MODEL:\n",
-    "    for epoch in tqdm.tqdm(range(num_epochs)):\n",
+    "# mininterval throttles tqdm below Jupyter's IOPub rate limit while keeping a\n",
+    "# visible progress bar.\n",
+    "    for epoch in tqdm.tqdm(range(num_epochs), mininterval=2):\n",
     "        train_logits = model(train_data)\n",
     "        train_loss = loss_fn(train_logits, train_labels)\n",
     "        train_loss.backward()\n",
@@ -1867,53 +1869,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 45,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "key_fourier_embed torch.Size([8, 128])\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div>                            <div id=\"d51ed6e5-a54c-4af4-a583-29f612438a5b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d51ed6e5-a54c-4af4-a583-29f612438a5b\")) {                    Plotly.newPlot(                        \"d51ed6e5-a54c-4af4-a583-29f612438a5b\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.0027102141,0.0011594994,0.0008332776,-0.0000450976,-0.00038484533,0.00034331827,-0.00013534419,-0.0006771815],[0.0011594994,0.0049333726,0.0010480853,0.00042903924,-0.0020712302,0.00036800973,-0.0007300802,-0.0010292965],[0.0008332776,0.0010480853,0.0058673383,-0.00017594105,-0.0011680584,-0.00018933232,-0.00034060967,-0.00046609537],[-0.0000450976,0.00042903924,-0.00017594105,0.00753589,-0.002141118,-0.0019649575,-0.00013577768,0.0003775559],[-0.00038484533,-0.0020712302,-0.0011680584,-0.002141118,0.01237848,0.0020916045,0.0013299129,0.0015913968],[0.00034331827,0.00036800973,-0.00018933232,-0.0019649575,0.0020916045,0.010890393,0.0002858049,0.00091429084],[-0.00013534419,-0.0007300802,-0.00034060967,-0.00013577768,0.0013299129,0.0002858049,0.0025899112,0.0012701292],[-0.0006771815,-0.0010292965,-0.00046609537,0.0003775559,0.0015913968,0.00091429084,0.0012701292,0.0025464832]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15,\"font\":{\"size\":20}},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15,\"font\":{\"size\":20}},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05,\"font\":{\"size\":30}},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"title\":{\"text\":\"y\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Dot Product of embedding of key Fourier Terms\"}},                        {\"responsive\": true}                    ).then(function(){\n",
-       "                            \n",
-       "var gd = document.getElementById('d51ed6e5-a54c-4af4-a583-29f612438a5b');\n",
-       "var x = new MutationObserver(function (mutations, observer) {{\n",
-       "        var display = window.getComputedStyle(gd).display;\n",
-       "        if (!display || display === 'none') {{\n",
-       "            console.log([gd, 'removed!']);\n",
-       "            Plotly.purge(gd);\n",
-       "            observer.disconnect();\n",
-       "        }}\n",
-       "}});\n",
-       "\n",
-       "// Listen for the removal of the full notebook cells\n",
-       "var notebookContainer = gd.closest('#notebook-container');\n",
-       "if (notebookContainer) {{\n",
-       "    x.observe(notebookContainer, {childList: true});\n",
-       "}}\n",
-       "\n",
-       "// Listen for the clearing of the current output cell\n",
-       "var outputEl = gd.closest('.output');\n",
-       "if (outputEl) {{\n",
-       "    x.observe(outputEl, {childList: true});\n",
-       "}}\n",
-       "\n",
-       "                        })                };                });            </script>        </div>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "key_freqs = [17, 25, 32, 47]\n",
-    "key_freq_indices = [33, 34, 49, 50, 63, 64, 93, 94]\n",
+    "# Derive the key frequencies from this run's embedding instead of hardcoding\n",
+    "# values from a past run — they vary with seed and training length.\n",
+    "fourier_norms = (fourier_basis @ W_E).norm(dim=-1)\n",
+    "key_freq_indices = [\n",
+    "    i for i, norm in enumerate(fourier_norms) if i > 0 and norm > fourier_norms.max() / 4\n",
+    "]\n",
+    "key_freqs = sorted({(i + 1) // 2 for i in key_freq_indices})\n",
+    "print(\"key_freqs\", key_freqs)\n",
     "fourier_embed = fourier_basis @ W_E\n",
     "key_fourier_embed = fourier_embed[key_freq_indices]\n",
     "print(\"key_fourier_embed\", key_fourier_embed.shape)\n",

demos/doc_sanitize.cfg

@@ -37,3 +37,7 @@ replace: \1
 [regex7]
 regex: [^\n]*DeprecationWarning:(?=\n\nHookedTransformer is deprecated and will be removed in 4\.0\. Use TransformerBridge\.boot_transformers\(\.\.\.\) instead, then call enable_compatibility_mode\(\) for HookedTransformer-equivalent numerics\.)
 replace: DeprecationWarning:
+
+[regex8]
+regex: /(?:var|tmp|private)/\S*
+replace: TMP-PATH

docs/source/content/contributing.md

@@ -328,7 +328,7 @@ set -a; source .env; set +a
 uv run python -m transformer_lens.tools.model_registry.verify_models --model <hf_repo>
 ```
 
-`verify_models` runs phases 1–4 (forward correctness vs HF, hook firing + gradients, weight processing, generation quality) and updates `data/supported_models.json` with the resulting status and per-phase scores. We recommend running `--dry-run` first to project memory and parameter count without loading the model, and verifying one model at a time — concurrent loads tend to OOM a single device.
+`verify_models` runs phases 1–4 (forward correctness vs HF, hook firing + gradients, weight processing, text-generation quality) and updates `data/supported_models.json` with the resulting status and per-phase scores. We recommend running `--dry-run` first to project memory and parameter count without loading the model, and verifying one model at a time — concurrent loads tend to OOM a single device.
 
 Running with `--no-hf-reference` skips the HuggingFace numerical comparison (Phase 1 becomes structural-only). A passing run is then recorded as **provisional** (status 4), which does *not* count as verified — re-run without the flag for a real HF-compared verification.
 
@@ -341,11 +341,11 @@ It's worth reading the per-phase scores in addition to the final status — the
 | 1 | 100% | — | Verification fails |
 | 2 | 75% | `logits_equivalence`, `loss_equivalence` | Verification fails |
 | 3 | 75% | `logits_equivalence`, `loss_equivalence` | Verification fails |
-| 4 | 50% | — | **Non-gating** — below 50% adds `"low text quality"` to the registry `note`; never fails verification. |
+| 4 | 54.5% (measured pass line, `p4_pass_threshold()`) | — | **Non-gating** — below the line adds a `"text quality poor (P4=…)"` note; never fails verification. |
 | 7 | 75% | `multimodal_forward` | Verification fails. A NULL score also fails. |
 | 8 | 75% | `audio_forward` | Verification fails. A NULL score also fails. |
 
-Phase 4 is intentionally lenient — it's a coherence metric, not a correctness check. A sub-100% Phase-4 score on a small parity-test model can still indicate a real adapter bug that the gates don't catch (missing `preprocess_weights` fold, wrong `default_prepend_bos`, and so on); the model can pass verification overall and still be worth a manual look.
+Phase 4 prompts each model with its resolved prompt profile (chat template, translation, code, own-language continuation, ...) and scores the generation against a known-good reference with one pinned multilingual judge, via the perplexity ratio `PPL(generated)/PPL(reference)`. It's intentionally lenient — a coherence metric, not a correctness check. A sub-100% Phase-4 score on a small parity-test model can still indicate a real adapter bug that the gates don't catch (missing `preprocess_weights` fold, wrong `default_prepend_bos`, and so on); the model can pass verification overall and still be worth a manual look.
 
 If verification fails by `~1e-3` or more against the HF reference, the bisection workflow lives at [Debugging Numerical Divergence](debugging_numerical_divergence.md).