Calibration script fixes for issue #39 validation on constrained hardware

scripts/calibrate_advisor.py

@@ -52,6 +52,9 @@
 logger = logging.getLogger("calibrate_advisor")
 
 _BYTES_PER_GB = 1024**3
+# Anything above this still allocated after a preset finishes means references
+# outlived the run and the next preset's measurement can't be trusted.
+_LEAK_WARN_GB = 0.25
 
 
 @dataclass
@@ -63,6 +66,18 @@ class CalibrationRow:
     actual: dict[str, float | None] = field(default_factory=dict)
     findings: int = 0
     findings_over: int = 0
+    # Top-line advisor verdict. ``headroom`` is the worst severity across all
+    # findings ("ample" / "tight" / "over"); ``is_feasible`` is ``headroom !=
+    # over``. Both live on ``PreflightReport`` but were previously dropped on
+    # the floor here, so a calibration JSON could not answer the one question
+    # the advisor exists to answer. ``severity_by_metric`` keeps the per-metric
+    # breakdown (vram / ram / disk / time) so a RED can be attributed.
+    headroom: str | None = None
+    is_feasible: bool | None = None
+    severity_by_metric: dict[str, str] = field(default_factory=dict)
+    # Resolved model name per driver, e.g. {"scoring/bert": "microsoft/deberta-v3-large"}.
+    # Recorded so a local preset swap can never masquerade as "transformers-heavy".
+    models: dict[str, str] = field(default_factory=dict)
     # Per-module records from _ModuleTracker: [{module, num, config, duration_s, peak_vram_gb?}, ...]
     modules: list[dict[str, Any]] = field(default_factory=list)
     cache_policy: str = "unknown"  # "cold" (embeddings cache cleared) | "warm" (kept as-is)
@@ -816,6 +831,15 @@ def _calibrate_one(
     }
     row.findings = len(report.findings)
     row.findings_over = sum(1 for f in report.findings if f.severity.value == "over")
+    row.headroom = report.headroom.value
+    row.is_feasible = report.is_feasible
+    # Last writer wins per metric; the resource phase emits at most one finding
+    # per metric so there is nothing to collapse in practice.
+    row.severity_by_metric = {f.metric: f.severity.value for f in report.findings if f.metric}
+    for driver in report.resource.drivers:
+        model = driver.get("model")
+        if model:
+            row.models[f"{driver.get('node_type', '?')}/{driver.get('module', '?')}"] = str(model)
     row.low_confidence = report.low_confidence
     if report.low_confidence:
         row.notes.append("low-confidence (heuristic HF metadata fallback in use)")
@@ -849,7 +873,22 @@ def _calibrate_one(
                 f"cli-smoke VERDICT MISMATCH: cli.is_feasible={cli_feasible} vs direct={report.is_feasible}"
             )
         elif divergence:
-            row.notes.append(f"cli-smoke numeric drift on {sorted(divergence)} (see cli_smoke.divergence)")
+            # ``autointent-advisor inspect`` has no n_trials flag, so under
+            # --max-trials the CLI necessarily costs the preset's bundled
+            # n_trials while the direct path costs the override. That is an
+            # apples-to-oranges comparison, not a wrapper regression — the
+            # historical "the two paths differ ~10x" reading of this field was
+            # this artifact. Only time_hours scales with n_trials, so a drift
+            # confined to that key under an override is expected.
+            expected_trials_artifact = max_trials is not None and set(divergence) == {"time_hours"}
+            smoke["divergence_expected"] = expected_trials_artifact
+            if expected_trials_artifact:
+                row.notes.append(
+                    f"cli-smoke time differs (cli n_trials={_preset_n_trials(raw_cfg)} vs "
+                    f"--max-trials {max_trials}); expected, not a wrapper regression"
+                )
+            else:
+                row.notes.append(f"cli-smoke numeric drift on {sorted(divergence)} (see cli_smoke.divergence)")
     row.cli_smoke = smoke
 
     if skip_fit:
@@ -978,6 +1017,12 @@ def _print_summary(rows: list[CalibrationRow]) -> None:
             print(f"    {marker} {row.error}")
         if row.low_confidence:
             print(f"    ! LOW-CONFIDENCE — advisor used heuristic HF metadata (exclude from prediction-accuracy stats)")
+        if row.headroom is not None:
+            verdict = "FEASIBLE" if row.is_feasible else "INFEASIBLE"
+            by_metric = " ".join(f"{m}={s}" for m, s in sorted(row.severity_by_metric.items()))
+            print(f"    · verdict={verdict} headroom={row.headroom} over={row.findings_over}  [{by_metric}]")
+        if row.models:
+            print(f"    · models: {', '.join(f'{k}={v}' for k, v in sorted(row.models.items()))}")
         print(f"    · cache-policy={row.cache_policy}")
         role_totals = _sum_time_by_role(row.modules)
         if role_totals:
@@ -1120,6 +1165,37 @@ def _load_dataset(dataset_arg: str, parser: argparse.ArgumentParser) -> tuple[Da
     return dataset, f"hub:{dataset_arg}"
 
 
+def _preset_n_trials(raw_cfg: dict[str, Any]) -> int | None:
+    """``hpo_config.n_trials`` as written in the preset, before any override."""
+    hpo = raw_cfg.get("hpo_config")
+    return hpo.get("n_trials") if isinstance(hpo, dict) else None
+
+
+def _release_accelerator_memory() -> float:
+    """Drop cached accelerator memory between presets; return GB still allocated.
+
+    Without this the sweep is not measuring what it thinks it is on a small
+    GPU: a preset that OOMs leaves its model, optimizer state and HPO trial
+    objects alive, so the *next* preset starts with several GB already gone and
+    OOMs too — an AMPLE preset then gets recorded as a failure it would never
+    hit on its own. A non-zero return value means references survived the
+    collection and the remaining presets in this process are suspect.
+    """
+    import gc
+
+    gc.collect()
+    try:
+        import torch
+    except ImportError:
+        return 0.0
+    if not torch.cuda.is_available():
+        return 0.0
+    torch.cuda.empty_cache()
+    still_allocated = torch.cuda.memory_allocated() / _BYTES_PER_GB
+    torch.cuda.reset_peak_memory_stats()
+    return still_allocated
+
+
 def _subsample_per_class(dataset: Dataset, cap: int) -> Dataset:
     """Cap each class in the train split to at most ``cap`` samples (first-N slice).
 
@@ -1278,6 +1354,18 @@ def _write_payload() -> None:
                 )
                 row.repeat_idx = repeat_idx
                 row.notes.insert(0, f"dataset={dataset_source}")
+                leaked_gb = _release_accelerator_memory()
+                if leaked_gb > _LEAK_WARN_GB:
+                    row.notes.append(
+                        f"accelerator memory still held after cleanup: {leaked_gb:.2f} GB — "
+                        f"later presets in this sweep may report a contaminated OOM"
+                    )
+                    logger.warning(
+                        "%s left %.2f GB of VRAM allocated after cleanup; "
+                        "run presets in separate processes for trustworthy numbers",
+                        preset,
+                        leaked_gb,
+                    )
                 rows.append(row)
                 _write_payload()
 

scripts/phase1b_metadata_counterfactual.py

@@ -0,0 +1,191 @@
+"""Counterfactual for issue #39: what would the advisor predict with CORRECT model metadata?
+
+Phase 1 showed all three ``transformers-*`` presets land on the low-confidence
+path — not because the box is offline, but because ``microsoft/deberta-v3-*``
+publishes no ``model.safetensors``, so ``HfApi().model_info().safetensors`` is
+``None`` and ``_hub_metadata`` substitutes a flat 350 M-param "large model"
+default for every deberta checkpoint.
+
+That matters for the verdict: a preset flagged OVER on a 350 M stand-in may be
+perfectly feasible at its real size. This script re-runs preflight with
+``resolve_model`` patched to report:
+
+  * ``total_params`` counted from the architecture (instantiated on the ``meta``
+    device, so nothing is downloaded or allocated), and
+  * ``total_file_bytes`` restricted to the files a torch load actually pulls
+    (excludes ``tf_model.h5`` and the discarded ELECTRA ``*.generator.bin``).
+
+and prints predicted-vs-corrected side by side.
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import json
+from pathlib import Path
+from typing import Any
+
+import torch
+from huggingface_hub import HfApi
+from transformers import AutoConfig, AutoModelForSequenceClassification
+
+from autointent import Dataset, setup_logging
+from autointent._advisor import (
+    HardwareProfile,
+    detect_hardware,
+    load_config,
+    run_preflight,
+    stats_from_dataset_obj,
+)
+from autointent._advisor import _hub as hub_mod
+from autointent._advisor._hub import ModelMeta
+
+setup_logging("ERROR", log_filename="phase1b.log")
+
+_BYTES_PER_GB = 1024**3
+# Files a torch/transformers load never reads. tf_model.h5 is the TensorFlow
+# mirror of the same weights; *.generator.bin is the ELECTRA-style generator
+# that deberta-v3 ships but discards at fine-tune time.
+_NON_TORCH_SUFFIXES = ("tf_model.h5", ".generator.bin", ".msgpack", ".onnx", ".h5")
+
+_PRESETS = ("transformers-heavy", "transformers-light", "transformers-no-hpo")
+
+
+def _true_param_count(model_name: str, n_labels: int) -> int:
+    """Exact parameter count without downloading weights (meta-device init)."""
+    cfg = AutoConfig.from_pretrained(model_name, num_labels=n_labels)
+    with torch.device("meta"):
+        model = AutoModelForSequenceClassification.from_config(cfg)
+    return sum(p.numel() for p in model.parameters())
+
+
+def _torch_only_bytes(model_name: str) -> int:
+    info = HfApi().model_info(model_name, files_metadata=True)
+    return sum(
+        s.size
+        for s in (info.siblings or [])
+        if s.size and not s.rfilename.endswith(_NON_TORCH_SUFFIXES)
+    )
+
+
+def _corrected_meta(model_name: str, n_labels: int) -> ModelMeta:
+    original = hub_mod.resolve_model(model_name)
+    params = _true_param_count(model_name, n_labels)
+    return ModelMeta(
+        name=model_name,
+        total_params=params,
+        weight_bytes_per_param=4,  # deberta-v3 ships fp32
+        total_file_bytes=_torch_only_bytes(model_name),
+        cached_locally=False,  # force the honest cold-disk prediction
+        confidence="hub",
+        hidden_size=original.hidden_size,
+        n_layers=original.n_layers,
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser("phase1b_metadata_counterfactual")
+    parser.add_argument("--dataset", default="DeepPavlov/banking77")
+    parser.add_argument("--output", default="calibration_runs/phase1b_counterfactual.json")
+    parser.add_argument(
+        "--assume-hardware",
+        metavar="VRAM_GB,RAM_GB",
+        help=(
+            "Skip GPU probing and use this profile instead (e.g. '5.67,15.03'). "
+            "Creating a CUDA context costs ~300 MB of VRAM, which is not affordable "
+            "while a real fit is running on the same 6 GB card — pass the numbers "
+            "detect_hardware() already reported instead."
+        ),
+    )
+    args = parser.parse_args()
+
+    if args.assume_hardware:
+        vram_s, ram_s = args.assume_hardware.split(",")
+        hardware = HardwareProfile(
+            accelerator="cuda",
+            device_name="assumed (no CUDA context created)",
+            vram_gb=float(vram_s),
+            ram_gb=float(ram_s),
+            free_disk_gb=100.0,
+            cpu_count=8,
+        )
+    else:
+        hardware = detect_hardware()
+    dataset = Dataset.from_hub(args.dataset)
+    stats = stats_from_dataset_obj(dataset)
+    print(
+        f"Hardware: {hardware.accelerator} {hardware.vram_gb:.2f} GB VRAM | "
+        f"dataset n_samples={stats.n_samples} n_classes={stats.n_classes}\n"
+    )
+
+    corrected_cache: dict[str, ModelMeta] = {}
+    results: list[dict[str, Any]] = []
+
+    for preset in _PRESETS:
+        cfg, _ = load_config(preset)
+        model_name = cfg["search_space"][0]["search_space"][0]["classification_model_config"][0]["model_name"]
+
+        baseline = run_preflight(copy.deepcopy(cfg), stats, hardware, preset_name=preset)
+        orig_meta = hub_mod.resolve_model(model_name)
+
+        if model_name not in corrected_cache:
+            corrected_cache[model_name] = _corrected_meta(model_name, stats.n_classes)
+        fixed = corrected_cache[model_name]
+
+        # Patch the memoized resolver for the duration of the second preflight.
+        real_resolver = hub_mod.resolve_model
+
+        def patched(name: str, _fixed: ModelMeta = fixed, _target: str = model_name) -> ModelMeta:
+            return _fixed if name == _target else real_resolver(name)
+
+        # _resource.py reaches the resolver as ``_hub.resolve_model(...)``
+        # (module-attribute access), so rebinding it here is enough.
+        hub_mod.resolve_model = patched  # type: ignore[assignment]
+        try:
+            corrected = run_preflight(copy.deepcopy(cfg), stats, hardware, preset_name=preset)
+        finally:
+            hub_mod.resolve_model = real_resolver  # type: ignore[assignment]
+
+        row = {
+            "preset": preset,
+            "model": model_name,
+            "params_assumed_M": round(orig_meta.total_params / 1e6, 1),
+            "params_true_M": round(fixed.total_params / 1e6, 1),
+            "disk_assumed_gb": round(orig_meta.disk_gb, 2),
+            "disk_torch_only_gb": round(fixed.disk_gb, 2),
+            "as_run": {
+                "vram_gb": round(baseline.resource.vram_gb, 2),
+                "headroom": baseline.headroom.value,
+                "is_feasible": baseline.is_feasible,
+                "low_confidence": baseline.low_confidence,
+                "disk_download_gb": round(baseline.resource.disk_download_gb, 2),
+            },
+            "corrected": {
+                "vram_gb": round(corrected.resource.vram_gb, 2),
+                "headroom": corrected.headroom.value,
+                "is_feasible": corrected.is_feasible,
+                "low_confidence": corrected.low_confidence,
+                "disk_download_gb": round(corrected.resource.disk_download_gb, 2),
+            },
+            "vram_budget_gb": round(hardware.vram_gb, 2),
+        }
+        results.append(row)
+        print(
+            f"{preset:22s} {model_name}\n"
+            f"   params : assumed {row['params_assumed_M']:>7.1f} M -> true {row['params_true_M']:>7.1f} M\n"
+            f"   VRAM   : as-run  {row['as_run']['vram_gb']:>7.2f} GB ({row['as_run']['headroom']})"
+            f" -> corrected {row['corrected']['vram_gb']:>7.2f} GB ({row['corrected']['headroom']})"
+            f"   [budget {row['vram_budget_gb']:.2f} GB]\n"
+            f"   disk   : as-run  {row['as_run']['disk_download_gb']:>7.2f} GB download"
+            f" -> corrected {row['corrected']['disk_download_gb']:>7.2f} GB\n"
+        )
+
+    out = Path(args.output)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps({"hardware": {"vram_gb": hardware.vram_gb}, "rows": results}, indent=2), encoding="utf-8")
+    print(f"Wrote {out}")
+
+
+if __name__ == "__main__":
+    main()