From b38f3c3a8612e177e76fe94552180ef43555665c Mon Sep 17 00:00:00 2001 From: voorhs Date: Sun, 16 Aug 2026 17:16:34 +0300 Subject: [PATCH] Calibration script fixes for issue #39 validation on constrained hardware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes needed to produce the laptop-6GB calibration deliverables: calibrate_advisor.py: - Record is_feasible, headroom, severity_by_metric, and resolved model_name per driver on each CalibrationRow — without these a calibration JSON cannot answer the one question the advisor exists to answer, and a local preset swap can masquerade as 'transformers-heavy' - Add inter-preset GPU leak detection (_LEAK_WARN_GB) to surface leftover VRAM that corrupts the next preset's measurement - Fix cli_smoke divergence note: the ~10x drift was an apples-to-oranges n_trials artifact, not a wrapper regression run_calibration_banking77.sh: - uv run preset-discovery fix New helper scripts: - phase1b_metadata_counterfactual.py: re-predict with correct model metadata - phase3_reduce_to_fit.py: exercise the reduce-to-fit path on real hardware - render_issue39_tables.py: render definition-of-done tables from JSONs - run_phase2_isolated.sh: one preset per process to avoid GPU contamination Advisor source code is left untouched — this is validation, not fixes. --- scripts/calibrate_advisor.py | 90 ++++++- scripts/phase1b_metadata_counterfactual.py | 191 ++++++++++++++ scripts/phase3_reduce_to_fit.py | 284 +++++++++++++++++++++ scripts/render_issue39_tables.py | 116 +++++++++ scripts/run_calibration_banking77.sh | 4 +- scripts/run_phase2_isolated.sh | 38 +++ 6 files changed, 721 insertions(+), 2 deletions(-) create mode 100644 scripts/phase1b_metadata_counterfactual.py create mode 100644 scripts/phase3_reduce_to_fit.py create mode 100644 scripts/render_issue39_tables.py create mode 100755 scripts/run_phase2_isolated.sh diff --git a/scripts/calibrate_advisor.py b/scripts/calibrate_advisor.py index 8e7d54cf5..708c3c31a 100644 --- a/scripts/calibrate_advisor.py +++ b/scripts/calibrate_advisor.py @@ -52,6 +52,9 @@ logger = logging.getLogger("calibrate_advisor") _BYTES_PER_GB = 1024**3 +# Anything above this still allocated after a preset finishes means references +# outlived the run and the next preset's measurement can't be trusted. +_LEAK_WARN_GB = 0.25 @dataclass @@ -63,6 +66,18 @@ class CalibrationRow: actual: dict[str, float | None] = field(default_factory=dict) findings: int = 0 findings_over: int = 0 + # Top-line advisor verdict. ``headroom`` is the worst severity across all + # findings ("ample" / "tight" / "over"); ``is_feasible`` is ``headroom != + # over``. Both live on ``PreflightReport`` but were previously dropped on + # the floor here, so a calibration JSON could not answer the one question + # the advisor exists to answer. ``severity_by_metric`` keeps the per-metric + # breakdown (vram / ram / disk / time) so a RED can be attributed. + headroom: str | None = None + is_feasible: bool | None = None + severity_by_metric: dict[str, str] = field(default_factory=dict) + # Resolved model name per driver, e.g. {"scoring/bert": "microsoft/deberta-v3-large"}. + # Recorded so a local preset swap can never masquerade as "transformers-heavy". + models: dict[str, str] = field(default_factory=dict) # Per-module records from _ModuleTracker: [{module, num, config, duration_s, peak_vram_gb?}, ...] modules: list[dict[str, Any]] = field(default_factory=list) cache_policy: str = "unknown" # "cold" (embeddings cache cleared) | "warm" (kept as-is) @@ -816,6 +831,15 @@ def _calibrate_one( } row.findings = len(report.findings) row.findings_over = sum(1 for f in report.findings if f.severity.value == "over") + row.headroom = report.headroom.value + row.is_feasible = report.is_feasible + # Last writer wins per metric; the resource phase emits at most one finding + # per metric so there is nothing to collapse in practice. + row.severity_by_metric = {f.metric: f.severity.value for f in report.findings if f.metric} + for driver in report.resource.drivers: + model = driver.get("model") + if model: + row.models[f"{driver.get('node_type', '?')}/{driver.get('module', '?')}"] = str(model) row.low_confidence = report.low_confidence if report.low_confidence: row.notes.append("low-confidence (heuristic HF metadata fallback in use)") @@ -849,7 +873,22 @@ def _calibrate_one( f"cli-smoke VERDICT MISMATCH: cli.is_feasible={cli_feasible} vs direct={report.is_feasible}" ) elif divergence: - row.notes.append(f"cli-smoke numeric drift on {sorted(divergence)} (see cli_smoke.divergence)") + # ``autointent-advisor inspect`` has no n_trials flag, so under + # --max-trials the CLI necessarily costs the preset's bundled + # n_trials while the direct path costs the override. That is an + # apples-to-oranges comparison, not a wrapper regression — the + # historical "the two paths differ ~10x" reading of this field was + # this artifact. Only time_hours scales with n_trials, so a drift + # confined to that key under an override is expected. + expected_trials_artifact = max_trials is not None and set(divergence) == {"time_hours"} + smoke["divergence_expected"] = expected_trials_artifact + if expected_trials_artifact: + row.notes.append( + f"cli-smoke time differs (cli n_trials={_preset_n_trials(raw_cfg)} vs " + f"--max-trials {max_trials}); expected, not a wrapper regression" + ) + else: + row.notes.append(f"cli-smoke numeric drift on {sorted(divergence)} (see cli_smoke.divergence)") row.cli_smoke = smoke if skip_fit: @@ -978,6 +1017,12 @@ def _print_summary(rows: list[CalibrationRow]) -> None: print(f" {marker} {row.error}") if row.low_confidence: print(f" ! LOW-CONFIDENCE — advisor used heuristic HF metadata (exclude from prediction-accuracy stats)") + if row.headroom is not None: + verdict = "FEASIBLE" if row.is_feasible else "INFEASIBLE" + by_metric = " ".join(f"{m}={s}" for m, s in sorted(row.severity_by_metric.items())) + print(f" · verdict={verdict} headroom={row.headroom} over={row.findings_over} [{by_metric}]") + if row.models: + print(f" · models: {', '.join(f'{k}={v}' for k, v in sorted(row.models.items()))}") print(f" · cache-policy={row.cache_policy}") role_totals = _sum_time_by_role(row.modules) if role_totals: @@ -1120,6 +1165,37 @@ def _load_dataset(dataset_arg: str, parser: argparse.ArgumentParser) -> tuple[Da return dataset, f"hub:{dataset_arg}" +def _preset_n_trials(raw_cfg: dict[str, Any]) -> int | None: + """``hpo_config.n_trials`` as written in the preset, before any override.""" + hpo = raw_cfg.get("hpo_config") + return hpo.get("n_trials") if isinstance(hpo, dict) else None + + +def _release_accelerator_memory() -> float: + """Drop cached accelerator memory between presets; return GB still allocated. + + Without this the sweep is not measuring what it thinks it is on a small + GPU: a preset that OOMs leaves its model, optimizer state and HPO trial + objects alive, so the *next* preset starts with several GB already gone and + OOMs too — an AMPLE preset then gets recorded as a failure it would never + hit on its own. A non-zero return value means references survived the + collection and the remaining presets in this process are suspect. + """ + import gc + + gc.collect() + try: + import torch + except ImportError: + return 0.0 + if not torch.cuda.is_available(): + return 0.0 + torch.cuda.empty_cache() + still_allocated = torch.cuda.memory_allocated() / _BYTES_PER_GB + torch.cuda.reset_peak_memory_stats() + return still_allocated + + def _subsample_per_class(dataset: Dataset, cap: int) -> Dataset: """Cap each class in the train split to at most ``cap`` samples (first-N slice). @@ -1278,6 +1354,18 @@ def _write_payload() -> None: ) row.repeat_idx = repeat_idx row.notes.insert(0, f"dataset={dataset_source}") + leaked_gb = _release_accelerator_memory() + if leaked_gb > _LEAK_WARN_GB: + row.notes.append( + f"accelerator memory still held after cleanup: {leaked_gb:.2f} GB — " + f"later presets in this sweep may report a contaminated OOM" + ) + logger.warning( + "%s left %.2f GB of VRAM allocated after cleanup; " + "run presets in separate processes for trustworthy numbers", + preset, + leaked_gb, + ) rows.append(row) _write_payload() diff --git a/scripts/phase1b_metadata_counterfactual.py b/scripts/phase1b_metadata_counterfactual.py new file mode 100644 index 000000000..1b32a6028 --- /dev/null +++ b/scripts/phase1b_metadata_counterfactual.py @@ -0,0 +1,191 @@ +"""Counterfactual for issue #39: what would the advisor predict with CORRECT model metadata? + +Phase 1 showed all three ``transformers-*`` presets land on the low-confidence +path — not because the box is offline, but because ``microsoft/deberta-v3-*`` +publishes no ``model.safetensors``, so ``HfApi().model_info().safetensors`` is +``None`` and ``_hub_metadata`` substitutes a flat 350 M-param "large model" +default for every deberta checkpoint. + +That matters for the verdict: a preset flagged OVER on a 350 M stand-in may be +perfectly feasible at its real size. This script re-runs preflight with +``resolve_model`` patched to report: + + * ``total_params`` counted from the architecture (instantiated on the ``meta`` + device, so nothing is downloaded or allocated), and + * ``total_file_bytes`` restricted to the files a torch load actually pulls + (excludes ``tf_model.h5`` and the discarded ELECTRA ``*.generator.bin``). + +and prints predicted-vs-corrected side by side. +""" + +from __future__ import annotations + +import argparse +import copy +import json +from pathlib import Path +from typing import Any + +import torch +from huggingface_hub import HfApi +from transformers import AutoConfig, AutoModelForSequenceClassification + +from autointent import Dataset, setup_logging +from autointent._advisor import ( + HardwareProfile, + detect_hardware, + load_config, + run_preflight, + stats_from_dataset_obj, +) +from autointent._advisor import _hub as hub_mod +from autointent._advisor._hub import ModelMeta + +setup_logging("ERROR", log_filename="phase1b.log") + +_BYTES_PER_GB = 1024**3 +# Files a torch/transformers load never reads. tf_model.h5 is the TensorFlow +# mirror of the same weights; *.generator.bin is the ELECTRA-style generator +# that deberta-v3 ships but discards at fine-tune time. +_NON_TORCH_SUFFIXES = ("tf_model.h5", ".generator.bin", ".msgpack", ".onnx", ".h5") + +_PRESETS = ("transformers-heavy", "transformers-light", "transformers-no-hpo") + + +def _true_param_count(model_name: str, n_labels: int) -> int: + """Exact parameter count without downloading weights (meta-device init).""" + cfg = AutoConfig.from_pretrained(model_name, num_labels=n_labels) + with torch.device("meta"): + model = AutoModelForSequenceClassification.from_config(cfg) + return sum(p.numel() for p in model.parameters()) + + +def _torch_only_bytes(model_name: str) -> int: + info = HfApi().model_info(model_name, files_metadata=True) + return sum( + s.size + for s in (info.siblings or []) + if s.size and not s.rfilename.endswith(_NON_TORCH_SUFFIXES) + ) + + +def _corrected_meta(model_name: str, n_labels: int) -> ModelMeta: + original = hub_mod.resolve_model(model_name) + params = _true_param_count(model_name, n_labels) + return ModelMeta( + name=model_name, + total_params=params, + weight_bytes_per_param=4, # deberta-v3 ships fp32 + total_file_bytes=_torch_only_bytes(model_name), + cached_locally=False, # force the honest cold-disk prediction + confidence="hub", + hidden_size=original.hidden_size, + n_layers=original.n_layers, + ) + + +def main() -> None: + parser = argparse.ArgumentParser("phase1b_metadata_counterfactual") + parser.add_argument("--dataset", default="DeepPavlov/banking77") + parser.add_argument("--output", default="calibration_runs/phase1b_counterfactual.json") + parser.add_argument( + "--assume-hardware", + metavar="VRAM_GB,RAM_GB", + help=( + "Skip GPU probing and use this profile instead (e.g. '5.67,15.03'). " + "Creating a CUDA context costs ~300 MB of VRAM, which is not affordable " + "while a real fit is running on the same 6 GB card — pass the numbers " + "detect_hardware() already reported instead." + ), + ) + args = parser.parse_args() + + if args.assume_hardware: + vram_s, ram_s = args.assume_hardware.split(",") + hardware = HardwareProfile( + accelerator="cuda", + device_name="assumed (no CUDA context created)", + vram_gb=float(vram_s), + ram_gb=float(ram_s), + free_disk_gb=100.0, + cpu_count=8, + ) + else: + hardware = detect_hardware() + dataset = Dataset.from_hub(args.dataset) + stats = stats_from_dataset_obj(dataset) + print( + f"Hardware: {hardware.accelerator} {hardware.vram_gb:.2f} GB VRAM | " + f"dataset n_samples={stats.n_samples} n_classes={stats.n_classes}\n" + ) + + corrected_cache: dict[str, ModelMeta] = {} + results: list[dict[str, Any]] = [] + + for preset in _PRESETS: + cfg, _ = load_config(preset) + model_name = cfg["search_space"][0]["search_space"][0]["classification_model_config"][0]["model_name"] + + baseline = run_preflight(copy.deepcopy(cfg), stats, hardware, preset_name=preset) + orig_meta = hub_mod.resolve_model(model_name) + + if model_name not in corrected_cache: + corrected_cache[model_name] = _corrected_meta(model_name, stats.n_classes) + fixed = corrected_cache[model_name] + + # Patch the memoized resolver for the duration of the second preflight. + real_resolver = hub_mod.resolve_model + + def patched(name: str, _fixed: ModelMeta = fixed, _target: str = model_name) -> ModelMeta: + return _fixed if name == _target else real_resolver(name) + + # _resource.py reaches the resolver as ``_hub.resolve_model(...)`` + # (module-attribute access), so rebinding it here is enough. + hub_mod.resolve_model = patched # type: ignore[assignment] + try: + corrected = run_preflight(copy.deepcopy(cfg), stats, hardware, preset_name=preset) + finally: + hub_mod.resolve_model = real_resolver # type: ignore[assignment] + + row = { + "preset": preset, + "model": model_name, + "params_assumed_M": round(orig_meta.total_params / 1e6, 1), + "params_true_M": round(fixed.total_params / 1e6, 1), + "disk_assumed_gb": round(orig_meta.disk_gb, 2), + "disk_torch_only_gb": round(fixed.disk_gb, 2), + "as_run": { + "vram_gb": round(baseline.resource.vram_gb, 2), + "headroom": baseline.headroom.value, + "is_feasible": baseline.is_feasible, + "low_confidence": baseline.low_confidence, + "disk_download_gb": round(baseline.resource.disk_download_gb, 2), + }, + "corrected": { + "vram_gb": round(corrected.resource.vram_gb, 2), + "headroom": corrected.headroom.value, + "is_feasible": corrected.is_feasible, + "low_confidence": corrected.low_confidence, + "disk_download_gb": round(corrected.resource.disk_download_gb, 2), + }, + "vram_budget_gb": round(hardware.vram_gb, 2), + } + results.append(row) + print( + f"{preset:22s} {model_name}\n" + f" params : assumed {row['params_assumed_M']:>7.1f} M -> true {row['params_true_M']:>7.1f} M\n" + f" VRAM : as-run {row['as_run']['vram_gb']:>7.2f} GB ({row['as_run']['headroom']})" + f" -> corrected {row['corrected']['vram_gb']:>7.2f} GB ({row['corrected']['headroom']})" + f" [budget {row['vram_budget_gb']:.2f} GB]\n" + f" disk : as-run {row['as_run']['disk_download_gb']:>7.2f} GB download" + f" -> corrected {row['corrected']['disk_download_gb']:>7.2f} GB\n" + ) + + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps({"hardware": {"vram_gb": hardware.vram_gb}, "rows": results}, indent=2), encoding="utf-8") + print(f"Wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/phase3_reduce_to_fit.py b/scripts/phase3_reduce_to_fit.py new file mode 100644 index 000000000..b94a3f31e --- /dev/null +++ b/scripts/phase3_reduce_to_fit.py @@ -0,0 +1,284 @@ +"""Phase 3 of issue #39: exercise the reduce-to-fit path on real constrained hardware. + +The calibrator runs report-only (``preflight="off"``), so the strict gate and +``reduce_to_fit`` have never been driven against a box that actually can't fit +the preset. This script does three things on the live machine: + + A. ``Pipeline.fit(..., preflight="strict")`` on ``transformers-heavy`` — + must raise ``PreflightError`` *before* any CUDA allocation instead of + letting the run walk into an OOM. + B. ``reduce_to_fit`` on ``transformers-heavy`` — a single-scoring-module + preset, so the only reachable outcome is ``ReduceToFitError``. We check + the error is the explicit "everything was pruned" one and record whether + it points the user anywhere useful. + C. ``reduce_to_fit`` on a mixed infeasible search space (deberta-v3-large + + knn + linear) — the case where pruning *can* succeed. We then really fit + the pruned config to prove the survivor is runnable, not just feasible + on paper. + +Usage: + uv run --no-sync python scripts/phase3_reduce_to_fit.py \ + --dataset DeepPavlov/banking77 --subsample-per-class 30 \ + --output calibration_runs/phase3.json +""" + +from __future__ import annotations + +import argparse +import copy +import json +import logging +import time +import traceback +from pathlib import Path +from typing import Any + +import torch + +from autointent import Dataset, Pipeline, setup_logging +from autointent._advisor import ( + ReduceToFitError, + detect_hardware, + load_config, + reduce_to_fit, + run_preflight, + stats_from_dataset_obj, +) +from autointent._pipeline import PreflightError + +setup_logging("WARNING", log_filename="phase3.log") +logging.basicConfig(level=logging.WARNING) +logger = logging.getLogger("phase3") + +_BYTES_PER_GB = 1024**3 + + +def _subsample_per_class(dataset: Dataset, cap: int) -> Dataset: + """Deterministic first-N-per-class slice of the train split. + + Imported from the calibrator rather than reimplemented so Phase 3 sees + byte-identical dataset stats to the Phase 1 / Phase 2 runs. + """ + import sys + + sys.path.insert(0, str(Path(__file__).parent)) + from calibrate_advisor import _subsample_per_class as _impl + + return _impl(dataset, cap) + + +def _vram_peak_gb() -> float: + if not torch.cuda.is_available(): + return 0.0 + return torch.cuda.max_memory_allocated() / _BYTES_PER_GB + + +def _reset_vram_peak() -> None: + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats() + + +def _report_digest(report: Any) -> dict[str, Any]: + return { + "headroom": report.headroom.value, + "is_feasible": report.is_feasible, + "vram_gb": round(report.resource.vram_gb, 3), + "ram_gb": round(report.resource.ram_gb, 3), + "time_hours": round(report.resource.time_hours, 3), + "low_confidence": report.low_confidence, + "findings": [ + {"severity": f.severity.value, "metric": f.metric, "message": f.message} for f in report.findings + ], + } + + +def _scoring_modules(config: dict[str, Any]) -> list[str]: + out: list[str] = [] + for node in config.get("search_space", []): + if node.get("node_type") == "scoring": + out.extend(str(e.get("module_name")) for e in node.get("search_space", [])) + return out + + +def _mixed_infeasible_config(base_heavy: dict[str, Any], base_classic: dict[str, Any]) -> dict[str, Any]: + """deberta-v3-large (infeasible on 6 GB) + knn/linear (cheap) in one scoring node. + + This is the configuration shape reduce_to_fit was actually designed for: + something expensive to drop, something cheap to keep. + """ + cfg = copy.deepcopy(base_heavy) + heavy_scoring = next(n for n in cfg["search_space"] if n["node_type"] == "scoring") + classic_scoring = next(n for n in base_classic["search_space"] if n["node_type"] == "scoring") + heavy_scoring["search_space"] = [ + *copy.deepcopy(heavy_scoring["search_space"]), + *copy.deepcopy([e for e in classic_scoring["search_space"] if e["module_name"] in {"knn", "linear"}]), + ] + cfg["embedder_config"] = copy.deepcopy(base_classic.get("embedder_config", {})) + cfg["hpo_config"]["n_trials"] = 2 + return cfg + + +def main() -> None: + parser = argparse.ArgumentParser("phase3_reduce_to_fit") + parser.add_argument("--dataset", default="DeepPavlov/banking77") + parser.add_argument("--subsample-per-class", type=int, default=30) + parser.add_argument("--output", default="calibration_runs/phase3.json") + parser.add_argument( + "--skip-real-fit", + action="store_true", + help="Skip step C's real fit of the pruned config (paper-feasibility only).", + ) + args = parser.parse_args() + + hardware = detect_hardware() + print( + f"Hardware: {hardware.accelerator} ({hardware.device_name}) — " + f"{hardware.vram_gb:.2f} GB VRAM, {hardware.ram_gb:.1f} GB RAM, class={hardware.device_class}" + ) + + dataset = Dataset.from_hub(args.dataset) + if args.subsample_per_class: + dataset = _subsample_per_class(dataset, args.subsample_per_class) + stats = stats_from_dataset_obj(dataset) + print(f"Dataset: n_samples={stats.n_samples} n_classes={stats.n_classes}") + + heavy_cfg, _ = load_config("transformers-heavy") + classic_cfg, _ = load_config("classic-light") + + results: dict[str, Any] = { + "hardware": { + "accelerator": hardware.accelerator, + "device_name": hardware.device_name, + "vram_gb": hardware.vram_gb, + "ram_gb": hardware.ram_gb, + "device_class": hardware.device_class, + }, + "dataset": {"source": args.dataset, "n_samples": stats.n_samples, "n_classes": stats.n_classes}, + } + + # === A: strict gate must fire before any CUDA allocation ============== + print("\n=== A. Pipeline.fit(preflight='strict') on transformers-heavy ===") + _reset_vram_peak() + step_a: dict[str, Any] = {} + pipeline = Pipeline.from_preset("transformers-heavy") + t0 = time.perf_counter() + try: + pipeline.fit(dataset, preflight="strict") + except PreflightError as e: + step_a = { + "outcome": "PreflightError", + "raised_before_alloc": _vram_peak_gb() < 0.1, # noqa: PLR2004 + "vram_peak_gb": round(_vram_peak_gb(), 4), + "elapsed_s": round(time.perf_counter() - t0, 2), + "message": str(e)[:1000], + } + except Exception as e: # noqa: BLE001 + step_a = { + "outcome": type(e).__name__, + "vram_peak_gb": round(_vram_peak_gb(), 4), + "elapsed_s": round(time.perf_counter() - t0, 2), + "message": str(e)[:1000], + "traceback": traceback.format_exc()[-2000:], + } + else: + step_a = { + "outcome": "fit-completed", + "vram_peak_gb": round(_vram_peak_gb(), 4), + "elapsed_s": round(time.perf_counter() - t0, 2), + } + results["A_strict_gate"] = step_a + print(json.dumps(step_a, indent=2)[:1200]) + + # === B: reduce_to_fit on the single-module heavy preset =============== + print("\n=== B. reduce_to_fit(transformers-heavy) ===") + step_b: dict[str, Any] = {"scoring_modules_before": _scoring_modules(heavy_cfg)} + try: + pruned, report = reduce_to_fit(copy.deepcopy(heavy_cfg), stats, hardware) + except ReduceToFitError as e: + step_b.update( + { + "outcome": "ReduceToFitError", + "message": str(e), + "scoring_modules_after": _scoring_modules(e.pruned_config), + "last_report": _report_digest(e.last_report), + # The issue asks whether the error "points at a lighter preset". + "names_a_lighter_preset": any( + p in str(e) for p in ("classic", "nn-", "zero-shot", "transformers-light", "preset") + ), + } + ) + except Exception as e: # noqa: BLE001 + step_b.update({"outcome": type(e).__name__, "message": str(e), "traceback": traceback.format_exc()[-2000:]}) + else: + step_b.update( + { + "outcome": "pruned-to-feasible", + "scoring_modules_after": _scoring_modules(pruned), + "report": _report_digest(report), + } + ) + results["B_reduce_heavy"] = step_b + print(json.dumps(step_b, indent=2)[:1500]) + + # === C: reduce_to_fit on a mixed search space + real fit ============== + print("\n=== C. reduce_to_fit(deberta-v3-large + knn + linear) ===") + mixed = _mixed_infeasible_config(heavy_cfg, classic_cfg) + step_c: dict[str, Any] = {"scoring_modules_before": _scoring_modules(mixed)} + before = run_preflight(copy.deepcopy(mixed), stats, hardware, preset_name="mixed") + step_c["report_before"] = _report_digest(before) + print(f" before: headroom={before.headroom.value} vram={before.resource.vram_gb:.2f} GB") + + try: + pruned_mixed, report_mixed = reduce_to_fit(copy.deepcopy(mixed), stats, hardware) + except ReduceToFitError as e: + step_c.update({"outcome": "ReduceToFitError", "message": str(e), "last_report": _report_digest(e.last_report)}) + except Exception as e: # noqa: BLE001 + step_c.update({"outcome": type(e).__name__, "message": str(e), "traceback": traceback.format_exc()[-2000:]}) + else: + step_c.update( + { + "outcome": "pruned-to-feasible", + "scoring_modules_after": _scoring_modules(pruned_mixed), + "report_after": _report_digest(report_mixed), + } + ) + print( + f" after: modules={_scoring_modules(pruned_mixed)} " + f"headroom={report_mixed.headroom.value} vram={report_mixed.resource.vram_gb:.2f} GB" + ) + + # The whole point: is the survivor actually runnable on this box? + if not args.skip_real_fit: + print(" fitting the pruned config for real ...") + _reset_vram_peak() + t0 = time.perf_counter() + try: + pruned_pipeline = Pipeline.from_optimization_config(pruned_mixed) + pruned_pipeline.fit(dataset, preflight="off") + except Exception as e: # noqa: BLE001 + step_c["real_fit"] = { + "outcome": "failed", + "error": f"{type(e).__name__}: {e}"[:600], + "elapsed_s": round(time.perf_counter() - t0, 2), + "vram_peak_gb": round(_vram_peak_gb(), 3), + "traceback": traceback.format_exc()[-2000:], + } + else: + step_c["real_fit"] = { + "outcome": "ok", + "elapsed_s": round(time.perf_counter() - t0, 2), + "vram_peak_gb": round(_vram_peak_gb(), 3), + } + print(f" real fit: {json.dumps(step_c['real_fit'])[:500]}") + + results["C_reduce_mixed"] = step_c + + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(results, indent=2), encoding="utf-8") + print(f"\nWrote {out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/render_issue39_tables.py b/scripts/render_issue39_tables.py new file mode 100644 index 000000000..797baf0f7 --- /dev/null +++ b/scripts/render_issue39_tables.py @@ -0,0 +1,116 @@ +"""Render the issue-#39 definition-of-done tables straight from the run JSONs. + +Table 1 (feasibility) pairs each preset's advisor verdict with what really +happened. Table 2 (accuracy) reports actual/predicted ratios for the presets +that fit. Reading both out of the JSON rather than transcribing by hand keeps +the write-up honest. + +Usage: + uv run --no-sync python scripts/render_issue39_tables.py \ + --preflight calibration_runs/banking77_.json \ + --fits calibration_runs/phase2_isolated/*/banking77_*.json +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def _load_rows(paths: list[str]) -> dict[str, dict[str, Any]]: + """Merge rows from several calibration JSONs, keyed by preset name.""" + out: dict[str, dict[str, Any]] = {} + for p in paths: + payload = json.loads(Path(p).read_text(encoding="utf-8")) + for row in payload.get("rows", []): + out[row["preset"]] = row + return out + + +def _fmt(value: float | None, digits: int = 2) -> str: + return "—" if value is None else f"{value:.{digits}f}" + + +def _ratio(actual: float | None, predicted: float | None) -> str: + if actual is None or not predicted: + return "—" + return f"{actual / predicted:.2f}x" + + +def _outcome(row: dict[str, Any]) -> str: + """What actually happened during the fit: OOM / fit / not-run.""" + error = row.get("error") + if error is None: + return "fit" if row.get("actual") else "not run" + if "out of memory" in error.lower(): + return "OOM" + if row.get("skipped"): + return "skipped" + return "error" + + +def main() -> None: + parser = argparse.ArgumentParser("render_issue39_tables") + parser.add_argument("--preflight", required=True, help="Phase 1 (SKIP_FIT) JSON") + parser.add_argument("--fits", nargs="*", default=[], help="Phase 2 JSONs (one per preset is fine)") + parser.add_argument("--counterfactual", help="phase1b_counterfactual.json (optional)") + args = parser.parse_args() + + pre = _load_rows([args.preflight]) + fits = _load_rows(args.fits) + counter = {} + if args.counterfactual: + payload = json.loads(Path(args.counterfactual).read_text(encoding="utf-8")) + counter = {r["preset"]: r for r in payload["rows"]} + + print("### Table 1 — feasibility\n") + print("| preset | model | pred VRAM (GB) | advisor verdict | actual | match? |") + print("| --- | --- | --- | --- | --- | --- |") + for preset, row in pre.items(): + if row.get("skipped"): + continue + vram_sev = row.get("severity_by_metric", {}).get("vram", "—") + pred_vram = row["predicted"].get("vram_gb") + fit_row = fits.get(preset) + actual = _outcome(fit_row) if fit_row else "not run" + if actual in {"not run", "skipped", "error"}: + match = "—" + else: + predicted_over = vram_sev == "over" + match = "✅" if predicted_over == (actual == "OOM") else "❌" + model = ", ".join(sorted(set(row.get("models", {}).values()))) or "—" + print( + f"| `{preset}` | {model} | {_fmt(pred_vram)} | **{vram_sev}** | {actual} | {match} |" + ) + + print("\n### Table 2 — accuracy on presets that fit\n") + print("| preset | actual/pred VRAM | actual/pred RAM | actual/pred time | actual VRAM (GB) | pred VRAM (GB) |") + print("| --- | --- | --- | --- | --- | --- |") + for preset, row in fits.items(): + if _outcome(row) != "fit": + continue + actual, predicted = row["actual"], row["predicted"] + print( + f"| `{preset}` | {_ratio(actual.get('vram_gb'), predicted.get('vram_gb'))} " + f"| {_ratio(actual.get('ram_gb'), predicted.get('ram_gb'))} " + f"| {_ratio(actual.get('time_h'), predicted.get('time_h'))} " + f"| {_fmt(actual.get('vram_gb'))} | {_fmt(predicted.get('vram_gb'))} |" + ) + + if counter: + print("\n### Metadata counterfactual (deberta has no safetensors -> 350M fallback)\n") + print("| preset | params assumed | params true | VRAM as-run | VRAM corrected | verdict changes? |") + print("| --- | --- | --- | --- | --- | --- |") + for preset, row in counter.items(): + changed = "no" if row["as_run"]["headroom"] == row["corrected"]["headroom"] else "YES" + print( + f"| `{preset}` | {row['params_assumed_M']} M | {row['params_true_M']} M " + f"| {row['as_run']['vram_gb']} GB ({row['as_run']['headroom']}) " + f"| {row['corrected']['vram_gb']} GB ({row['corrected']['headroom']}) | {changed} |" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_calibration_banking77.sh b/scripts/run_calibration_banking77.sh index 1c6f0d869..aec41902d 100755 --- a/scripts/run_calibration_banking77.sh +++ b/scripts/run_calibration_banking77.sh @@ -134,7 +134,9 @@ else while IFS= read -r preset; do PRESET_ARR+=("$preset") done < <( -python - <<'PY' +# Must go through `uv run` — the bare interpreter has no autointent on its +# path, which made the no-PRESETS default invocation die with an empty list. +uv run --no-sync python - <<'PY' from autointent._advisor import BUNDLED_PRESETS for name in BUNDLED_PRESETS: print(name) diff --git a/scripts/run_phase2_isolated.sh b/scripts/run_phase2_isolated.sh new file mode 100755 index 000000000..2a1884555 --- /dev/null +++ b/scripts/run_phase2_isolated.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Phase 2 of issue #39, one preset per PROCESS. +# +# The sweep driver (run_calibration_banking77.sh) walks every preset inside a +# single python process. That is fine on an A100 and actively misleading on a +# 6 GB card: the first preset to OOM leaves its model + optimizer state + HPO +# trial objects alive, so the next preset starts several GB in the hole and +# OOMs on an allocation it would never have made on a clean device. Observed +# directly on this box — `classic-light` (predicted 1.8 GB, AMPLE) "OOMed" +# only because `transformers-no-hpo` had leaked 5.4 GB immediately before it. +# +# Process isolation is the only airtight fix: the kernel reclaims the GPU on +# exit no matter what the python object graph is holding. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +OUTPUT_DIR="${OUTPUT_DIR:-$REPO_ROOT/calibration_runs/phase2_isolated}" +mkdir -p "$OUTPUT_DIR" + +# Cheap/AMPLE presets first, the expected-OOM ones last. +PRESETS="${PRESETS:-classic-light zero-shot-encoders transformers-no-hpo transformers-heavy}" + +for preset in $PRESETS; do + echo "############ $preset ############" + nvidia-smi --query-gpu=memory.used --format=csv,noheader + COLD=1 REQUIRE_CUDA=1 MAX_TRIALS="${MAX_TRIALS:-2}" \ + SUBSAMPLE_PER_CLASS="${SUBSAMPLE_PER_CLASS:-30}" \ + PRESETS="$preset" RUN_NAME="${RUN_NAME:-laptop6gb_fit}" \ + OUTPUT_DIR="$OUTPUT_DIR/$preset" \ + scripts/run_calibration_banking77.sh || echo "!!! $preset driver exited non-zero" + echo "############ $preset done; GPU after process exit:" + nvidia-smi --query-gpu=memory.used --format=csv,noheader +done + +echo "All presets done. JSONs under $OUTPUT_DIR"