From 5eda1b60c58bb97a7ea836698f2db2eb910fac86 Mon Sep 17 00:00:00 2001 From: toti330 Date: Thu, 30 Jul 2026 16:39:41 +0000 Subject: [PATCH 1/2] Add vein estimator validation tools --- benchmarks/evaluate_vein_accuracy.py | 347 ++++++++++++++++++++++ benchmarks/fit_vein_scales.py | 105 +++++++ benchmarks/run_comparison.py | 11 + benchmarks/test_evaluate_vein_accuracy.py | 91 ++++++ benchmarks/vein_estimator_corpora.json | 19 ++ inserter/src/bin/vein_accuracy.rs | 235 +++++++++++++++ inserter/src/generate_csv.rs | 44 +++ 7 files changed, 852 insertions(+) create mode 100644 benchmarks/evaluate_vein_accuracy.py create mode 100644 benchmarks/fit_vein_scales.py create mode 100644 benchmarks/test_evaluate_vein_accuracy.py create mode 100644 benchmarks/vein_estimator_corpora.json create mode 100644 inserter/src/bin/vein_accuracy.rs diff --git a/benchmarks/evaluate_vein_accuracy.py b/benchmarks/evaluate_vein_accuracy.py new file mode 100644 index 0000000..081b7bc --- /dev/null +++ b/benchmarks/evaluate_vein_accuracy.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +"""Measure planet ore estimates against exact vein-generation output.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + + +ORE_NAMES = ( + "iron", + "copper", + "silicium", + "titanium", + "stone", + "coal", + "oil", + "fireice", + "diamond", + "fractal", + "crysrub", + "grat", + "bamboo", + "mag", +) +MAX_I32 = 2_147_483_647 +DEFAULT_CORPUS_MANIFEST = Path(__file__).with_name("vein_estimator_corpora.json") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--scales", type=Path) + parser.add_argument("--seed-set", required=True) + parser.add_argument("--estimate-prefix", default="estimate") + parser.add_argument("--corpus-manifest", type=Path, default=DEFAULT_CORPUS_MANIFEST) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--exporter", required=True, type=Path) + parser.add_argument("--allow-held-out", action="store_true") + return parser.parse_args() + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def percentile_nearest_rank(values: list[float], fraction: float) -> float | None: + if not values: + return None + values.sort() + return values[max(0, math.ceil(fraction * len(values)) - 1)] + + +@dataclass +class Accuracy: + count: int = 0 + reference_sum: int = 0 + predicted_sum: int = 0 + absolute_error_sum: int = 0 + signed_error_sum: int = 0 + non_null_count: int = 0 + non_null_absolute_error_sum: int = 0 + non_null_signed_error_sum: int = 0 + null_count: int = 0 + correct_null_count: int = 0 + true_positive: int = 0 + false_positive: int = 0 + false_negative: int = 0 + relative_errors: list[float] = field(default_factory=list) + + def add(self, predicted: int, actual: int) -> None: + error = predicted - actual + absolute_error = abs(error) + self.count += 1 + self.reference_sum += actual + self.predicted_sum += predicted + self.absolute_error_sum += absolute_error + self.signed_error_sum += error + if actual > 0: + self.non_null_count += 1 + self.non_null_absolute_error_sum += absolute_error + self.non_null_signed_error_sum += error + self.relative_errors.append(absolute_error / actual) + if predicted > 0: + self.true_positive += 1 + else: + self.false_negative += 1 + else: + self.null_count += 1 + if predicted > 0: + self.false_positive += 1 + else: + self.correct_null_count += 1 + + def result(self) -> dict[str, object]: + predicted_positive = self.true_positive + self.false_positive + reference_positive = self.true_positive + self.false_negative + precision = self.true_positive / predicted_positive if predicted_positive else None + recall = self.true_positive / reference_positive if reference_positive else None + return { + "all_amounts": { + "count": self.count, + "reference_sum": self.reference_sum, + "predicted_sum": self.predicted_sum, + "mean_absolute_error": self.absolute_error_sum / self.count, + "weighted_absolute_percentage_error": ( + self.absolute_error_sum / self.reference_sum + if self.reference_sum + else 0.0 + ), + "weighted_signed_bias": ( + self.signed_error_sum / self.reference_sum + if self.reference_sum + else 0.0 + ), + }, + "non_null_reference": { + "count": self.non_null_count, + "mean_absolute_error": ( + self.non_null_absolute_error_sum / self.non_null_count + if self.non_null_count + else 0.0 + ), + "mean_absolute_relative_error": ( + sum(self.relative_errors) / self.non_null_count + if self.non_null_count + else 0.0 + ), + "p95_absolute_relative_error": percentile_nearest_rank( + self.relative_errors, 0.95 + ), + "mean_signed_error": ( + self.non_null_signed_error_sum / self.non_null_count + if self.non_null_count + else 0.0 + ), + }, + "null_reference": { + "count": self.null_count, + "correct": self.correct_null_count, + "false_positive": self.false_positive, + "accuracy": ( + self.correct_null_count / self.null_count + if self.null_count + else 1.0 + ), + }, + "existence": { + "true_positive": self.true_positive, + "false_positive": self.false_positive, + "false_negative": self.false_negative, + "precision": precision, + "recall": recall, + }, + } + + +def load_scales(path: Path | None) -> tuple[int, dict[int, list[int]], str | None]: + if path is None: + return 1, {}, None + artifact = json.loads(path.read_text(encoding="ascii")) + denominator = int(artifact["denominator"]) + scales = { + int(theme): [int(value) for value in values] + for theme, values in artifact["theme_scales"].items() + } + for theme, values in scales.items(): + if len(values) != len(ORE_NAMES): + raise ValueError(f"theme {theme} does not have {len(ORE_NAMES)} scales") + return denominator, scales, file_sha256(path) + + +def load_seed_set( + manifest_path: Path, seed_set: str, allow_held_out: bool = False +) -> tuple[dict[str, int], str]: + manifest = json.loads(manifest_path.read_text(encoding="ascii")) + if seed_set == "held_out_validation" and not allow_held_out: + raise ValueError("held-out validation requires --allow-held-out") + try: + definition = manifest["seed_sets"][seed_set] + except KeyError as error: + raise ValueError(f"unknown seed set: {seed_set}") from error + start = int(definition["start_seed"]) + end = int(definition["end_seed_exclusive"]) + if start >= end: + raise ValueError(f"invalid seed range for {seed_set}") + return {"start_seed": start, "end_seed_exclusive": end}, file_sha256( + manifest_path + ) + + +def validate_seed_values(seeds: set[int], definition: dict[str, int]) -> None: + expected = set( + range(definition["start_seed"], definition["end_seed_exclusive"]) + ) + if seeds == expected: + return + missing = sorted(expected - seeds)[:5] + unexpected = sorted(seeds - expected)[:5] + raise ValueError( + f"input does not match seed set; missing={missing}, unexpected={unexpected}" + ) + + +def scale_estimate( + estimate: int, + theme_id: int, + ore_index: int, + denominator: int, + scales: dict[int, list[int]], +) -> int: + scale = scales.get(theme_id, [denominator] * len(ORE_NAMES))[ore_index] + value = (estimate * scale + denominator // 2) // denominator + return min(value, MAX_I32) + + +def evaluate( + input_path: Path, + denominator: int, + scales: dict[int, list[int]], + estimate_prefix: str, +) -> tuple[set[int], int, dict[str, object], dict[str, dict[str, object]]]: + overall = Accuracy() + by_ore = {ore: Accuracy() for ore in ORE_NAMES} + seeds: set[int] = set() + planet_count = 0 + with input_path.open(newline="", encoding="ascii") as source: + reader = csv.DictReader(source) + for row in reader: + seeds.add(int(row["seed"])) + planet_count += 1 + gas = row["gas"] == "1" + theme_id = int(row["theme_id"]) + for ore_index, ore in enumerate(ORE_NAMES): + estimate = int(row[f"{estimate_prefix}_{ore}"]) + actual = int(row[f"actual_{ore}"]) + predicted = 0 if gas else scale_estimate( + estimate, theme_id, ore_index, denominator, scales + ) + by_ore[ore].add(predicted, actual) + overall.add(predicted, actual) + return ( + seeds, + planet_count, + overall.result(), + {ore: values.result() for ore, values in by_ore.items()}, + ) + + +def accuracy_requirement_results( + overall: dict[str, object], by_ore: dict[str, dict[str, object]] +) -> dict[str, object]: + overall_wape = overall["all_amounts"]["weighted_absolute_percentage_error"] + overall_p95 = overall["non_null_reference"]["p95_absolute_relative_error"] + all_ore_wape = all( + values["all_amounts"]["weighted_absolute_percentage_error"] <= 0.15 + for values in by_ore.values() + ) + all_existence = all( + values["existence"]["precision"] is not None + and values["existence"]["precision"] >= 0.99 + and values["existence"]["recall"] is not None + and values["existence"]["recall"] >= 0.99 + for values in by_ore.values() + ) + return { + "overall_wape_at_most_0_10": overall_wape <= 0.10, + "each_ore_wape_at_most_0_15": all_ore_wape, + "overall_non_null_p95_are_at_most_0_25": overall_p95 is not None + and overall_p95 <= 0.25, + "each_ore_existence_precision_and_recall_at_least_0_99": all_existence, + "passed": overall_wape <= 0.10 + and all_ore_wape + and overall_p95 is not None + and overall_p95 <= 0.25 + and all_existence, + } + + +def main() -> None: + args = parse_args() + if not args.exporter.is_file(): + raise ValueError(f"exporter does not exist: {args.exporter}") + seed_definition, corpus_sha256 = load_seed_set( + args.corpus_manifest, args.seed_set, args.allow_held_out + ) + denominator, scales, scale_sha256 = load_scales(args.scales) + seeds, planet_count, overall, by_ore = evaluate( + args.input, denominator, scales, args.estimate_prefix + ) + validate_seed_values(seeds, seed_definition) + artifact = { + "schema_version": 2, + "created_utc": datetime.now(timezone.utc).isoformat(), + "seed_set": args.seed_set, + "estimate_prefix": args.estimate_prefix, + "corpus": { + "manifest": str(args.corpus_manifest), + "sha256": corpus_sha256, + **seed_definition, + }, + "provenance": { + "source_revision": args.source_revision, + "exporter": str(args.exporter), + "exporter_sha256": file_sha256(args.exporter), + }, + "input": { + "path": str(args.input), + "sha256": file_sha256(args.input), + "seed_count": len(seeds), + "planet_count": planet_count, + }, + "scales": { + "path": str(args.scales) if args.scales else None, + "sha256": scale_sha256, + "denominator": denominator, + }, + "overall": overall, + "by_ore": by_ore, + } + artifact["accuracy_requirements"] = accuracy_requirement_results(overall, by_ore) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(artifact, indent=2) + "\n", encoding="ascii") + print( + json.dumps( + { + "overall": overall, + "accuracy_requirements": artifact["accuracy_requirements"], + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/fit_vein_scales.py b/benchmarks/fit_vein_scales.py new file mode 100644 index 0000000..f8e9688 --- /dev/null +++ b/benchmarks/fit_vein_scales.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Fit fixed-point theme and ore scales on a calibration oracle.""" + +from __future__ import annotations + +import argparse +import csv +import json +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +from evaluate_vein_accuracy import ( + DEFAULT_CORPUS_MANIFEST, + ORE_NAMES, + file_sha256, + load_seed_set, + validate_seed_values, +) + + +DENOMINATOR = 65_536 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--corpus-manifest", type=Path, default=DEFAULT_CORPUS_MANIFEST) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--exporter", required=True, type=Path) + return parser.parse_args() + + +def weighted_median_scale(samples: list[tuple[int, int]]) -> int: + if not samples: + return DENOMINATOR + ordered = sorted(samples, key=lambda pair: pair[1] / pair[0]) + total_weight = sum(estimate for estimate, _ in ordered) + cumulative = 0 + for estimate, actual in ordered: + cumulative += estimate + if cumulative * 2 >= total_weight: + return max(1, (actual * DENOMINATOR + estimate // 2) // estimate) + raise AssertionError("weighted median was not found") + + +def main() -> None: + args = parse_args() + if not args.exporter.is_file(): + raise ValueError(f"exporter does not exist: {args.exporter}") + seed_definition, corpus_sha256 = load_seed_set( + args.corpus_manifest, "calibration" + ) + samples: dict[tuple[int, int], list[tuple[int, int]]] = defaultdict(list) + seeds: set[int] = set() + with args.input.open(newline="", encoding="ascii") as source: + for row in csv.DictReader(source): + seeds.add(int(row["seed"])) + if row["gas"] == "1": + continue + theme_id = int(row["theme_id"]) + for ore_index, ore in enumerate(ORE_NAMES): + estimate = int(row[f"estimate_{ore}"]) + if estimate > 0: + samples[(theme_id, ore_index)].append( + (estimate, int(row[f"actual_{ore}"])) + ) + validate_seed_values(seeds, seed_definition) + + theme_scales = { + str(theme_id): [ + weighted_median_scale(samples[(theme_id, ore_index)]) + for ore_index in range(len(ORE_NAMES)) + ] + for theme_id in range(1, 26) + } + artifact = { + "schema_version": 2, + "created_utc": datetime.now(timezone.utc).isoformat(), + "method": "weighted median of actual divided by historical estimate", + "corpus": { + "manifest": str(args.corpus_manifest), + "sha256": corpus_sha256, + **seed_definition, + }, + "provenance": { + "source_revision": args.source_revision, + "exporter": str(args.exporter), + "exporter_sha256": file_sha256(args.exporter), + }, + "input": { + "path": str(args.input), + "sha256": file_sha256(args.input), + "seed_count": len(seeds), + }, + "denominator": DENOMINATOR, + "theme_scales": theme_scales, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(artifact, indent=2) + "\n", encoding="ascii") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/run_comparison.py b/benchmarks/run_comparison.py index f3c8ddd..863746b 100755 --- a/benchmarks/run_comparison.py +++ b/benchmarks/run_comparison.py @@ -163,6 +163,9 @@ def summarize(results: list[dict[str, object]], candidate_name: str) -> dict[str } baseline = by_variant["baseline"] candidate = by_variant[candidate_name] + baseline_median = statistics.median(baseline) + candidate_median = statistics.median(candidate) + maximum_elapsed_overhead = 0.05 paired_changes = [ (new / old - 1.0) * 100.0 for old, new in zip(baseline, candidate, strict=True) ] @@ -175,6 +178,14 @@ def summarize(results: list[dict[str, object]], candidate_name: str) -> dict[str **describe(paired_changes), "bootstrap_95_percent_ci": bootstrap_ci(paired_changes), }, + "performance_requirement": { + "maximum_elapsed_time_overhead": maximum_elapsed_overhead, + "measured_elapsed_time_overhead": baseline_median / candidate_median - 1.0, + "minimum_candidate_throughput": baseline_median + / (1.0 + maximum_elapsed_overhead), + "passed": candidate_median + >= baseline_median / (1.0 + maximum_elapsed_overhead), + }, } diff --git a/benchmarks/test_evaluate_vein_accuracy.py b/benchmarks/test_evaluate_vein_accuracy.py new file mode 100644 index 0000000..a99cefb --- /dev/null +++ b/benchmarks/test_evaluate_vein_accuracy.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +import unittest + +from evaluate_vein_accuracy import ( + ORE_NAMES, + Accuracy, + accuracy_requirement_results, + percentile_nearest_rank, + scale_estimate, + validate_seed_values, +) +from fit_vein_scales import DENOMINATOR, weighted_median_scale +from run_comparison import summarize + + +class AccuracyTest(unittest.TestCase): + def test_reports_amount_and_existence_errors(self) -> None: + accuracy = Accuracy() + accuracy.add(90, 100) + accuracy.add(20, 0) + accuracy.add(0, 50) + accuracy.add(0, 0) + + result = accuracy.result() + self.assertAlmostEqual( + result["all_amounts"]["weighted_absolute_percentage_error"], + 80 / 150, + ) + self.assertEqual(result["null_reference"]["correct"], 1) + self.assertEqual(result["existence"]["true_positive"], 1) + self.assertEqual(result["existence"]["false_positive"], 1) + self.assertEqual(result["existence"]["false_negative"], 1) + + def test_undefined_existence_metrics_fail_the_accuracy_gate(self) -> None: + accuracy = Accuracy() + accuracy.add(0, 0) + result = accuracy.result() + by_ore = {ore: result for ore in ORE_NAMES} + + self.assertIsNone(result["existence"]["precision"]) + self.assertIsNone(result["existence"]["recall"]) + self.assertFalse( + accuracy_requirement_results(result, by_ore)[ + "each_ore_existence_precision_and_recall_at_least_0_99" + ] + ) + + def test_percentile_uses_nearest_rank(self) -> None: + self.assertEqual(percentile_nearest_rank(list(range(1, 21)), 0.95), 19) + + def test_seed_values_must_match_the_locked_range(self) -> None: + definition = {"start_seed": 10, "end_seed_exclusive": 13} + validate_seed_values({10, 11, 12}, definition) + with self.assertRaises(ValueError): + validate_seed_values({10, 12}, definition) + + def test_fixed_point_scale_rounds_and_clamps(self) -> None: + scales = {7: [98_304] * 14} + self.assertEqual(scale_estimate(3, 7, 0, 65_536, scales), 5) + self.assertEqual( + scale_estimate(2_147_483_647, 7, 0, 65_536, scales), + 2_147_483_647, + ) + + def test_weighted_median_scale_uses_estimate_weights(self) -> None: + self.assertEqual( + weighted_median_scale([(2, 2), (10, 20), (2, 6)]), + 2 * DENOMINATOR, + ) + + def test_performance_gate_uses_elapsed_time_overhead(self) -> None: + passing = [ + {"variant": "baseline", "throughput_seeds_per_second": 100.0}, + {"variant": "candidate", "throughput_seeds_per_second": 96.0}, + ] + failing = [ + {"variant": "baseline", "throughput_seeds_per_second": 100.0}, + {"variant": "candidate", "throughput_seeds_per_second": 95.0}, + ] + + self.assertTrue( + summarize(passing, "candidate")["performance_requirement"]["passed"] + ) + self.assertFalse( + summarize(failing, "candidate")["performance_requirement"]["passed"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/vein_estimator_corpora.json b/benchmarks/vein_estimator_corpora.json new file mode 100644 index 0000000..042bfa1 --- /dev/null +++ b/benchmarks/vein_estimator_corpora.json @@ -0,0 +1,19 @@ +{ + "schema_version": 1, + "star_count": 64, + "resource_multiplier": 1.0, + "seed_sets": { + "calibration": { + "start_seed": 0, + "end_seed_exclusive": 1000 + }, + "held_out_validation": { + "start_seed": 1000000, + "end_seed_exclusive": 1001000 + }, + "performance": { + "start_seed": 2000000, + "end_seed_exclusive": 2000200 + } + } +} diff --git a/inserter/src/bin/vein_accuracy.rs b/inserter/src/bin/vein_accuracy.rs new file mode 100644 index 0000000..0f7b330 --- /dev/null +++ b/inserter/src/bin/vein_accuracy.rs @@ -0,0 +1,235 @@ +#[path = "../algorithm/mod.rs"] +mod algorithm; + +use algorithm::data::enums::{ThemeDistribute, VeinType, ORES}; +use algorithm::data::game_desc::GameDesc; +use algorithm::data::planet::Planet; +use algorithm::data::random::DspRandom; +use algorithm::data::vector_f3::VectorF3; +use algorithm::data::vein::EstimatedVein; +use algorithm::generate_stars; +use anyhow::{bail, Context, Result}; +use std::env; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::Path; + +const STAR_COUNT: usize = 64; +const REC_MULTIPLIER: f32 = 1.0; +const HELD_OUT_START: i32 = 1_000_000; +const HELD_OUT_END: i32 = 1_001_000; +const ORE_NAMES: [&str; 14] = [ + "iron", "copper", "silicium", "titanium", "stone", "coal", "oil", "fireice", "diamond", + "fractal", "crysrub", "grat", "bamboo", "mag", +]; + +fn parse_seed(value: Option, name: &str) -> Result { + value + .with_context(|| format!("missing {name}"))? + .parse() + .with_context(|| format!("invalid {name}")) +} + +fn write_header(writer: &mut impl Write) -> Result<()> { + write!( + writer, + "seed,star_index,planet_index,planet_seed,theme_id,algorithm_id,star_type,spectr,gas" + )?; + for ore in ORE_NAMES { + write!(writer, ",estimate_{ore}")?; + } + for ore in ORE_NAMES { + write!(writer, ",spacing_{ore}")?; + } + for ore in ORE_NAMES { + write!(writer, ",actual_{ore}")?; + } + writeln!(writer)?; + Ok(()) +} + +fn base_amounts(veins: &[EstimatedVein]) -> ([i64; 16], [i32; 16]) { + let mut amounts = [0_i64; 16]; + let mut spots = [0_i32; 16]; + for vein in veins { + let index = vein.vein_type as usize; + amounts[index] = vein.estimate(); + spots[index] = vein.min_group + 1; + } + (amounts, spots) +} + +fn estimate_spacing(planet: &Planet<'_>, veins: &[EstimatedVein]) -> [i64; 16] { + let (mut amounts, spots) = base_amounts(veins); + if planet.get_theme().distribute == ThemeDistribute::Birth { + return amounts; + } + + let mut source = DspRandom::new(planet.seed); + for _ in 0..5 { + source.advance(); + } + let mut random = DspRandom::new(source.next_seed()); + let mut birth_point = VectorF3::new( + (random.next_f64() * 2.0 - 1.0) as f32, + (random.next_f64() - 0.5) as f32, + (random.next_f64() * 2.0 - 1.0) as f32, + ); + birth_point.normalize(); + birth_point *= (random.next_f64() * 0.4 + 0.2) as f32; + + let mut centers = [VectorF3::zero(); 512]; + let mut center_count = 0; + let min_spacing = 2.1 / planet.radius; + let min_spacing_sq = (min_spacing as f64) * (min_spacing as f64); + for ore in &ORES[1..15] { + let index = *ore as usize; + let nominal = spots[index]; + if nominal == 0 { + continue; + } + let requested = if nominal > 1 { + nominal + random.next_i32(3) - 1 + } else { + nominal + }; + let threshold = min_spacing_sq * if ore == &VeinType::Oil { 100.0 } else { 196.0 }; + let mut accepted = 0; + for _ in 0..requested { + for _ in 0..200 { + let mut direction = VectorF3::new( + (random.next_f64() * 2.0 - 1.0) as f32, + (random.next_f64() * 2.0 - 1.0) as f32, + (random.next_f64() * 2.0 - 1.0) as f32, + ); + if ore != &VeinType::Oil { + direction += birth_point; + } + direction.normalize(); + if centers[..center_count] + .iter() + .all(|center| center.distance_sq_from(&direction) as f64 >= threshold) + { + if center_count == centers.len() { + break; + } + centers[center_count] = direction; + center_count += 1; + accepted += 1; + break; + } + } + } + amounts[index] = (amounts[index] * accepted as i64 + nominal as i64 / 2) / nominal as i64; + } + amounts +} + +fn run() -> Result<()> { + let mut args = env::args().skip(1); + let start_seed = parse_seed(args.next(), "start seed")?; + let end_seed = parse_seed(args.next(), "end seed")?; + let output = args.next().context("missing output path")?; + let allow_held_out = match args.next().as_deref() { + None => false, + Some("--allow-held-out") if args.next().is_none() => true, + Some(_) => bail!("expected only the optional --allow-held-out flag"), + }; + if start_seed >= end_seed { + bail!("start seed must be less than end seed"); + } + if start_seed < HELD_OUT_END && end_seed > HELD_OUT_START && !allow_held_out { + bail!("held-out seed range requires --allow-held-out"); + } + + let file = File::create(Path::new(&output)) + .with_context(|| format!("cannot create output file: {output}"))?; + let mut writer = BufWriter::new(file); + write_header(&mut writer)?; + let game_desc = GameDesc { + star_count: STAR_COUNT, + resource_multiplier: REC_MULTIPLIER, + }; + + for seed in start_seed..end_seed { + let habitable_count = std::cell::Cell::new(0); + for solar_system in generate_stars(seed, &game_desc, &habitable_count) { + let star = &solar_system.star; + for planet in solar_system.get_planets() { + let estimated_veins = planet.get_estimated_veins(); + let (estimates, _) = base_amounts(estimated_veins); + let spacing = estimate_spacing(planet, estimated_veins); + let mut actual = [0_i32; 16]; + for vein in planet.get_actual_veins() { + actual[vein.vein_type as usize] = vein.amount; + } + write!( + writer, + "{},{},{},{},{},{},{},{},{}", + seed, + star.index, + planet.index, + planet.seed, + planet.get_theme().id, + planet.get_algo() as i32, + star.star_type as i32, + star.get_spectr() as i32, + planet.is_gas_giant() as u8, + )?; + for ore in &ORES[1..15] { + write!(writer, ",{}", estimates[*ore as usize])?; + } + for ore in &ORES[1..15] { + write!(writer, ",{}", spacing[*ore as usize])?; + } + for ore in &ORES[1..15] { + write!(writer, ",{}", actual[*ore as usize])?; + } + writeln!(writer)?; + } + } + } + writer.flush()?; + Ok(()) +} + +fn main() -> Result<()> { + run() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn header_has_metadata_and_three_fields_per_ore() { + let mut output = Vec::new(); + write_header(&mut output).unwrap(); + let header = String::from_utf8(output).unwrap(); + let fields: Vec<_> = header.trim_end().split(',').collect(); + + assert_eq!(fields.len(), 9 + ORE_NAMES.len() * 3); + assert_eq!(fields[0], "seed"); + assert_eq!(fields[9], "estimate_iron"); + assert_eq!(fields[fields.len() - 1], "actual_mag"); + } + + #[test] + fn spacing_estimate_has_a_deterministic_fixture() { + let game_desc = GameDesc { + star_count: STAR_COUNT, + resource_multiplier: REC_MULTIPLIER, + }; + let habitable_count = std::cell::Cell::new(0); + let systems = generate_stars(0, &game_desc, &habitable_count); + let planet = &systems[0].get_planets()[2]; + + assert_eq!( + estimate_spacing(planet, planet.get_estimated_veins()), + [ + 0, 5_148_000, 148_500, 4_620_000, 15_972_000, 2_310_000, 89_100, 0, 2_904_000, 0, + 0, 0, 0, 0, 0, 0, + ] + ); + } +} diff --git a/inserter/src/generate_csv.rs b/inserter/src/generate_csv.rs index 2cdfd4e..0d54836 100644 --- a/inserter/src/generate_csv.rs +++ b/inserter/src/generate_csv.rs @@ -177,4 +177,48 @@ mod tests { assert_eq!(COPY_HEADER_FIELDS.len(), 8); assert_eq!(COPY_FOOTER, [255, 255]); } + + #[test] + fn generated_planet_ores_use_binary_nulls() { + let (_, planets) = gen_formatted(0).unwrap(); + let mut offset = 0; + let mut gas_rows = 0; + let mut rocky_nulls = 0; + let mut rocky_amounts = 0; + + while offset < planets.len() { + let columns = i16::from_be_bytes(planets[offset..offset + 2].try_into().unwrap()); + assert_eq!(columns, 26); + offset += 2; + let mut fields = Vec::with_capacity(columns as usize); + for _ in 0..columns { + let length = i32::from_be_bytes(planets[offset..offset + 4].try_into().unwrap()); + offset += 4; + if length == -1 { + fields.push(None); + } else { + let end = offset + length as usize; + fields.push(Some(&planets[offset..end])); + offset = end; + } + } + + let is_gas = fields[3].unwrap() == [1]; + for ore in &fields[12..26] { + if is_gas { + assert!(ore.is_none()); + gas_rows += 1; + } else if ore.is_none() { + rocky_nulls += 1; + } else { + assert_eq!(ore.unwrap().len(), 4); + rocky_amounts += 1; + } + } + } + + assert!(gas_rows > 0); + assert!(rocky_nulls > 0); + assert!(rocky_amounts > 0); + } } From 42efba32d1d05d8c72f93010648359de39f9d837 Mon Sep 17 00:00:00 2001 From: toti330 Date: Thu, 30 Jul 2026 16:53:47 +0000 Subject: [PATCH 2/2] Document vein estimator benchmark results --- .../results/2026-07-30-vein-baselines.json | 244 +++++++++ .../2026-07-30-vein-calibration-midpoint.json | 487 ++++++++++++++++++ .../2026-07-30-vein-calibration-spacing.json | 487 ++++++++++++++++++ .../2026-07-30-vein-estimator-benchmark.md | 75 +++ .../2026-07-30-vein-final-comparison.json | 241 +++++++++ docs/vein-estimator-study.md | 277 ++++++++++ 6 files changed, 1811 insertions(+) create mode 100644 benchmarks/results/2026-07-30-vein-baselines.json create mode 100644 benchmarks/results/2026-07-30-vein-calibration-midpoint.json create mode 100644 benchmarks/results/2026-07-30-vein-calibration-spacing.json create mode 100644 benchmarks/results/2026-07-30-vein-estimator-benchmark.md create mode 100644 benchmarks/results/2026-07-30-vein-final-comparison.json create mode 100644 docs/vein-estimator-study.md diff --git a/benchmarks/results/2026-07-30-vein-baselines.json b/benchmarks/results/2026-07-30-vein-baselines.json new file mode 100644 index 0000000..c1087b4 --- /dev/null +++ b/benchmarks/results/2026-07-30-vein-baselines.json @@ -0,0 +1,244 @@ +{ + "schema_version": 1, + "build": { + "rust": "rustc 1.97.0 (2d8144b78 2026-07-07)", + "cargo": "cargo 1.97.0 (c980f4866 2026-06-30)", + "builder_image": "rust:1.97.0@sha256:b92b8c8574f8f3b207fcb0912fb3e2de4041580b5934d90312d53938c9a038a9", + "target": "x86_64-unknown-linux-musl", + "profile": "release", + "cargo_flags": ["--release", "--locked", "--target", "x86_64-unknown-linux-musl"] + }, + "created_utc": "2026-07-30T13:48:42.967470+00:00", + "host": { + "platform": "Linux-6.17.0-1021-azure-x86_64-with-glibc2.39", + "processor_count": 2, + "processor": "Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz", + "physical_cores": 1, + "memory_bytes": 8270495744, + "docker": "29.6.1", + "cpu_set": "0" + }, + "workload": { + "corpus_manifest": "benchmarks/vein_estimator_corpora.json", + "corpus_manifest_sha256": "6d8c5f92824ecdaa0ad0298c2f418ca38a59bb6cd23ff0b7c49d3b2c8c434a21", + "seed_set": "performance", + "start_seed": 2000000, + "end_seed": 2000200, + "seed_count": 200, + "workers": 1, + "writer_sinks": 1, + "channel_size": 64 + }, + "executables": { + "baseline": { + "path": "/tmp/opencode/dsp-vein-fast-0df70/inserter/target/x86_64-unknown-linux-musl/release/dsp_seed_finder", + "sha256": "2b078c659e56a4ffaa31e905170a3aa96c363294db28f87a2241db76e2d4ba5e", + "source_commit": "0df70ce645d4573ea135ea179a1261890851ec00", + "cargo_lock_sha256": "e141f9481732d58d1882448ae576074950cb8d1efb19b39a9f46ada2b49301d4", + "vein_mode": "historical midpoint estimate without terrain or placement" + }, + "exact-current": { + "path": "/tmp/opencode/dsp-vein-exact-59232/inserter/target/x86_64-unknown-linux-musl/release/dsp_seed_finder", + "sha256": "17445fbe6f040420e974e197a7d86e841c03263079daeaaaeaa190113552e5fa", + "source_commit": "59232b78b5e8065b5697fe6bf6cd253c3a15f9e9", + "cargo_lock_sha256": "b8ad27ccee4fea907a956d954ec8f72f6ad82c5255da4e57aba610193610a047", + "vein_mode": "exact terrain and vein placement" + } + }, + "results": [ + { + "sequence": 0, + "variant": "baseline", + "throughput_seeds_per_second": 248.582174, + "wall_seconds": 0.8, + "user_seconds": 0.69, + "system_seconds": 0.03, + "max_rss_kib": 2044, + "minor_page_faults": 12354, + "major_page_faults": 1, + "voluntary_context_switches": 50, + "involuntary_context_switches": 786 + }, + { + "sequence": 1, + "variant": "exact-current", + "throughput_seeds_per_second": 4.735595, + "wall_seconds": 42.23, + "user_seconds": 42.04, + "system_seconds": 0.08, + "max_rss_kib": 5656, + "minor_page_faults": 11789, + "major_page_faults": 0, + "voluntary_context_switches": 626, + "involuntary_context_switches": 2898 + }, + { + "sequence": 2, + "variant": "exact-current", + "throughput_seeds_per_second": 4.746686, + "wall_seconds": 42.13, + "user_seconds": 41.98, + "system_seconds": 0.09, + "max_rss_kib": 5684, + "minor_page_faults": 11798, + "major_page_faults": 0, + "voluntary_context_switches": 630, + "involuntary_context_switches": 2781 + }, + { + "sequence": 3, + "variant": "baseline", + "throughput_seeds_per_second": 248.028807, + "wall_seconds": 0.8, + "user_seconds": 0.69, + "system_seconds": 0.05, + "max_rss_kib": 1980, + "minor_page_faults": 13683, + "major_page_faults": 0, + "voluntary_context_switches": 55, + "involuntary_context_switches": 838 + }, + { + "sequence": 4, + "variant": "baseline", + "throughput_seeds_per_second": 248.558563, + "wall_seconds": 0.8, + "user_seconds": 0.67, + "system_seconds": 0.04, + "max_rss_kib": 2044, + "minor_page_faults": 14145, + "major_page_faults": 0, + "voluntary_context_switches": 50, + "involuntary_context_switches": 810 + }, + { + "sequence": 5, + "variant": "exact-current", + "throughput_seeds_per_second": 4.735332, + "wall_seconds": 42.23, + "user_seconds": 42.02, + "system_seconds": 0.09, + "max_rss_kib": 5684, + "minor_page_faults": 11821, + "major_page_faults": 0, + "voluntary_context_switches": 626, + "involuntary_context_switches": 2994 + }, + { + "sequence": 6, + "variant": "exact-current", + "throughput_seeds_per_second": 4.735349, + "wall_seconds": 42.23, + "user_seconds": 41.99, + "system_seconds": 0.1, + "max_rss_kib": 5652, + "minor_page_faults": 11723, + "major_page_faults": 0, + "voluntary_context_switches": 631, + "involuntary_context_switches": 2767 + }, + { + "sequence": 7, + "variant": "baseline", + "throughput_seeds_per_second": 248.537383, + "wall_seconds": 0.8, + "user_seconds": 0.68, + "system_seconds": 0.04, + "max_rss_kib": 2044, + "minor_page_faults": 13640, + "major_page_faults": 0, + "voluntary_context_switches": 44, + "involuntary_context_switches": 823 + }, + { + "sequence": 8, + "variant": "baseline", + "throughput_seeds_per_second": 247.779547, + "wall_seconds": 0.8, + "user_seconds": 0.71, + "system_seconds": 0.03, + "max_rss_kib": 1980, + "minor_page_faults": 12524, + "major_page_faults": 0, + "voluntary_context_switches": 47, + "involuntary_context_switches": 811 + }, + { + "sequence": 9, + "variant": "exact-current", + "throughput_seeds_per_second": 4.735753, + "wall_seconds": 42.23, + "user_seconds": 42.06, + "system_seconds": 0.08, + "max_rss_kib": 5740, + "minor_page_faults": 11823, + "major_page_faults": 0, + "voluntary_context_switches": 627, + "involuntary_context_switches": 2568 + }, + { + "sequence": 10, + "variant": "exact-current", + "throughput_seeds_per_second": 4.735672, + "wall_seconds": 42.23, + "user_seconds": 42.0, + "system_seconds": 0.1, + "max_rss_kib": 5740, + "minor_page_faults": 11652, + "major_page_faults": 0, + "voluntary_context_switches": 628, + "involuntary_context_switches": 2677 + }, + { + "sequence": 11, + "variant": "baseline", + "throughput_seeds_per_second": 248.292693, + "wall_seconds": 0.8, + "user_seconds": 0.71, + "system_seconds": 0.03, + "max_rss_kib": 1980, + "minor_page_faults": 13787, + "major_page_faults": 0, + "voluntary_context_switches": 52, + "involuntary_context_switches": 829 + } + ], + "summary": { + "throughput": { + "baseline": { + "median": 248.415038, + "mad": 0.15533049999999093, + "p25": 248.0947785, + "p75": 248.55326799999997, + "minimum": 247.779547, + "maximum": 248.582174 + }, + "exact-current": { + "median": 4.7356335000000005, + "mad": 0.00020199999999981344, + "p25": 4.7354105, + "p75": 4.73573275, + "minimum": 4.735332, + "maximum": 4.746686 + } + }, + "paired_change_percent": { + "median": -98.09370971809756, + "mad": 0.0012106326215217678, + "p25": -98.09484047632034, + "p75": -98.08971886243113, + "minimum": -98.09495792727277, + "maximum": -98.0862360072554, + "bootstrap_95_percent_ci": [ + -98.09492035071908, + -98.08747960301346 + ] + }, + "performance_requirement": { + "maximum_elapsed_time_overhead": 0.05, + "measured_elapsed_time_overhead": 51.45655897991262, + "minimum_candidate_throughput": 236.58575047619047, + "passed": false + } + } +} diff --git a/benchmarks/results/2026-07-30-vein-calibration-midpoint.json b/benchmarks/results/2026-07-30-vein-calibration-midpoint.json new file mode 100644 index 0000000..0d52ebd --- /dev/null +++ b/benchmarks/results/2026-07-30-vein-calibration-midpoint.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "created_utc": "2026-07-30T16:41:39.213319+00:00", + "seed_set": "calibration", + "estimate_prefix": "estimate", + "corpus": { + "manifest": "benchmarks/vein_estimator_corpora.json", + "sha256": "6d8c5f92824ecdaa0ad0298c2f418ca38a59bb6cd23ff0b7c49d3b2c8c434a21", + "start_seed": 0, + "end_seed_exclusive": 1000 + }, + "provenance": { + "source_revision": "5eda1b60c58bb97a7ea836698f2db2eb910fac86", + "exporter": "inserter/target/x86_64-unknown-linux-musl/release/vein_accuracy", + "exporter_sha256": "6e6d8cb6c084a754fbc7634989bf4539b45f3615ccbacd8f16636023cd85ec2b" + }, + "input": { + "path": "/tmp/opencode/vein-estimator/calibration.csv", + "sha256": "1c1dcb36334eeded7f18533f036c57ef42a80901342eb161f27d6bf8edae2562", + "seed_count": 1000, + "planet_count": 244239 + }, + "scales": { + "path": null, + "sha256": null, + "denominator": 1 + }, + "overall": { + "all_amounts": { + "count": 3419346, + "reference_sum": 26151770250049, + "predicted_sum": 26621776675160, + "mean_absolute_error": 838148.7469893366, + "weighted_absolute_percentage_error": 0.10958801404343288, + "weighted_signed_bias": 0.017972260409794605 + }, + "non_null_reference": { + "count": 1394252, + "mean_absolute_error": 2055525.5186458402, + "mean_absolute_relative_error": 0.2030753235667965, + "p95_absolute_relative_error": 0.9139781310203802, + "mean_signed_error": 337102.9233675118 + }, + "null_reference": { + "count": 2025094, + "correct": 2025094, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 1394252, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "by_ore": { + "iron": { + "all_amounts": { + "count": 244239, + "reference_sum": 5708260912997, + "predicted_sum": 5837331669126, + "mean_absolute_error": 2192250.6165804807, + "weighted_absolute_percentage_error": 0.09379968899527445, + "weighted_signed_bias": 0.022611222243734855 + }, + "non_null_reference": { + "count": 207779, + "mean_absolute_error": 2576935.582243634, + "mean_absolute_relative_error": 0.19483515614757096, + "p95_absolute_relative_error": 0.9049680239691479, + "mean_signed_error": 621192.4984189932 + }, + "null_reference": { + "count": 36460, + "correct": 36460, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 207779, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "copper": { + "all_amounts": { + "count": 244239, + "reference_sum": 6015150214264, + "predicted_sum": 6167752665766, + "mean_absolute_error": 2140138.5412649084, + "weighted_absolute_percentage_error": 0.08689812865196368, + "weighted_signed_bias": 0.025369682562561256 + }, + "non_null_reference": { + "count": 207779, + "mean_absolute_error": 2515679.145534438, + "mean_absolute_relative_error": 0.17988289177877287, + "p95_absolute_relative_error": 0.8385694093549569, + "mean_signed_error": 734445.9810760472 + }, + "null_reference": { + "count": 36460, + "correct": 36460, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 207779, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "silicium": { + "all_amounts": { + "count": 244239, + "reference_sum": 2625376653954, + "predicted_sum": 2637020778252, + "mean_absolute_error": 1693166.1807983164, + "weighted_absolute_percentage_error": 0.15751538515785313, + "weighted_signed_bias": 0.004435220477969566 + }, + "non_null_reference": { + "count": 184775, + "mean_absolute_error": 2238058.259136788, + "mean_absolute_relative_error": 0.3060317134803064, + "p95_absolute_relative_error": 1.0607395342834436, + "mean_signed_error": 63017.85575970775 + }, + "null_reference": { + "count": 59464, + "correct": 59464, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 184775, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "titanium": { + "all_amounts": { + "count": 244239, + "reference_sum": 5491308738790, + "predicted_sum": 5590566592582, + "mean_absolute_error": 2388830.9500612104, + "weighted_absolute_percentage_error": 0.10624893084058523, + "weighted_signed_bias": 0.018075445856987324 + }, + "non_null_reference": { + "count": 183922, + "mean_absolute_error": 3172245.204010396, + "mean_absolute_relative_error": 0.12992644968950184, + "p95_absolute_relative_error": 0.32303927940793226, + "mean_signed_error": 539673.6322571525 + }, + "null_reference": { + "count": 60317, + "correct": 60317, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 183922, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "stone": { + "all_amounts": { + "count": 244239, + "reference_sum": 3602806854910, + "predicted_sum": 3629484914522, + "mean_absolute_error": 1765206.329087492, + "weighted_absolute_percentage_error": 0.11966565124701083, + "weighted_signed_bias": 0.007404798726760064 + }, + "non_null_reference": { + "count": 202477, + "mean_absolute_error": 2129289.8877897244, + "mean_absolute_relative_error": 0.23757499219382996, + "p95_absolute_relative_error": 0.9390206699050946, + "mean_signed_error": 131758.46941627937 + }, + "null_reference": { + "count": 41762, + "correct": 41762, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 202477, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "coal": { + "all_amounts": { + "count": 244239, + "reference_sum": 765040278335, + "predicted_sum": 802013405232, + "mean_absolute_error": 457191.66793591523, + "weighted_absolute_percentage_error": 0.14595837493422006, + "weighted_signed_bias": 0.048328340277020036 + }, + "non_null_reference": { + "count": 199704, + "mean_absolute_error": 559147.7175469695, + "mean_absolute_relative_error": 0.18793767128074823, + "p95_absolute_relative_error": 0.830386826597333, + "mean_signed_error": 185139.64115390778 + }, + "null_reference": { + "count": 44535, + "correct": 44535, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 199704, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "oil": { + "all_amounts": { + "count": 244239, + "reference_sum": 53799007611, + "predicted_sum": 53918038152, + "mean_absolute_error": 10209.829957541588, + "weighted_absolute_percentage_error": 0.046351015933798353, + "weighted_signed_bias": 0.0022125043989782156 + }, + "non_null_reference": { + "count": 22234, + "mean_absolute_error": 112154.29787712512, + "mean_absolute_relative_error": 0.048367751272539866, + "p95_absolute_relative_error": 0.11049076463864003, + "mean_signed_error": 5353.536970405685 + }, + "null_reference": { + "count": 222005, + "correct": 222005, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 22234, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "fireice": { + "all_amounts": { + "count": 244239, + "reference_sum": 900623606178, + "predicted_sum": 903476881789, + "mean_absolute_error": 474657.6639889616, + "weighted_absolute_percentage_error": 0.12872182385599998, + "weighted_signed_bias": 0.003168111063742289 + }, + "non_null_reference": { + "count": 59554, + "mean_absolute_error": 1946635.2082983512, + "mean_absolute_relative_error": 0.19780048807264897, + "p95_absolute_relative_error": 0.8869417353188125, + "mean_signed_error": 47910.729942573125 + }, + "null_reference": { + "count": 184685, + "correct": 184685, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 59554, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "diamond": { + "all_amounts": { + "count": 244239, + "reference_sum": 409429398141, + "predicted_sum": 410306195058, + "mean_absolute_error": 265918.1436584657, + "weighted_absolute_percentage_error": 0.15862950189676717, + "weighted_signed_bias": 0.002141509429906758 + }, + "non_null_reference": { + "count": 42877, + "mean_absolute_error": 1514741.7377381814, + "mean_absolute_relative_error": 0.21371999432351707, + "p95_absolute_relative_error": 0.9284616476286238, + "mean_signed_error": 20449.119971080065 + }, + "null_reference": { + "count": 201362, + "correct": 201362, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 42877, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "fractal": { + "all_amounts": { + "count": 244239, + "reference_sum": 102123967023, + "predicted_sum": 103740906022, + "mean_absolute_error": 81036.93327846903, + "weighted_absolute_percentage_error": 0.193807390409564, + "weighted_signed_bias": 0.01583310016380228 + }, + "non_null_reference": { + "count": 18699, + "mean_absolute_error": 1058472.6213701267, + "mean_absolute_relative_error": 0.2192910194855938, + "p95_absolute_relative_error": 0.9562855382140969, + "mean_signed_error": 86471.95031819884 + }, + "null_reference": { + "count": 225540, + "correct": 225540, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 18699, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "crysrub": { + "all_amounts": { + "count": 244239, + "reference_sum": 90881746220, + "predicted_sum": 97224785050, + "mean_absolute_error": 80539.90338971253, + "weighted_absolute_percentage_error": 0.21644594522184787, + "weighted_signed_bias": 0.06979442070407876 + }, + "non_null_reference": { + "count": 20606, + "mean_absolute_error": 954624.1611181209, + "mean_absolute_relative_error": 0.25963550135124874, + "p95_absolute_relative_error": 1.1038335589161048, + "mean_signed_error": 307824.84858779 + }, + "null_reference": { + "count": 223633, + "correct": 223633, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 20606, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "grat": { + "all_amounts": { + "count": 244239, + "reference_sum": 252784775375, + "predicted_sum": 252805153750, + "mean_absolute_error": 128624.99975433899, + "weighted_absolute_percentage_error": 0.1242766352063579, + "weighted_signed_bias": 8.061551558937512e-05 + }, + "non_null_reference": { + "count": 28765, + "mean_absolute_error": 1092134.2365722232, + "mean_absolute_relative_error": 0.16953283472412708, + "p95_absolute_relative_error": 0.8877518337016752, + "mean_signed_error": 708.4434208239179 + }, + "null_reference": { + "count": 215474, + "correct": 215474, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 28765, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "bamboo": { + "all_amounts": { + "count": 244239, + "reference_sum": 125109074240, + "predicted_sum": 127011016187, + "mean_absolute_error": 49923.88614840382, + "weighted_absolute_percentage_error": 0.09746183562680001, + "weighted_signed_bias": 0.015202270167481657 + }, + "non_null_reference": { + "count": 13081, + "mean_absolute_error": 932142.8047549882, + "mean_absolute_relative_error": 0.18699698712239582, + "p95_absolute_relative_error": 0.9353056335488524, + "mean_signed_error": 145397.28973320083 + }, + "null_reference": { + "count": 231158, + "correct": 231158, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 13081, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "mag": { + "all_amounts": { + "count": 244239, + "reference_sum": 9075022011, + "predicted_sum": 9123673672, + "mean_absolute_error": 6386.811946495031, + "weighted_absolute_percentage_error": 0.17189033383161015, + "weighted_signed_bias": 0.005361051570015856 + }, + "non_null_reference": { + "count": 2000, + "mean_absolute_error": 779954.2815, + "mean_absolute_relative_error": 0.20436673844717915, + "p95_absolute_relative_error": 0.9219429800908081, + "mean_signed_error": 24325.8305 + }, + "null_reference": { + "count": 242239, + "correct": 242239, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 2000, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + } + }, + "accuracy_requirements": { + "overall_wape_at_most_0_10": false, + "each_ore_wape_at_most_0_15": false, + "overall_non_null_p95_are_at_most_0_25": false, + "each_ore_existence_precision_and_recall_at_least_0_99": true, + "passed": false + } +} diff --git a/benchmarks/results/2026-07-30-vein-calibration-spacing.json b/benchmarks/results/2026-07-30-vein-calibration-spacing.json new file mode 100644 index 0000000..f6698c8 --- /dev/null +++ b/benchmarks/results/2026-07-30-vein-calibration-spacing.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "created_utc": "2026-07-30T16:41:40.293096+00:00", + "seed_set": "calibration", + "estimate_prefix": "spacing", + "corpus": { + "manifest": "benchmarks/vein_estimator_corpora.json", + "sha256": "6d8c5f92824ecdaa0ad0298c2f418ca38a59bb6cd23ff0b7c49d3b2c8c434a21", + "start_seed": 0, + "end_seed_exclusive": 1000 + }, + "provenance": { + "source_revision": "5eda1b60c58bb97a7ea836698f2db2eb910fac86", + "exporter": "inserter/target/x86_64-unknown-linux-musl/release/vein_accuracy", + "exporter_sha256": "6e6d8cb6c084a754fbc7634989bf4539b45f3615ccbacd8f16636023cd85ec2b" + }, + "input": { + "path": "/tmp/opencode/vein-estimator/calibration-rng.csv", + "sha256": "4af4707d10a6793a28be9ccfbf95462a6408ad76081a5b9ef7756ead9e04aaf7", + "seed_count": 1000, + "planet_count": 244239 + }, + "scales": { + "path": null, + "sha256": null, + "denominator": 1 + }, + "overall": { + "all_amounts": { + "count": 3419346, + "reference_sum": 26151770250049, + "predicted_sum": 26623761059238, + "mean_absolute_error": 507946.0169959402, + "weighted_absolute_percentage_error": 0.06641398134138723, + "weighted_signed_bias": 0.018048139941429613 + }, + "non_null_reference": { + "count": 1394252, + "mean_absolute_error": 1245716.8298349222, + "mean_absolute_relative_error": 0.13683443249121127, + "p95_absolute_relative_error": 0.5790738404849181, + "mean_signed_error": 338526.18406787293 + }, + "null_reference": { + "count": 2025094, + "correct": 2025094, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 1394252, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "by_ore": { + "iron": { + "all_amounts": { + "count": 244239, + "reference_sum": 5708260912997, + "predicted_sum": 5837763070128, + "mean_absolute_error": 829681.7390547784, + "weighted_absolute_percentage_error": 0.03549954028969007, + "weighted_signed_bias": 0.022686797100696588 + }, + "non_null_reference": { + "count": 207779, + "mean_absolute_error": 975270.0622536445, + "mean_absolute_relative_error": 0.039814299603461965, + "p95_absolute_relative_error": 0.11844994026306531, + "mean_signed_error": 623268.747712714 + }, + "null_reference": { + "count": 36460, + "correct": 36460, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 207779, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "copper": { + "all_amounts": { + "count": 244239, + "reference_sum": 6015150214264, + "predicted_sum": 6167810010927, + "mean_absolute_error": 1452015.181109487, + "weighted_absolute_percentage_error": 0.05895758595986996, + "weighted_signed_bias": 0.025379216017081478 + }, + "non_null_reference": { + "count": 207779, + "mean_absolute_error": 1706807.4050746225, + "mean_absolute_relative_error": 0.12019826871808767, + "p95_absolute_relative_error": 0.506834510487662, + "mean_signed_error": 734721.9722060459 + }, + "null_reference": { + "count": 36460, + "correct": 36460, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 207779, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "silicium": { + "all_amounts": { + "count": 244239, + "reference_sum": 2625376653954, + "predicted_sum": 2638623783960, + "mean_absolute_error": 825620.20497136, + "weighted_absolute_percentage_error": 0.0768075136717252, + "weighted_signed_bias": 0.00504580170850872 + }, + "non_null_reference": { + "count": 184775, + "mean_absolute_error": 1091320.001309701, + "mean_absolute_relative_error": 0.18942066817226808, + "p95_absolute_relative_error": 0.9020408934782139, + "mean_signed_error": 71693.3026978758 + }, + "null_reference": { + "count": 59464, + "correct": 59464, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 184775, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "titanium": { + "all_amounts": { + "count": 244239, + "reference_sum": 5491308738790, + "predicted_sum": 5591923675016, + "mean_absolute_error": 1594174.769926179, + "weighted_absolute_percentage_error": 0.07090470963353532, + "weighted_signed_bias": 0.01832257864418862 + }, + "non_null_reference": { + "count": 183922, + "mean_absolute_error": 2116982.4797033523, + "mean_absolute_relative_error": 0.0892832181015097, + "p95_absolute_relative_error": 0.3268225439534779, + "mean_signed_error": 547052.2081425822 + }, + "null_reference": { + "count": 60317, + "correct": 60317, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 183922, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "stone": { + "all_amounts": { + "count": 244239, + "reference_sum": 3602806854910, + "predicted_sum": 3628174937354, + "mean_absolute_error": 950954.4146102793, + "weighted_absolute_percentage_error": 0.06446644647449523, + "weighted_signed_bias": 0.0070411996717025535 + }, + "non_null_reference": { + "count": 202477, + "mean_absolute_error": 1147094.0169500734, + "mean_absolute_relative_error": 0.16830296152283789, + "p95_absolute_relative_error": 0.6799221075150149, + "mean_signed_error": 125288.71152772907 + }, + "null_reference": { + "count": 41762, + "correct": 41762, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 202477, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "coal": { + "all_amounts": { + "count": 244239, + "reference_sum": 765040278335, + "predicted_sum": 801008493679, + "mean_absolute_error": 462328.86049320543, + "weighted_absolute_percentage_error": 0.147598422927158, + "weighted_signed_bias": 0.04701479956359898 + }, + "non_null_reference": { + "count": 199704, + "mean_absolute_error": 565430.529974362, + "mean_absolute_relative_error": 0.1902782124223807, + "p95_absolute_relative_error": 0.673210665133803, + "mean_signed_error": 180107.63602131154 + }, + "null_reference": { + "count": 44535, + "correct": 44535, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 199704, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "oil": { + "all_amounts": { + "count": 244239, + "reference_sum": 53799007611, + "predicted_sum": 53879385296, + "mean_absolute_error": 10295.023223154369, + "weighted_absolute_percentage_error": 0.04673777990815363, + "weighted_signed_bias": 0.00149403657370746 + }, + "non_null_reference": { + "count": 22234, + "mean_absolute_error": 113090.14019069893, + "mean_absolute_relative_error": 0.04970875016709826, + "p95_absolute_relative_error": 0.13759510393053642, + "mean_signed_error": 3615.079832688675 + }, + "null_reference": { + "count": 222005, + "correct": 222005, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 22234, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "fireice": { + "all_amounts": { + "count": 244239, + "reference_sum": 900623606178, + "predicted_sum": 904637028568, + "mean_absolute_error": 355257.8374461081, + "weighted_absolute_percentage_error": 0.09634193281721636, + "weighted_signed_bias": 0.004456270480219662 + }, + "non_null_reference": { + "count": 59554, + "mean_absolute_error": 1456960.3882190953, + "mean_absolute_relative_error": 0.1491404620178326, + "p95_absolute_relative_error": 0.628773720708769, + "mean_signed_error": 67391.31527689156 + }, + "null_reference": { + "count": 184685, + "correct": 184685, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 59554, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "diamond": { + "all_amounts": { + "count": 244239, + "reference_sum": 409429398141, + "predicted_sum": 410984532234, + "mean_absolute_error": 288293.70428555633, + "weighted_absolute_percentage_error": 0.1719773088124737, + "weighted_signed_bias": 0.0037982961166468077 + }, + "non_null_reference": { + "count": 42877, + "mean_absolute_error": 1642198.9887585419, + "mean_absolute_relative_error": 0.18472879814363524, + "p95_absolute_relative_error": 0.6784244873271325, + "mean_signed_error": 36269.65722881731 + }, + "null_reference": { + "count": 201362, + "correct": 201362, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 42877, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "fractal": { + "all_amounts": { + "count": 244239, + "reference_sum": 102123967023, + "predicted_sum": 103514989866, + "mean_absolute_error": 64583.37230745295, + "weighted_absolute_percentage_error": 0.1544571634731687, + "weighted_signed_bias": 0.013620924485696084 + }, + "non_null_reference": { + "count": 18699, + "mean_absolute_error": 843562.6647949088, + "mean_absolute_relative_error": 0.1612244505187301, + "p95_absolute_relative_error": 0.6613015800791258, + "mean_signed_error": 74390.22637574202 + }, + "null_reference": { + "count": 225540, + "correct": 225540, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 18699, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "crysrub": { + "all_amounts": { + "count": 244239, + "reference_sum": 90881746220, + "predicted_sum": 96978787459, + "mean_absolute_error": 100777.26374985158, + "weighted_absolute_percentage_error": 0.27083258349170397, + "weighted_signed_bias": 0.06708763302413578 + }, + "non_null_reference": { + "count": 20606, + "mean_absolute_error": 1194493.7455595457, + "mean_absolute_relative_error": 0.2926618166245399, + "p95_absolute_relative_error": 1.1992458245099364, + "mean_signed_error": 295886.69508880907 + }, + "null_reference": { + "count": 223633, + "correct": 223633, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 20606, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "grat": { + "all_amounts": { + "count": 244239, + "reference_sum": 252784775375, + "predicted_sum": 252386662721, + "mean_absolute_error": 134322.97572459764, + "weighted_absolute_percentage_error": 0.1297819823972063, + "weighted_signed_bias": -0.0015749075608268324 + }, + "non_null_reference": { + "count": 28765, + "mean_absolute_error": 1140514.8363636364, + "mean_absolute_relative_error": 0.15185353969604998, + "p95_absolute_relative_error": 0.6285536099213467, + "mean_signed_error": -13840.175699634972 + }, + "null_reference": { + "count": 215474, + "correct": 215474, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 28765, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "bamboo": { + "all_amounts": { + "count": 244239, + "reference_sum": 125109074240, + "predicted_sum": 126883965240, + "mean_absolute_error": 37518.7924778598, + "weighted_absolute_percentage_error": 0.07324450613727121, + "weighted_signed_bias": 0.014186748729314233 + }, + "non_null_reference": { + "count": 13081, + "mean_absolute_error": 700523.8403791758, + "mean_absolute_relative_error": 0.18056166164950355, + "p95_absolute_relative_error": 0.8291640407431253, + "mean_signed_error": 135684.65713630457 + }, + "null_reference": { + "count": 231158, + "correct": 231158, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 13081, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + }, + "mag": { + "all_amounts": { + "count": 244239, + "reference_sum": 9075022011, + "predicted_sum": 9191736790, + "mean_absolute_error": 5420.098563292513, + "weighted_absolute_percentage_error": 0.1458728641534311, + "weighted_signed_bias": 0.012861101478159269 + }, + "non_null_reference": { + "count": 2000, + "mean_absolute_error": 661899.7265, + "mean_absolute_relative_error": 0.17660359907930578, + "p95_absolute_relative_error": 0.6780290203939799, + "mean_signed_error": 58357.3895 + }, + "null_reference": { + "count": 242239, + "correct": 242239, + "false_positive": 0, + "accuracy": 1.0 + }, + "existence": { + "true_positive": 2000, + "false_positive": 0, + "false_negative": 0, + "precision": 1.0, + "recall": 1.0 + } + } + }, + "accuracy_requirements": { + "overall_wape_at_most_0_10": true, + "each_ore_wape_at_most_0_15": false, + "overall_non_null_p95_are_at_most_0_25": false, + "each_ore_existence_precision_and_recall_at_least_0_99": true, + "passed": false + } +} diff --git a/benchmarks/results/2026-07-30-vein-estimator-benchmark.md b/benchmarks/results/2026-07-30-vein-estimator-benchmark.md new file mode 100644 index 0000000..996c20f --- /dev/null +++ b/benchmarks/results/2026-07-30-vein-estimator-benchmark.md @@ -0,0 +1,75 @@ +# Vein Estimator Benchmark Record: 2026-07-30 + +## Purpose + +Record the rejected vein-estimator experiment, its running configuration, and +the production decision for future comparison. This is a benchmark and +debugging record, not a production performance claim. + +## Running Configuration + +| Setting | Value | +| --- | --- | +| Calibration corpus | Seeds 0 through 999 | +| Held-out corpus | Seeds 1000000 through 1000999; not opened | +| Performance corpus | Seeds 2000000 through 2000199 | +| Corpus SHA-256 | `6d8c5f92824ecdaa0ad0298c2f418ca38a59bb6cd23ff0b7c49d3b2c8c434a21` | +| Stars per seed | 64 | +| Resource multiplier | 1.0 | +| Builder | `rust:1.97.0@sha256:b92b8c8574f8f3b207fcb0912fb3e2de4041580b5934d90312d53938c9a038a9` | +| Target | `x86_64-unknown-linux-musl` | +| Processor | Intel Xeon Platinum 8370C at 2.80 GHz | +| Available processors | 2 logical processors, 1 physical core | +| Benchmark workers | 1 | +| Processor set | 0 | +| Comparison order | A, B, B, A for three rounds | +| Disabled during comparison | UI, logs, checkpoints, database writes | + +## Provenance + +| Item | Source revision | SHA-256 | +| --- | --- | --- | +| Historical fast baseline | `0df70ce645d4573ea135ea179a1261890851ec00` | `2b078c659e56a4ffaa31e905170a3aa96c363294db28f87a2241db76e2d4ba5e` | +| Exact baseline | `59232b78b5e8065b5697fe6bf6cd253c3a15f9e9` | `17445fbe6f040420e974e197a7d86e841c03263079daeaaaeaa190113552e5fa` | +| Validation tooling | `5eda1b60c58bb97a7ea836698f2db2eb910fac86` | exporter: `6e6d8cb6c084a754fbc7634989bf4539b45f3615ccbacd8f16636023cd85ec2b` | + +The production generator's runtime behavior remains the exact baseline. + +## Results + +| Measurement | Historical fast median | Exact median | Change | +| --- | ---: | ---: | ---: | +| Initial comparison | 248.415 seeds/s | 4.736 seeds/s | -98.09% | +| Repeated comparison | 248.325 seeds/s | 4.691 seeds/s | -98.11% | + +The 5% elapsed-time-overhead limit requires a candidate throughput of at least +236.500 seeds/s. The exact generator is 52.94 times slower than the historical +fast baseline. + +| Candidate | Overall WAPE | P95 absolute relative error | Outcome | +| --- | ---: | ---: | --- | +| Historical midpoint | 10.96% | 91.40% | Rejected | +| Spacing replay | 6.64% | 57.91% | Rejected | + +The spacing replay also exceeded the 15% per-ore WAPE limit for Kimberlite ore, +fractal silicon, and organic crystal. The required limits are overall WAPE at +most 10%, WAPE at most 15% for every ore, P95 absolute relative error at most +25%, and existence precision and recall at least 99% for every ore. + +## Decision And Lessons + +No estimator was shipped. The spacing replay improved overall WAPE, but replaying +group counts and spacing without terrain did not control tail error or rare-ore +error. The held-out corpus remains protected, and candidate performance was not +measured because calibration already failed. + +Future estimator work should use the same corpus manifest, source/exporter +provenance, calibration gates, held-out protection, and performance comparison. + +## Detailed Records + +- [Study and reproduction guide](../../docs/vein-estimator-study.md) +- [Initial raw comparison](2026-07-30-vein-baselines.json) +- [Repeated raw comparison](2026-07-30-vein-final-comparison.json) +- [Historical midpoint calibration](2026-07-30-vein-calibration-midpoint.json) +- [Spacing replay calibration](2026-07-30-vein-calibration-spacing.json) diff --git a/benchmarks/results/2026-07-30-vein-final-comparison.json b/benchmarks/results/2026-07-30-vein-final-comparison.json new file mode 100644 index 0000000..112f17c --- /dev/null +++ b/benchmarks/results/2026-07-30-vein-final-comparison.json @@ -0,0 +1,241 @@ +{ + "schema_version": 1, + "build": { + "rust": "rustc 1.97.0 (2d8144b78 2026-07-07)", + "cargo": "cargo 1.97.0 (c980f4866 2026-06-30)", + "builder_image": "rust:1.97.0@sha256:b92b8c8574f8f3b207fcb0912fb3e2de4041580b5934d90312d53938c9a038a9", + "target": "x86_64-unknown-linux-musl", + "profile": "release" + }, + "created_utc": "2026-07-30T16:02:05.312280+00:00", + "host": { + "platform": "Linux-6.17.0-1021-azure-x86_64-with-glibc2.39", + "processor_count": 2, + "processor": "Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz", + "physical_cores": 1, + "memory_bytes": 8270495744, + "docker": "29.6.1", + "cpu_set": "0" + }, + "workload": { + "corpus_manifest": "benchmarks/vein_estimator_corpora.json", + "corpus_manifest_sha256": "6d8c5f92824ecdaa0ad0298c2f418ca38a59bb6cd23ff0b7c49d3b2c8c434a21", + "seed_set": "performance", + "start_seed": 2000000, + "end_seed": 2000200, + "seed_count": 200, + "workers": 1, + "writer_sinks": 1, + "channel_size": 64 + }, + "executables": { + "baseline": { + "path": "/tmp/opencode/dsp-vein-fast-0df70/inserter/target/x86_64-unknown-linux-musl/release/dsp_seed_finder", + "sha256": "2b078c659e56a4ffaa31e905170a3aa96c363294db28f87a2241db76e2d4ba5e", + "source_commit": "0df70ce645d4573ea135ea179a1261890851ec00", + "vein_mode": "historical midpoint estimate without terrain or placement" + }, + "exact-final": { + "path": "/home/ubuntu/DSPSeedDatabase/inserter/target/x86_64-unknown-linux-musl/release/dsp_seed_finder", + "sha256": "17445fbe6f040420e974e197a7d86e841c03263079daeaaaeaa190113552e5fa", + "source_commit": "59232b78b5e8065b5697fe6bf6cd253c3a15f9e9", + "vein_mode": "exact terrain and vein placement" + } + }, + "results": [ + { + "sequence": 0, + "variant": "baseline", + "throughput_seeds_per_second": 249.252103, + "wall_seconds": 0.8, + "user_seconds": 0.69, + "system_seconds": 0.03, + "max_rss_kib": 2044, + "minor_page_faults": 12340, + "major_page_faults": 0, + "voluntary_context_switches": 47, + "involuntary_context_switches": 768 + }, + { + "sequence": 1, + "variant": "exact-final", + "throughput_seeds_per_second": 4.691188, + "wall_seconds": 42.63, + "user_seconds": 42.3, + "system_seconds": 0.11, + "max_rss_kib": 5696, + "minor_page_faults": 11382, + "major_page_faults": 0, + "voluntary_context_switches": 630, + "involuntary_context_switches": 2877 + }, + { + "sequence": 2, + "variant": "exact-final", + "throughput_seeds_per_second": 4.71291, + "wall_seconds": 42.43, + "user_seconds": 42.29, + "system_seconds": 0.07, + "max_rss_kib": 5604, + "minor_page_faults": 11813, + "major_page_faults": 0, + "voluntary_context_switches": 631, + "involuntary_context_switches": 2624 + }, + { + "sequence": 3, + "variant": "baseline", + "throughput_seeds_per_second": 248.188785, + "wall_seconds": 0.8, + "user_seconds": 0.68, + "system_seconds": 0.05, + "max_rss_kib": 1980, + "minor_page_faults": 13717, + "major_page_faults": 0, + "voluntary_context_switches": 41, + "involuntary_context_switches": 817 + }, + { + "sequence": 4, + "variant": "baseline", + "throughput_seeds_per_second": 248.741973, + "wall_seconds": 0.8, + "user_seconds": 0.67, + "system_seconds": 0.03, + "max_rss_kib": 1980, + "minor_page_faults": 12124, + "major_page_faults": 0, + "voluntary_context_switches": 42, + "involuntary_context_switches": 773 + }, + { + "sequence": 5, + "variant": "exact-final", + "throughput_seeds_per_second": 4.669216, + "wall_seconds": 42.83, + "user_seconds": 42.54, + "system_seconds": 0.11, + "max_rss_kib": 5684, + "minor_page_faults": 11707, + "major_page_faults": 0, + "voluntary_context_switches": 633, + "involuntary_context_switches": 2933 + }, + { + "sequence": 6, + "variant": "exact-final", + "throughput_seeds_per_second": 4.680233, + "wall_seconds": 42.73, + "user_seconds": 42.51, + "system_seconds": 0.13, + "max_rss_kib": 5696, + "minor_page_faults": 11774, + "major_page_faults": 0, + "voluntary_context_switches": 634, + "involuntary_context_switches": 2859 + }, + { + "sequence": 7, + "variant": "baseline", + "throughput_seeds_per_second": 247.944041, + "wall_seconds": 0.8, + "user_seconds": 0.7, + "system_seconds": 0.03, + "max_rss_kib": 2044, + "minor_page_faults": 12458, + "major_page_faults": 0, + "voluntary_context_switches": 45, + "involuntary_context_switches": 799 + }, + { + "sequence": 8, + "variant": "baseline", + "throughput_seeds_per_second": 248.06408, + "wall_seconds": 0.8, + "user_seconds": 0.69, + "system_seconds": 0.03, + "max_rss_kib": 1980, + "minor_page_faults": 11953, + "major_page_faults": 0, + "voluntary_context_switches": 29, + "involuntary_context_switches": 766 + }, + { + "sequence": 9, + "variant": "exact-final", + "throughput_seeds_per_second": 4.690959, + "wall_seconds": 42.63, + "user_seconds": 42.36, + "system_seconds": 0.11, + "max_rss_kib": 5696, + "minor_page_faults": 11732, + "major_page_faults": 0, + "voluntary_context_switches": 633, + "involuntary_context_switches": 3026 + }, + { + "sequence": 10, + "variant": "exact-final", + "throughput_seeds_per_second": 4.724479, + "wall_seconds": 42.33, + "user_seconds": 42.16, + "system_seconds": 0.07, + "max_rss_kib": 5684, + "minor_page_faults": 11767, + "major_page_faults": 0, + "voluntary_context_switches": 629, + "involuntary_context_switches": 2493 + }, + { + "sequence": 11, + "variant": "baseline", + "throughput_seeds_per_second": 248.461737, + "wall_seconds": 0.8, + "user_seconds": 0.69, + "system_seconds": 0.04, + "max_rss_kib": 1980, + "minor_page_faults": 13682, + "major_page_faults": 0, + "voluntary_context_switches": 47, + "involuntary_context_switches": 816 + } + ], + "summary": { + "throughput": { + "baseline": { + "median": 248.325261, + "mad": 0.32120050000001754, + "p25": 248.09525624999998, + "p75": 248.671914, + "minimum": 247.944041, + "maximum": 249.252103 + }, + "exact-final": { + "median": 4.6910735, + "mad": 0.01633849999999981, + "p25": 4.6829145, + "p75": 4.7074795, + "minimum": 4.669216, + "maximum": 4.724479 + } + }, + "paired_change_percent": { + "median": -98.11067812071981, + "mad": 0.00840786752820577, + "p25": -98.1165165739744, + "p75": -98.10305215792123, + "minimum": -98.12286766737192, + "maximum": -98.09850842345193, + "bootstrap_95_percent_ci": [ + -98.12038099129529, + -98.0997935018071 + ] + }, + "performance_requirement": { + "maximum_elapsed_time_overhead": 0.05, + "measured_elapsed_time_overhead": 51.93570032530934, + "minimum_candidate_throughput": 236.50024857142859, + "passed": false + } + } +} diff --git a/docs/vein-estimator-study.md b/docs/vein-estimator-study.md new file mode 100644 index 0000000..3076a0a --- /dev/null +++ b/docs/vein-estimator-study.md @@ -0,0 +1,277 @@ +# Planet Vein Estimator Validation Tools + +## Purpose + +This change adds developer tooling for exporting exact planet vein amounts, +measuring deterministic estimator candidates, and comparing process throughput. +It does not change production generation. The current study records a rejected +candidate so future work can reproduce the baselines and use the protected +held-out corpus correctly. + +## Study Outcome + +The release gate did not pass. The production generator still uses exact terrain +and vein placement. This decision prevents the release of an inaccurate +estimator. + +The best fast calibration candidate had 6.64% weighted absolute percentage +error (WAPE). However, its P95 absolute relative error was 57.91%. Kimberlite +ore, fractal silicon, and organic crystal also had more than 15% WAPE. The +required limits are 10% overall WAPE, 15% WAPE for each ore, and 25% P95 error. + +The held-out seed set was not opened. A candidate must pass all calibration +gates before the held-out test. Thus, the held-out set remains valid for future +work. + +## Tooling + +- `inserter/src/bin/vein_accuracy.rs` exports historical midpoint estimates, + the spacing candidate, and exact actual amounts without a database. +- `benchmarks/evaluate_vein_accuracy.py` validates a locked seed corpus and + reports amount, NULL, and existence metrics for an exporter CSV. +- `benchmarks/fit_vein_scales.py` generates fixed-point theme and ore scales + from the calibration corpus. +- `benchmarks/run_comparison.py` records controlled A-B-B-A process runs and + reports the maximum 5% elapsed-time-overhead requirement. + +The exporter and evaluator reject the held-out seed set unless the caller passes +`--allow-held-out`. The held-out range must only be opened after all calibration +accuracy gates pass. + +## Workflow + +1. Build `vein_accuracy` with the locked dependencies and the musl target. +2. Export the calibration range, then evaluate an estimator column against the + exact `actual_*` columns. The evaluator verifies that the CSV contains every + seed in the selected corpus and records hashes for the corpus, input, and + exporter. +3. Optionally fit fixed-point theme and ore scales from the historical + `estimate_*` columns, then re-evaluate with the generated scale artifact. +4. Open the held-out corpus only if every calibration accuracy gate passes. +5. Run the process comparison on the performance corpus. A candidate must meet + the elapsed-time-overhead requirement as well as the accuracy requirements. + +The evaluator reports the following accuracy requirements: overall WAPE at most +10%; WAPE at most 15% for every ore; non-null-reference P95 absolute relative +error at most 25%; and precision and recall at least 99% for the existence of +every ore. The comparison tool requires at most 5% elapsed-time overhead, which +is equivalent to candidate throughput of at least baseline throughput divided by +1.05. + +## Test Inputs + +The file [`vein_estimator_corpora.json`](../benchmarks/vein_estimator_corpora.json) +defines three separate seed sets. Its SHA-256 value is +`6d8c5f92824ecdaa0ad0298c2f418ca38a59bb6cd23ff0b7c49d3b2c8c434a21`. + +| Set | Seeds | Use | +| --- | --- | --- | +| Calibration | 0 through 999 | Model measurements | +| Held-out validation | 1000000 through 1000999 | Not opened | +| Performance | 2000000 through 2000199 | Controlled process tests | + +Each seed has 64 stars and a resource multiplier of 1.0. + +## Test System + +| Item | Value | +| --- | --- | +| Processor | Intel Xeon Platinum 8370C at 2.80 GHz | +| Available processors | 2 logical processors, 1 physical core | +| Memory | 8,270,495,744 bytes | +| Kernel | Linux 6.17.0-1021-azure | +| Docker | 29.6.1 | +| Rust | 1.97.0 | +| Cargo | 1.97.0 | +| Target | `x86_64-unknown-linux-musl` | +| Worker count | 1 | +| Processor set | 0 | + +The build used the locked dependencies and the release profile. The builder was +`rust:1.97.0@sha256:b92b8c8574f8f3b207fcb0912fb3e2de4041580b5934d90312d53938c9a038a9`. + +## Baselines + +The fast baseline is commit +`0df70ce645d4573ea135ea179a1261890851ec00`. This is the last buildable source +version before precise planet ores entered the main branch. It calculates +midpoint estimates and does not simulate terrain or vein placement. Its +executable SHA-256 value is +`2b078c659e56a4ffaa31e905170a3aa96c363294db28f87a2241db76e2d4ba5e`. + +The exact baseline is commit +`59232b78b5e8065b5697fe6bf6cd253c3a15f9e9`. Its executable SHA-256 value is +`17445fbe6f040420e974e197a7d86e841c03263079daeaaaeaa190113552e5fa`. + +The test used the sequence `A, B, B, A` for three rounds. It disabled the user +interface, logs, checkpoints, and database writes. + +| Test | Fast median | Exact median | Exact change | +| --- | ---: | ---: | ---: | +| Initial | 248.415 seeds/s | 4.736 seeds/s | -98.09% | +| Final repeat | 248.325 seeds/s | 4.691 seeds/s | -98.11% | + +For a maximum 5% increase in elapsed time, the final throughput limit is +236.500 seeds/s (`248.325 / 1.05`). The exact implementation is 52.94 times +slower than the fast baseline. The spacing candidate did not receive a release +performance test because it failed the calibration accuracy gates first. + +Raw results are in +[`2026-07-30-vein-baselines.json`](../benchmarks/results/2026-07-30-vein-baselines.json) +and +[`2026-07-30-vein-final-comparison.json`](../benchmarks/results/2026-07-30-vein-final-comparison.json). + +## Candidate Algorithm + +The historical midpoint candidate uses theme spot count, patch count, vein +opacity, star resource coefficients, and rare-ore rolls. It does not use +terrain. Its calibration result was 10.96% WAPE and 91.40% P95 relative error. + +The best fast candidate also replays these operations: + +1. It starts the placement random-number stream from the planet seed. +2. It applies the exact minus-one, zero, or plus-one group-count roll. +3. It creates candidate group directions. +4. It rejects directions that are too close to an earlier group. +5. It accepts terrain without a height query. +6. It scales the midpoint amount by accepted groups divided by nominal groups. + +The candidate uses a fixed stack array for 512 group centers. It does not +allocate center storage. The file `vein_accuracy.rs` contains this calibration +implementation. Production does not call it. + +The exact group-count roll is important. For a nominal count of two, the exact +result commonly has one, two, or three groups. A midpoint cannot predict this +three-mode result. Terrain rejection also changes the random-number position +for later ores. + +Exploratory terrain candidates also failed the accuracy limits and were removed +before the final measurements. + +## Calibration Accuracy + +The table reports the best fast candidate. MAE includes null reference amounts +as zero. Mean absolute relative error and P95 absolute relative error use +non-null reference amounts. Signed bias is +`sum(estimate - exact) / sum(exact)`. + +| Ore | MAE | WAPE | Mean absolute relative error | P95 absolute relative error | Signed bias | Precision | Recall | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Iron | 829682 | 3.55% | 3.98% | 11.84% | +2.27% | 100.00% | 100.00% | +| Copper | 1452015 | 5.90% | 12.02% | 50.68% | +2.54% | 100.00% | 100.00% | +| Silicon | 825620 | 7.68% | 18.94% | 90.20% | +0.50% | 100.00% | 100.00% | +| Titanium | 1594175 | 7.09% | 8.93% | 32.68% | +1.83% | 100.00% | 100.00% | +| Stone | 950954 | 6.45% | 16.83% | 67.99% | +0.70% | 100.00% | 100.00% | +| Coal | 462329 | 14.76% | 19.03% | 67.32% | +4.70% | 100.00% | 100.00% | +| Oil | 10295 | 4.67% | 4.97% | 13.76% | +0.15% | 100.00% | 100.00% | +| Fire ice | 355258 | 9.63% | 14.91% | 62.88% | +0.45% | 100.00% | 100.00% | +| Kimberlite ore | 288294 | 17.20% | 18.47% | 67.84% | +0.38% | 100.00% | 100.00% | +| Fractal silicon | 64583 | 15.45% | 16.12% | 66.13% | +1.36% | 100.00% | 100.00% | +| Organic crystal | 100777 | 27.08% | 29.27% | 119.92% | +6.71% | 100.00% | 100.00% | +| Optical grating crystal | 134323 | 12.98% | 15.19% | 62.86% | -0.16% | 100.00% | 100.00% | +| Spiniform stalagmite crystal | 37519 | 7.32% | 18.06% | 82.92% | +1.42% | 100.00% | 100.00% | +| Unipolar magnet | 5420 | 14.59% | 17.66% | 67.80% | +1.29% | 100.00% | 100.00% | + +The source uses internal names for some ores. The raw result keeps these names: +`crysrub`, `grat`, `bamboo`, and `mag`. The full machine-readable result is +[`2026-07-30-vein-calibration-spacing.json`](../benchmarks/results/2026-07-30-vein-calibration-spacing.json). + +There were 1,394,252 non-null reference amounts and 2,025,094 null reference +amounts. The candidate had no false presence and no missed presence. Precision +and recall were 100% for each ore. + +## Verification + +- The production generator runtime behavior has no change from commit `59232b78`. +- Two branch runs and one source-commit run for seeds 0 through 9 were byte equal. +- The output SHA-256 value was `940014de4f0352f8577bec3a351523108a0579186837b5100a30413b96382f00`. +- The branch executable hashes equal the exact baseline executable hashes. +- The database schema SHA-256 value stayed `b9c2d2a23b81af92dfd58188cf00664eca84c202ed4a647055c0eb5c3879a78d`. +- The schema still has one nullable column for each ore. It has no minimum, + maximum, or average ore columns. +- A focused test checks binary SQL NULL fields for gas planets and absent rocky + planet ores. +- Rust and Python tests pass. + +## Result Artifacts + +- [`2026-07-30-vein-estimator-benchmark.md`](../benchmarks/results/2026-07-30-vein-estimator-benchmark.md) + is the concise timeline record of configuration, provenance, results, and the + decision not to ship an estimator. +- [`2026-07-30-vein-baselines.json`](../benchmarks/results/2026-07-30-vein-baselines.json) + records the initial historical-fast versus exact A-B-B-A comparison. +- [`2026-07-30-vein-final-comparison.json`](../benchmarks/results/2026-07-30-vein-final-comparison.json) + records the repeated comparison and the computed elapsed-time requirement. +- [`2026-07-30-vein-calibration-midpoint.json`](../benchmarks/results/2026-07-30-vein-calibration-midpoint.json) + records the historical midpoint candidate's calibration result. +- [`2026-07-30-vein-calibration-spacing.json`](../benchmarks/results/2026-07-30-vein-calibration-spacing.json) + records the rejected spacing candidate's calibration result. + +The calibration results are schema version 2. They record the corpus, input, +source revision, exporter path and hash, estimator prefix, metrics, and gate +outcome. The performance artifacts contain the alternating run order, host and +workload configuration, executable hashes, raw process measurements, and the +computed overhead requirement. + +## Reproduction + +Build the oracle exporter with the pinned builder and musl target: + +```bash +docker run --rm -v "$PWD/inserter:/app" -w /app \ + rust:1.97.0@sha256:b92b8c8574f8f3b207fcb0912fb3e2de4041580b5934d90312d53938c9a038a9 \ + sh -c 'rustup target add x86_64-unknown-linux-musl && cargo build --release --locked --target x86_64-unknown-linux-musl --bin vein_accuracy' +``` + +Export and evaluate a calibration candidate. Substitute the source revision that +produced the exporter when creating a new artifact: + +```bash +inserter/target/x86_64-unknown-linux-musl/release/vein_accuracy \ + 0 1000 /tmp/vein-calibration.csv + +python3 benchmarks/evaluate_vein_accuracy.py \ + --input /tmp/vein-calibration.csv \ + --output /tmp/vein-calibration-result.json \ + --estimate-prefix spacing \ + --seed-set calibration \ + --source-revision "$(git rev-parse HEAD)" \ + --exporter inserter/target/x86_64-unknown-linux-musl/release/vein_accuracy +``` + +To fit scales for the historical estimate, then evaluate the scaled estimate: + +```bash +python3 benchmarks/fit_vein_scales.py \ + --input /tmp/vein-calibration.csv \ + --output /tmp/vein-scales.json \ + --source-revision "$(git rev-parse HEAD)" \ + --exporter inserter/target/x86_64-unknown-linux-musl/release/vein_accuracy + +python3 benchmarks/evaluate_vein_accuracy.py \ + --input /tmp/vein-calibration.csv \ + --output /tmp/vein-scaled-result.json \ + --scales /tmp/vein-scales.json \ + --estimate-prefix estimate \ + --seed-set calibration \ + --source-revision "$(git rev-parse HEAD)" \ + --exporter inserter/target/x86_64-unknown-linux-musl/release/vein_accuracy +``` + +Measure an eligible candidate against a baseline with the locked performance +corpus. The candidate must honor the benchmark environment supplied by the +comparison script. + +```bash +python3 benchmarks/run_comparison.py \ + --baseline /path/to/baseline/dsp_seed_finder \ + --candidate /path/to/candidate/dsp_seed_finder \ + --candidate-name candidate \ + --output /tmp/vein-performance.json \ + --start-seed 2000000 --end-seed 2000200 \ + --workers 1 --rounds 3 --cpus 0 +``` + +Do not use the held-out range until a calibration candidate passes all accuracy +gates. Both the exporter and evaluator require an explicit `--allow-held-out` +flag for the held-out range.