Add declarative SQL benchmark matrix profiles and vx-bench matrix CLI

bench-orchestrator/README.md

@@ -70,6 +70,23 @@ vx-bench prepare-data <benchmark> [options]
 - `--formats-json`: Exact data formats as JSON, e.g. `'["parquet","vortex"]'`
 - `--opt`: Benchmark-specific options such as `scale-factor=10.0`
 
+
+### `matrix` - Resolve CI Benchmark Matrices
+
+Emit the GitHub Actions `include:` array for a named profile. The benchmark workflows can use this
+to keep benchmark coverage in Python instead of copying large JSON matrices between YAML files.
+
+```bash
+vx-bench matrix            # list available profiles
+vx-bench matrix develop    # emit compact JSON
+vx-bench matrix nightly --pretty
+```
+
+**Options:**
+
+- `--list`: List available profiles and exit
+- `--pretty`: Pretty-print the JSON output
+
 ### `compare` - Compare Results
 
 Compare benchmark results within a run or across multiple runs. Results are displayed in a pivot table format.
@@ -137,6 +154,24 @@ vx-bench clean --older-than "30 days" [options]
 - `--keep-labeled`: Don't delete labeled runs (default: true)
 - `--dry-run, -n`: Show what would be deleted
 
+## Declarative Benchmark Matrix
+
+CI benchmark coverage is declared in `bench_orchestrator/benchmarks.py` and rendered by
+`bench_orchestrator/matrix.py`. This keeps the source of truth out of workflow YAML while still
+emitting the JSON shape GitHub Actions expects.
+
+The model separates three review concerns:
+
+- **Benchmark definitions** declare the suite, storage location, scale factor, and supported
+  engine/format targets.
+- **Profiles** choose how much declared coverage each workflow should run (`develop`, `pr`, and
+  `nightly`).
+- **Matrix rendering** converts a profile into stable GitHub Actions `include` entries with fields
+  such as `targets`, `data_formats`, `scale_factor`, `iterations`, and remote-storage metadata.
+
+When adding coverage, update the declarations first and add focused tests for the resolved profile
+entries rather than duplicating large inline JSON matrices in workflow files.
+
 ## Example Workflows
 
 ### 1. Basic Performance Comparison

bench-orchestrator/bench_orchestrator/benchmarks.py

@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+"""SQL benchmark declarations and CI profiles.
+
+Edit this file when changing benchmark coverage. Matrix rendering lives in
+``bench_orchestrator.matrix`` so workflow shape and benchmark coverage do not drift together.
+"""
+
+from .config import Benchmark, Engine, Format
+from .matrix import STANDARD, BenchmarkDef, Profile, Storage, all_targets, defaults, df, duck
+
+
+def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None = 10) -> BenchmarkDef:
+    suffix = "" if scale_factor in {1, 100} else f"-{int(scale_factor)}"
+    if storage is Storage.NVME:
+        target_set = df(
+            Format.ARROW,
+            Format.PARQUET,
+            Format.VORTEX,
+            Format.VORTEX_COMPACT,
+            Format.LANCE,
+        ) | duck(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB)
+        local_dir = None
+        remote_storage = None
+    else:
+        target_set = STANDARD
+        local_dir = f"vortex-bench/data/tpch/{scale_factor:.1f}"
+        remote_storage = f"s3://vortex-ci-benchmark-datasets/${{{{github.ref_name}}}}/${{{{github.run_id}}}}/tpch/{scale_factor:.1f}/"
+
+    name = f"TPC-H on {storage.label}" if scale_factor == 100 else f"TPC-H SF={scale_factor:g} on {storage.label}"
+    return BenchmarkDef(
+        id=f"tpch-{storage.value}{suffix}",
+        benchmark=Benchmark.TPCH,
+        name=name,
+        targets=target_set,
+        storage=storage,
+        scale_factor=scale_factor,
+        iterations=iterations,
+        nightly=scale_factor == 100,
+        local_dir=local_dir,
+        remote_storage=remote_storage,
+    )
+
+
+def _clickbench(benchmark: Benchmark, name: str) -> BenchmarkDef:
+    return BenchmarkDef(
+        id=f"{benchmark.value}-nvme",
+        benchmark=benchmark,
+        name=name,
+        targets=df(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.LANCE)
+        | duck(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB),
+    )
+
+
+def _fineweb(storage: Storage) -> BenchmarkDef:
+    if storage is Storage.NVME:
+        return BenchmarkDef(
+            id="fineweb",
+            benchmark=Benchmark.FINEWEB,
+            name="FineWeb NVMe",
+            targets=STANDARD,
+            scale_factor=100,
+        )
+    return BenchmarkDef(
+        id="fineweb-s3",
+        benchmark=Benchmark.FINEWEB,
+        name="FineWeb S3",
+        targets=STANDARD,
+        storage=Storage.S3,
+        scale_factor=100,
+        local_dir="vortex-bench/data/fineweb",
+        remote_storage="s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/",
+    )
+
+
+BENCHMARKS: list[BenchmarkDef] = [
+    _clickbench(Benchmark.CLICKBENCH, "Clickbench on NVME"),
+    _clickbench(Benchmark.CLICKBENCH_SORTED, "Clickbench Sorted on NVME"),
+    _tpch(1.0, Storage.NVME),
+    _tpch(1.0, Storage.S3),
+    _tpch(10.0, Storage.NVME),
+    _tpch(10.0, Storage.S3),
+    _tpch(100, Storage.NVME, iterations=None),
+    _tpch(100.0, Storage.S3, iterations=None),
+    BenchmarkDef(
+        id="tpcds-nvme",
+        benchmark=Benchmark.TPCDS,
+        name="TPC-DS SF=1 on NVME",
+        targets=STANDARD | duck(Format.DUCKDB),
+        scale_factor=1.0,
+    ),
+    BenchmarkDef(
+        id="statpopgen",
+        benchmark=Benchmark.STATPOPGEN,
+        name="Statistical and Population Genetics",
+        targets=STANDARD.only(Engine.DUCKDB),
+        scale_factor=100,
+        local_dir="vortex-bench/data/statpopgen",
+    ),
+    _fineweb(Storage.NVME),
+    _fineweb(Storage.S3),
+    BenchmarkDef(
+        id="polarsignals",
+        benchmark=Benchmark.POLARSIGNALS,
+        name="PolarSignals Profiling",
+        targets=df(Format.VORTEX),
+        scale_factor=1,
+    ),
+    BenchmarkDef(
+        id="appian-nvme",
+        benchmark=Benchmark.APPIAN,
+        name="Appian on NVME",
+        targets=STANDARD | duck(Format.DUCKDB),
+        iterations=10,
+    ),
+]
+
+PROFILES: dict[str, Profile] = {
+    "develop": Profile(
+        targets=all_targets,
+        description="Every regular SQL benchmark at full target coverage.",
+    ),
+    "pr": Profile(
+        targets=defaults,
+        description="Every regular SQL benchmark at default targets.",
+    ),
+    "nightly": Profile(
+        nightly=True,
+        targets=defaults,
+        description="Large-scale SF=100 TPC-H on NVMe and S3 at default targets.",
+    ),
+}

bench-orchestrator/bench_orchestrator/cli.py

@@ -3,6 +3,7 @@
 
 """CLI for benchmark orchestration."""
 
+import json
 import subprocess
 from contextlib import contextmanager
 from datetime import datetime, timedelta
@@ -15,6 +16,7 @@
 from rich.console import Console
 from rich.table import Table
 
+from .benchmarks import BENCHMARKS, PROFILES
 from .comparison import analyzer
 from .comparison.reporter import pivot_comparison_table
 from .config import (
@@ -29,6 +31,7 @@
     parse_targets_json,
     resolve_axis_targets,
 )
+from .matrix import resolve_matrix
 from .runner.builder import BenchmarkBuilder
 from .runner.executor import BenchmarkExecutor
 from .storage.store import ResultStore
@@ -214,6 +217,31 @@ def prepare_data(
         raise typer.Exit(1) from exc
 
 
+@app.command("matrix")
+def matrix(
+    profile: Annotated[
+        str | None,
+        typer.Argument(help="Profile to resolve; omit to list available profiles"),
+    ] = None,
+    list_profiles: Annotated[bool, typer.Option("--list", help="List available profiles and exit")] = False,
+    pretty: Annotated[bool, typer.Option("--pretty", help="Pretty-print the JSON output")] = False,
+) -> None:
+    """Emit the GitHub Actions benchmark matrix for a profile."""
+    if profile is None or list_profiles:
+        for name, prof in PROFILES.items():
+            console.print(f"[bold cyan]{name}[/bold cyan]: {prof.description}")
+        return
+
+    prof = PROFILES.get(profile)
+    if prof is None:
+        known = ", ".join(PROFILES)
+        console.print(f"[red]Unknown profile '{profile}'. Available: {known}[/red]")
+        raise typer.Exit(1)
+
+    entries = resolve_matrix(prof, BENCHMARKS)
+    typer.echo(json.dumps(entries, indent=2 if pretty else None))
+
+
 @app.command()
 def run(
     benchmark: Annotated[Benchmark, typer.Argument(help="Benchmark suite to run")],