Skip to content

Remove Arrow macro benchmarks #8816

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AdamGS wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

Remove Arrow macro benchmarks #8816

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b99d87c
Remove arrow format
AdamGS Jul 17, 2026
5c70d76
Remove Arrow engine
AdamGS Jul 17, 2026
39d3ca5
do this right
AdamGS Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions .github/workflows/sql-benchmarks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ on:
"name": "TPC-H SF=1 on NVME",
"data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"],
"pr_targets": [
{"engine": "datafusion", "format": "arrow"},
{"engine": "datafusion", "format": "parquet"},
{"engine": "datafusion", "format": "vortex"},
{"engine": "datafusion", "format": "vortex-compact"},
Expand All @@ -78,7 +77,6 @@ on:
{"engine": "duckdb", "format": "duckdb"}
],
"develop_targets": [
{"engine": "datafusion", "format": "arrow"},
{"engine": "datafusion", "format": "parquet"},
{"engine": "datafusion", "format": "vortex"},
{"engine": "datafusion", "format": "vortex-compact"},
Expand Down Expand Up @@ -123,7 +121,6 @@ on:
"name": "TPC-H SF=10 on NVME",
"data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"],
"pr_targets": [
{"engine": "datafusion", "format": "arrow"},
{"engine": "datafusion", "format": "parquet"},
{"engine": "datafusion", "format": "vortex"},
{"engine": "datafusion", "format": "vortex-compact"},
Expand All @@ -133,7 +130,6 @@ on:
{"engine": "duckdb", "format": "duckdb"}
],
"develop_targets": [
{"engine": "datafusion", "format": "arrow"},
{"engine": "datafusion", "format": "parquet"},
{"engine": "datafusion", "format": "vortex"},
{"engine": "datafusion", "format": "vortex-compact"},
Expand Down Expand Up @@ -350,14 +346,12 @@ on:
"name": "TPC-H SF=1 on NVME",
"data_formats": ["parquet", "vortex"],
"pr_targets": [
{"engine": "datafusion", "format": "arrow"},
{"engine": "datafusion", "format": "parquet"},
{"engine": "datafusion", "format": "vortex"},
{"engine": "duckdb", "format": "parquet"},
{"engine": "duckdb", "format": "vortex"}
],
"develop_targets": [
{"engine": "datafusion", "format": "arrow"},
{"engine": "datafusion", "format": "parquet"},
{"engine": "datafusion", "format": "vortex"},
{"engine": "datafusion", "format": "lance"},
Expand Down Expand Up @@ -395,14 +389,12 @@ on:
"name": "TPC-H SF=10 on NVME",
"data_formats": ["parquet", "vortex"],
"pr_targets": [
{"engine": "datafusion", "format": "arrow"},
{"engine": "datafusion", "format": "parquet"},
{"engine": "datafusion", "format": "vortex"},
{"engine": "duckdb", "format": "parquet"},
{"engine": "duckdb", "format": "vortex"}
],
"develop_targets": [
{"engine": "datafusion", "format": "arrow"},
{"engine": "datafusion", "format": "parquet"},
{"engine": "datafusion", "format": "vortex"},
{"engine": "datafusion", "format": "lance"},
Expand Down
2 changes: 1 addition & 1 deletion bench-orchestrator/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ vx-bench clean --older-than "30 days" --no-keep-labeled

| Engine | Supported Formats |
|------------|-------------------------------------------|
| datafusion | arrow, parquet, vortex, vortex-compact, lance |
| datafusion | parquet, vortex, vortex-compact, lance |
| duckdb | parquet, vortex, vortex-compact, duckdb |
| lance | lance |

Expand Down
2 changes: 0 additions & 2 deletions bench-orchestrator/bench_orchestrator/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ def binary_name(self) -> str:
class Format(Enum):
"""Data formats for benchmarks."""

ARROW = "arrow"
PARQUET = "parquet"
VORTEX = "vortex"
VORTEX_COMPACT = "vortex-compact"
Expand Down Expand Up @@ -60,7 +59,6 @@ class Benchmark(Enum):
# Engine to supported formats mapping.
ENGINE_FORMATS: dict[Engine, list[Format]] = {
Engine.DATAFUSION: [
Format.ARROW,
Format.PARQUET,
Format.VORTEX,
Format.VORTEX_COMPACT,
Expand Down
6 changes: 3 additions & 3 deletions bench-orchestrator/tests/test_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,15 @@ def test_resolve_axis_targets_offers_vortex_native_on_duckdb_only() -> None:
def test_resolve_axis_targets_filters_unsupported_combinations() -> None:
targets, warnings = resolve_axis_targets(
[Engine.DATAFUSION, Engine.DUCKDB],
[Format.ARROW, Format.PARQUET],
[Format.LANCE, Format.PARQUET],
)

assert targets == [
BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.ARROW),
BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE),
BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.PARQUET),
BenchmarkTarget(engine=Engine.DUCKDB, format=Format.PARQUET),
]
assert warnings == ["Format arrow is not supported by engine duckdb"]
assert warnings == ["Format lance is not supported by engine duckdb"]


def test_resolve_axis_targets_skips_engines_a_benchmark_cannot_run() -> None:
Expand Down
2 changes: 0 additions & 2 deletions benchmarks/datafusion-bench/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ pub mod tracer;
use std::sync::Arc;

use datafusion::datasource::file_format::FileFormat;
use datafusion::datasource::file_format::arrow::ArrowFormat;
use datafusion::datasource::file_format::csv::CsvFormat;
use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::provider::DefaultTableFactory;
Expand Down Expand Up @@ -109,7 +108,6 @@ pub fn make_object_store(
pub fn format_to_df_format(format: Format) -> Arc<dyn FileFormat> {
match format {
Format::Csv => Arc::new(CsvFormat::default()) as _,
Format::Arrow => Arc::new(ArrowFormat),
Format::Parquet => Arc::new(ParquetFormat::new()),
Format::OnDiskVortex | Format::VortexCompact | Format::VortexNative => Arc::new(
VortexFormat::new_with_options(SESSION.clone(), vortex_table_options()),
Expand Down
137 changes: 36 additions & 101 deletions benchmarks/datafusion-bench/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ use datafusion::datasource::listing::ListingOptions;
use datafusion::datasource::listing::ListingTable;
use datafusion::datasource::listing::ListingTableConfig;
use datafusion::datasource::listing::ListingTableUrl;
use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder;
use datafusion::prelude::SessionContext;
use datafusion_bench::format_to_df_format;
use datafusion_bench::metrics::MetricsSetExt;
Expand All @@ -24,10 +23,12 @@ use datafusion_bench::tracer::get_static_tracer;
use datafusion_bench::tracer::set_labels;
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::collect;
use futures::StreamExt;
use parking_lot::Mutex;
use tokio::fs::File;
use vortex::file::multi::MultiFileDataSource;
use vortex::io::filesystem::FileSystemRef;
use vortex::io::object_store::ObjectStoreFileSystem;
use vortex::io::session::RuntimeSessionExt;
use vortex::scan::DataSource as _;
use vortex::scan::DataSourceRef;
use vortex_arrow::ToArrowType;
use vortex_bench::Benchmark;
Expand All @@ -49,6 +50,7 @@ use vortex_bench::runner::filter_queries;
use vortex_bench::setup_logging_and_tracing;
use vortex_bench::v3;
use vortex_datafusion::metrics::VortexMetricsFinder;
use vortex_datafusion::v2::VortexTable;

/// Common arguments shared across benchmarks
#[derive(Parser)]
Expand Down Expand Up @@ -256,43 +258,39 @@ async fn register_benchmark_tables<B: Benchmark + ?Sized>(
benchmark: &B,
format: Format,
) -> anyhow::Result<()> {
match format {
Format::Arrow => register_arrow_tables(session, benchmark).await,
Comment thread
AdamGS marked this conversation as resolved.
_ if use_scan_api() && matches!(format, Format::OnDiskVortex | Format::VortexCompact) => {
register_v2_tables(session, benchmark, format).await
if use_scan_api() && matches!(format, Format::OnDiskVortex | Format::VortexCompact) {
register_v2_tables(session, benchmark, format).await
} else {
let benchmark_base = benchmark.data_url().join(&format!("{}/", format.name()))?;
let file_format = format_to_df_format(format);

for table in benchmark.table_specs().iter() {
let pattern = benchmark.pattern(table.name, format);
let table_url = ListingTableUrl::try_new(benchmark_base.clone(), pattern)?;

let listing_options = ListingOptions::new(Arc::clone(&file_format))
.with_session_config_options(session.state().config());
let mut config =
ListingTableConfig::new(table_url).with_listing_options(listing_options);

config = match table.schema.as_ref() {
Some(schema) => config.with_schema(Arc::new(schema.clone())),
None => config.infer_schema(&session.state()).await?,
};

let listing_table = Arc::new(
ListingTable::try_new(config)?.with_cache(
session
.runtime_env()
.cache_manager
.get_file_statistic_cache(),
),
);

session.register_table(table.name, listing_table)?;
}
_ => {
let benchmark_base = benchmark.data_url().join(&format!("{}/", format.name()))?;
let file_format = format_to_df_format(format);

for table in benchmark.table_specs().iter() {
let pattern = benchmark.pattern(table.name, format);
let table_url = ListingTableUrl::try_new(benchmark_base.clone(), pattern)?;

let listing_options = ListingOptions::new(Arc::clone(&file_format))
.with_session_config_options(session.state().config());
let mut config =
ListingTableConfig::new(table_url).with_listing_options(listing_options);

config = match table.schema.as_ref() {
Some(schema) => config.with_schema(Arc::new(schema.clone())),
None => config.infer_schema(&session.state()).await?,
};

let listing_table = Arc::new(
ListingTable::try_new(config)?.with_cache(
session
.runtime_env()
.cache_manager
.get_file_statistic_cache(),
),
);

session.register_table(table.name, listing_table)?;
}

Ok(())
}
Ok(())
}
}

Expand All @@ -302,12 +300,6 @@ async fn register_v2_tables<B: Benchmark + ?Sized>(
benchmark: &B,
format: Format,
) -> anyhow::Result<()> {
use vortex::file::multi::MultiFileDataSource;
use vortex::io::object_store::ObjectStoreFileSystem;
use vortex::io::session::RuntimeSessionExt;
use vortex::scan::DataSource as _;
use vortex_datafusion::v2::VortexTable;

let benchmark_base = benchmark.data_url().join(&format!("{}/", format.name()))?;

for table in benchmark.table_specs().iter() {
Expand Down Expand Up @@ -345,63 +337,6 @@ async fn register_v2_tables<B: Benchmark + ?Sized>(
Ok(())
}

/// Load Arrow IPC files into in-memory DataFusion tables.
async fn register_arrow_tables<B: Benchmark + ?Sized>(
session: &SessionContext,
benchmark: &B,
) -> anyhow::Result<()> {
use datafusion::datasource::MemTable;

let parquet_dir = benchmark
.data_url()
.to_file_path()
.map_err(|_| anyhow::anyhow!("Arrow format requires local file path"))?
.join(Format::Parquet.name());

// Read all arrow files from the directory
let data_files = std::fs::read_dir(&parquet_dir)?.collect::<Result<Vec<_>, _>>()?;

for table in benchmark.table_specs().iter() {
let pattern = benchmark.pattern(table.name, Format::Parquet);

// Find files matching this table's pattern
let matching_files: Vec<_> = data_files
.iter()
.filter(|entry| {
let filename = entry.file_name();
let filename_str = filename.to_str().unwrap_or("");
match &pattern {
Some(p) => p.matches(filename_str),
None => filename_str == format!("{}.{}", table.name, Format::Parquet.ext()),
}
})
.collect();

// Load all matching files into memory
let mut all_batches = Vec::new();
let mut schema = None;

for dir_entry in matching_files {
let file = File::open(dir_entry.path()).await?;
let mut reader = ParquetRecordBatchStreamBuilder::new(file).await?.build()?;
if schema.is_none() {
schema = Some(reader.schema()).cloned();
}

while let Some(batch) = reader.next().await {
all_batches.push(batch?);
}
}

if let Some(schema) = schema {
let mem_table = MemTable::try_new(schema, vec![all_batches])?;
session.register_table(table.name, Arc::new(mem_table))?;
}
}

Ok(())
}

/// Wrapper around DataFusion record batches implementing `BenchmarkQueryResult`.
pub struct DataFusionQueryResult(pub Vec<RecordBatch>);

Expand Down
15 changes: 2 additions & 13 deletions benchmarks/random-access-bench/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ async fn benchmark_random_access(
let timing = TimingMeasurement {
name: measurement_name.to_string(),
storage: storage.to_string(),
target: Target::new(format_to_engine(format), format),
target: Target::new(Engine::default(), format),
runs,
};
Ok(RandomAccessRun {
Expand Down Expand Up @@ -326,17 +326,6 @@ fn push_v3_random_access_record(records: &mut Vec<v3::V3Record>, run: &RandomAcc
records.push(v3::random_access_record(&run.timing, &dataset));
}

/// Map format to the appropriate engine for random access benchmarks.
fn format_to_engine(format: Format) -> Engine {
match format {
Format::OnDiskVortex | Format::VortexCompact => Engine::Vortex,
Format::Parquet => Engine::Arrow,
#[cfg(feature = "lance")]
Format::Lance => Engine::Arrow, // Is this right here?
_ => Engine::default(),
}
}

/// Open a random accessor for any supported format.
///
/// For Vortex and Parquet, the path comes from [`BenchDataset::path`].
Expand Down Expand Up @@ -528,7 +517,7 @@ mod tests {
RandomAccessRun {
timing: TimingMeasurement {
name: format!("random-access/{dataset}/parquet-tokio-local-disk"),
target: Target::new(Engine::Arrow, Format::Parquet),
target: Target::new(Engine::Vortex, Format::Parquet),
storage: STORAGE_NVME.to_string(),
runs: vec![Duration::from_nanos(10)],
},
Expand Down
6 changes: 1 addition & 5 deletions benchmarks/random-access-bench/src/render.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ mod tests {
RandomAccessRun {
timing: TimingMeasurement {
name: format!("random-access/{dataset}/{}-tokio-local-disk", format.ext()),
target: Target::new(Engine::Arrow, format),
target: Target::new(Engine::Vortex, format),
storage: "nvme".to_string(),
runs: vec![Duration::from_micros(micros)],
},
Expand Down Expand Up @@ -273,10 +273,6 @@ mod tests {
rendered.contains("vortex-cached") && rendered.contains("vortex-reopen"),
"expected ext-based column headers, got:\n{rendered}"
);
assert!(
!rendered.contains("arrow"),
"expected no engine header row, got:\n{rendered}"
);
assert!(
rendered.contains("random-access/taxi")
&& rendered.contains("random-access/taxi/uniform"),
Expand Down
Loading
Loading