From 60ac9c678297bbf9e0afe400c746598cf6aad40b Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 16 Jul 2026 13:47:56 +0100 Subject: [PATCH 1/3] tests Signed-off-by: Adam Gutglick --- .../test_files/parquet_metadata_functions.slt | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt index 773ab6761fd26..03b473973de7a 100644 --- a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt +++ b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt @@ -52,5 +52,33 @@ logical_plan 02)--TableScan: test_table projection=[column1] physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet]]}, projection=[input_file_name() as input_file_name(), CAST(__datafusion_file_row_index@1 AS Int64) as file_row_index(), column1], file_type=parquet + +# Make sure it also behaves consistently regardless of filter pushdown + +statement ok +SET datafusion.execution.parquet.pushdown_filters = false; + +query TII +SELECT input_file_name(), file_row_index(), column1 +FROM test_table +WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet'; +---- +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30 +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60 + +statement ok +SET datafusion.execution.parquet.pushdown_filters = true; + +query TII +SELECT input_file_name(), file_row_index(), column1 +FROM test_table +WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet'; +---- +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30 +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60 + +statement ok +RESET datafusion.execution.parquet.pushdown_filters; + statement ok DROP TABLE test_table; From 4f52bd8edbc0b32b887cf1c7fce745842945f67d Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 16 Jul 2026 13:48:03 +0100 Subject: [PATCH 2/3] fix Signed-off-by: Adam Gutglick --- datafusion/datasource-parquet/src/projection_read_plan.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rs index c9d8beab1466d..96c99ab20750e 100644 --- a/datafusion/datasource-parquet/src/projection_read_plan.rs +++ b/datafusion/datasource-parquet/src/projection_read_plan.rs @@ -30,6 +30,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use datafusion_functions::core::input_file_name::InputFileNameFunc; use parquet::arrow::ProjectionMask; use parquet::schema::types::SchemaDescriptor; @@ -313,8 +314,10 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> { return Ok(recursion); } - if ScalarFunctionExpr::try_downcast_func::(node.as_ref()) + if ScalarFunctionExpr::try_downcast_func::(node.as_ref()) .is_some() + || ScalarFunctionExpr::try_downcast_func::(node.as_ref()) + .is_some() { self.has_unpushable_udfs = true; return Ok(TreeNodeRecursion::Jump); From bc4a25fe12760166c3cbfc892ab69e224faa59ce Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 16 Jul 2026 15:34:44 +0100 Subject: [PATCH 3/3] fix pre-existing test Signed-off-by: Adam Gutglick --- datafusion/sqllogictest/test_files/input_file_name.slt | 5 ++--- .../sqllogictest/test_files/parquet_metadata_functions.slt | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/datafusion/sqllogictest/test_files/input_file_name.slt b/datafusion/sqllogictest/test_files/input_file_name.slt index 8fb72d4a9d14b..32110aa2d69af 100644 --- a/datafusion/sqllogictest/test_files/input_file_name.slt +++ b/datafusion/sqllogictest/test_files/input_file_name.slt @@ -121,8 +121,7 @@ physical_plan 01)SortPreservingMergeExec: [column1@0 ASC NULLS LAST] 02)--SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[true] 03)----FilterExec: __datafusion_extracted_1@0 LIKE %first.parquet, projection=[column1@1] -04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 -05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet, predicate=input_file_name() LIKE %first.parquet +04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet statement ok -DROP TABLE pq_table; \ No newline at end of file +DROP TABLE pq_table; diff --git a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt index 03b473973de7a..25a3c4eb4c6fa 100644 --- a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt +++ b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt @@ -58,7 +58,7 @@ physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion statement ok SET datafusion.execution.parquet.pushdown_filters = false; -query TII +query TII rowsort SELECT input_file_name(), file_row_index(), column1 FROM test_table WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet'; @@ -69,7 +69,7 @@ WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_funct statement ok SET datafusion.execution.parquet.pushdown_filters = true; -query TII +query TII rowsort SELECT input_file_name(), file_row_index(), column1 FROM test_table WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet';