Skip to content

fix: Handle input_file_name() pushdown into ParquetSource with filter pushdown enabled #23638

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AdamGS wants to merge 3 commits into
base: main
Choose a base branch
from
+34 −4
Open

fix: Handle input_file_name() pushdown into ParquetSource with filter pushdown enabled #23638

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
60ac9c6
tests
AdamGS Jul 16, 2026
4f52bd8
fix
AdamGS Jul 16, 2026
bc4a25f
fix pre-existing test
AdamGS Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion datafusion/datasource-parquet/src/projection_read_plan.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;

use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion_functions::core::input_file_name::InputFileNameFunc;
use parquet::arrow::ProjectionMask;
use parquet::schema::types::SchemaDescriptor;

Expand Down Expand Up @@ -313,8 +314,10 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {
return Ok(recursion);
}

if ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())
if ScalarFunctionExpr::try_downcast_func::<InputFileNameFunc>(node.as_ref())
.is_some()
|| ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())
.is_some()
{
self.has_unpushable_udfs = true;
return Ok(TreeNodeRecursion::Jump);
Expand Down
5 changes: 2 additions & 3 deletions datafusion/sqllogictest/test_files/input_file_name.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,7 @@ physical_plan
01)SortPreservingMergeExec: [column1@0 ASC NULLS LAST]
02)--SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[true]
03)----FilterExec: __datafusion_extracted_1@0 LIKE %first.parquet, projection=[column1@1]
04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet, predicate=input_file_name() LIKE %first.parquet
04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet

statement ok
DROP TABLE pq_table;
DROP TABLE pq_table;
28 changes: 28 additions & 0 deletions datafusion/sqllogictest/test_files/parquet_metadata_functions.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,5 +52,33 @@ logical_plan
02)--TableScan: test_table projection=[column1]
physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet]]}, projection=[input_file_name() as input_file_name(), CAST(__datafusion_file_row_index@1 AS Int64) as file_row_index(), column1], file_type=parquet


# Make sure it also behaves consistently regardless of filter pushdown

statement ok
SET datafusion.execution.parquet.pushdown_filters = false;

query TII rowsort
SELECT input_file_name(), file_row_index(), column1
FROM test_table
WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet';
----
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60

statement ok
SET datafusion.execution.parquet.pushdown_filters = true;

query TII rowsort
SELECT input_file_name(), file_row_index(), column1
FROM test_table
WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet';
----
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60

statement ok
RESET datafusion.execution.parquet.pushdown_filters;

statement ok
DROP TABLE test_table;
Loading