diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rs index c9d8beab1466..96c99ab20750 100644 --- a/datafusion/datasource-parquet/src/projection_read_plan.rs +++ b/datafusion/datasource-parquet/src/projection_read_plan.rs @@ -30,6 +30,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use datafusion_functions::core::input_file_name::InputFileNameFunc; use parquet::arrow::ProjectionMask; use parquet::schema::types::SchemaDescriptor; @@ -313,8 +314,10 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> { return Ok(recursion); } - if ScalarFunctionExpr::try_downcast_func::(node.as_ref()) + if ScalarFunctionExpr::try_downcast_func::(node.as_ref()) .is_some() + || ScalarFunctionExpr::try_downcast_func::(node.as_ref()) + .is_some() { self.has_unpushable_udfs = true; return Ok(TreeNodeRecursion::Jump); diff --git a/datafusion/sqllogictest/test_files/input_file_name.slt b/datafusion/sqllogictest/test_files/input_file_name.slt index 8fb72d4a9d14..32110aa2d69a 100644 --- a/datafusion/sqllogictest/test_files/input_file_name.slt +++ b/datafusion/sqllogictest/test_files/input_file_name.slt @@ -121,8 +121,7 @@ physical_plan 01)SortPreservingMergeExec: [column1@0 ASC NULLS LAST] 02)--SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[true] 03)----FilterExec: __datafusion_extracted_1@0 LIKE %first.parquet, projection=[column1@1] -04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 -05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet, predicate=input_file_name() LIKE %first.parquet +04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet statement ok -DROP TABLE pq_table; \ No newline at end of file +DROP TABLE pq_table; diff --git a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt index 773ab6761fd2..25a3c4eb4c6f 100644 --- a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt +++ b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt @@ -52,5 +52,33 @@ logical_plan 02)--TableScan: test_table projection=[column1] physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet]]}, projection=[input_file_name() as input_file_name(), CAST(__datafusion_file_row_index@1 AS Int64) as file_row_index(), column1], file_type=parquet + +# Make sure it also behaves consistently regardless of filter pushdown + +statement ok +SET datafusion.execution.parquet.pushdown_filters = false; + +query TII rowsort +SELECT input_file_name(), file_row_index(), column1 +FROM test_table +WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet'; +---- +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30 +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60 + +statement ok +SET datafusion.execution.parquet.pushdown_filters = true; + +query TII rowsort +SELECT input_file_name(), file_row_index(), column1 +FROM test_table +WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet'; +---- +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30 +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60 + +statement ok +RESET datafusion.execution.parquet.pushdown_filters; + statement ok DROP TABLE test_table;