From 22e4e4a48a1f24b728458d0f7971cfeaa6a375f2 Mon Sep 17 00:00:00 2001 From: Justin O'Dwyer Date: Sun, 12 Jul 2026 00:35:15 +0200 Subject: [PATCH 1/3] Add tests showing repartitions. --- .../enforce_distribution.rs | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index e01311e25be8b..0669808ee10fd 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -870,6 +870,114 @@ fn range_inner_hash_join_rehashes_incompatible_range_partitioning() -> Result<() Ok(()) } +#[test] +fn left_hash_join_uses_range_partitioning() -> Result<()> { + let left = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let right = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let join_on = vec![( + Arc::new(Column::new_with_schema("a", &left.schema())?) as _, + Arc::new(Column::new_with_schema("a", &right.schema())?) as _, + )]; + let join = hash_join_exec(left, right, &join_on, &JoinType::Left); + + let plan = TestConfig::default() + .with_query_execution_partitions(4) + .to_plan(join, &DISTRIB_DISTRIB_SORT); + + println!("{}", displayable(plan.as_ref()).indent(true)); + + Ok(()) +} + +#[test] +fn left_semi_hash_join_uses_range_partitioning() -> Result<()> { + let left = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let right = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let join_on = vec![( + Arc::new(Column::new_with_schema("a", &left.schema())?) as _, + Arc::new(Column::new_with_schema("a", &right.schema())?) as _, + )]; + let join = hash_join_exec(left, right, &join_on, &JoinType::LeftSemi); + + let plan = TestConfig::default() + .with_query_execution_partitions(4) + .to_plan(join, &DISTRIB_DISTRIB_SORT); + + println!("{}", displayable(plan.as_ref()).indent(true)); + + Ok(()) +} + +#[test] +fn left_anti_hash_join_uses_range_partitioning() -> Result<()> { + let left = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let right = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let join_on = vec![( + Arc::new(Column::new_with_schema("a", &left.schema())?) as _, + Arc::new(Column::new_with_schema("a", &right.schema())?) as _, + )]; + let join = hash_join_exec(left, right, &join_on, &JoinType::LeftAnti); + + let plan = TestConfig::default() + .with_query_execution_partitions(4) + .to_plan(join, &DISTRIB_DISTRIB_SORT); + + println!("{}", displayable(plan.as_ref()).indent(true)); + + Ok(()) +} + +#[test] +fn left_mark_hash_join_uses_range_partitioning() -> Result<()> { + let left = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let right = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let join_on = vec![( + Arc::new(Column::new_with_schema("a", &left.schema())?) as _, + Arc::new(Column::new_with_schema("a", &right.schema())?) as _, + )]; + let join = hash_join_exec(left, right, &join_on, &JoinType::LeftMark); + + let plan = TestConfig::default() + .with_query_execution_partitions(4) + .to_plan(join, &DISTRIB_DISTRIB_SORT); + + println!("{}", displayable(plan.as_ref()).indent(true)); + + Ok(()) +} + #[test] fn multi_hash_joins() -> Result<()> { let left = parquet_exec(); From b5135118b7dcb95f980626ab5863aa10d63b7379 Mon Sep 17 00:00:00 2001 From: Justin O'Dwyer Date: Sun, 12 Jul 2026 10:58:27 +0200 Subject: [PATCH 2/3] Adding more tests. --- .../enforce_distribution.rs | 108 ++++++++++++++++++ .../test_files/range_partitioning.slt | 7 +- 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index 0669808ee10fd..d20ff3f4267b7 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -978,6 +978,114 @@ fn left_mark_hash_join_uses_range_partitioning() -> Result<()> { Ok(()) } +#[test] +fn left_hash_join_range_rehash_incompatible_split_points() -> Result<()> { + let left = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 15, 30], + SortOptions::default(), + )?); + let right = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let join_on = vec![( + Arc::new(Column::new_with_schema("a", &left.schema())?) as _, + Arc::new(Column::new_with_schema("a", &right.schema())?) as _, + )]; + let join = hash_join_exec(left, right, &join_on, &JoinType::Left); + + let plan = TestConfig::default() + .with_query_execution_partitions(4) + .to_plan(join, &DISTRIB_DISTRIB_SORT); + + println!("{}", displayable(plan.as_ref()).indent(true)); + + Ok(()) +} + +#[test] +fn left_hash_join_range_rehash_incompatible_partition_counts() -> Result<()> { + let left = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let right = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20], + SortOptions::default(), + )?); + let join_on = vec![( + Arc::new(Column::new_with_schema("a", &left.schema())?) as _, + Arc::new(Column::new_with_schema("a", &right.schema())?) as _, + )]; + let join = hash_join_exec(left, right, &join_on, &JoinType::LeftSemi); + + let plan = TestConfig::default() + .with_query_execution_partitions(4) + .to_plan(join, &DISTRIB_DISTRIB_SORT); + + println!("{}", displayable(plan.as_ref()).indent(true)); + + Ok(()) +} + +#[test] +fn left_hash_join_range_rehash_incompatible_sort_options() -> Result<()> { + let left = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let right = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [30, 20, 10], + SortOptions::default().with_descending(true), + )?); + let join_on = vec![( + Arc::new(Column::new_with_schema("a", &left.schema())?) as _, + Arc::new(Column::new_with_schema("a", &right.schema())?) as _, + )]; + let join = hash_join_exec(left, right, &join_on, &JoinType::LeftAnti); + + let plan = TestConfig::default() + .with_query_execution_partitions(4) + .to_plan(join, &DISTRIB_DISTRIB_SORT); + + println!("{}", displayable(plan.as_ref()).indent(true)); + + Ok(()) +} + +#[test] +fn left_hash_join_range_rehash_incompatible_key_expressions() -> Result<()> { + let left = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let right = parquet_exec_with_output_partitioning(range_partitioning( + "a", + [10, 20, 30], + SortOptions::default(), + )?); + let join_on = vec![( + Arc::new(Column::new_with_schema("b", &left.schema())?) as _, + Arc::new(Column::new_with_schema("b", &right.schema())?) as _, + )]; + let join = hash_join_exec(left, right, &join_on, &JoinType::LeftMark); + + let plan = TestConfig::default() + .with_query_execution_partitions(4) + .to_plan(join, &DISTRIB_DISTRIB_SORT); + + println!("{}", displayable(plan.as_ref()).indent(true)); + + Ok(()) +} + #[test] fn multi_hash_joins() -> Result<()> { let left = parquet_exec(); diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt index 5d004ca7fdfa5..9f597f28bf660 100644 --- a/datafusion/sqllogictest/test_files/range_partitioning.slt +++ b/datafusion/sqllogictest/test_files/range_partitioning.slt @@ -395,9 +395,10 @@ ORDER BY l.non_range_key, l.value, r.value; 2 350 350 ########## -# TEST 12: Non-Inner Range Join Repartitions -# Only inner partitioned hash joins opt in to Range satisfying KeyPartitioned -# requirements. Non-inner joins keep using Hash repartitioning. +# TEST 12: Left Range Join Repartitions +# Only inner and right-side (Right/RightSemi/RightAnti/RightMark) partitioned +# hash joins opt in to Range satisfying KeyPartitioned requirements. Other +# join types, such as Left, keep using Hash repartitioning. ########## query TT From b1991a4e4138c14f9172f6a993f336e3c090e135 Mon Sep 17 00:00:00 2001 From: Justin O'Dwyer Date: Wed, 15 Jul 2026 18:09:15 +0200 Subject: [PATCH 3/3] co-partitioned range left-side hash join --- .../enforce_distribution.rs | 191 +++--------------- .../physical-plan/src/joins/hash_join/exec.rs | 11 +- .../src/test_context/range_partitioning.rs | 30 ++- .../test_files/range_partitioning.slt | 180 ++++++++++++++++- 4 files changed, 236 insertions(+), 176 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index d03358996a972..5687c60048295 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -934,88 +934,7 @@ fn range_window_rehashes_incompatible_range_partitioning() -> Result<()> { } #[test] -fn left_hash_join_uses_range_partitioning() -> Result<()> { - let left = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let right = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let join_on = vec![( - Arc::new(Column::new_with_schema("a", &left.schema())?) as _, - Arc::new(Column::new_with_schema("a", &right.schema())?) as _, - )]; - let join = hash_join_exec(left, right, &join_on, &JoinType::Left); - - let plan = TestConfig::default() - .with_query_execution_partitions(4) - .to_plan(join, &DISTRIB_DISTRIB_SORT); - - println!("{}", displayable(plan.as_ref()).indent(true)); - - Ok(()) -} - -#[test] -fn left_semi_hash_join_uses_range_partitioning() -> Result<()> { - let left = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let right = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let join_on = vec![( - Arc::new(Column::new_with_schema("a", &left.schema())?) as _, - Arc::new(Column::new_with_schema("a", &right.schema())?) as _, - )]; - let join = hash_join_exec(left, right, &join_on, &JoinType::LeftSemi); - - let plan = TestConfig::default() - .with_query_execution_partitions(4) - .to_plan(join, &DISTRIB_DISTRIB_SORT); - - println!("{}", displayable(plan.as_ref()).indent(true)); - - Ok(()) -} - -#[test] -fn left_anti_hash_join_uses_range_partitioning() -> Result<()> { - let left = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let right = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let join_on = vec![( - Arc::new(Column::new_with_schema("a", &left.schema())?) as _, - Arc::new(Column::new_with_schema("a", &right.schema())?) as _, - )]; - let join = hash_join_exec(left, right, &join_on, &JoinType::LeftAnti); - - let plan = TestConfig::default() - .with_query_execution_partitions(4) - .to_plan(join, &DISTRIB_DISTRIB_SORT); - - println!("{}", displayable(plan.as_ref()).indent(true)); - - Ok(()) -} - -#[test] -fn left_mark_hash_join_uses_range_partitioning() -> Result<()> { +fn range_left_mark_hash_join_reuses_range_partitioning() -> Result<()> { let left = parquet_exec_with_output_partitioning(range_partitioning( "a", [10, 20, 30], @@ -1036,67 +955,20 @@ fn left_mark_hash_join_uses_range_partitioning() -> Result<()> { .with_query_execution_partitions(4) .to_plan(join, &DISTRIB_DISTRIB_SORT); - println!("{}", displayable(plan.as_ref()).indent(true)); - - Ok(()) -} - -#[test] -fn left_hash_join_range_rehash_incompatible_split_points() -> Result<()> { - let left = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 15, 30], - SortOptions::default(), - )?); - let right = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let join_on = vec![( - Arc::new(Column::new_with_schema("a", &left.schema())?) as _, - Arc::new(Column::new_with_schema("a", &right.schema())?) as _, - )]; - let join = hash_join_exec(left, right, &join_on, &JoinType::Left); - - let plan = TestConfig::default() - .with_query_execution_partitions(4) - .to_plan(join, &DISTRIB_DISTRIB_SORT); - - println!("{}", displayable(plan.as_ref()).indent(true)); - - Ok(()) -} - -#[test] -fn left_hash_join_range_rehash_incompatible_partition_counts() -> Result<()> { - let left = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let right = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20], - SortOptions::default(), - )?); - let join_on = vec![( - Arc::new(Column::new_with_schema("a", &left.schema())?) as _, - Arc::new(Column::new_with_schema("a", &right.schema())?) as _, - )]; - let join = hash_join_exec(left, right, &join_on, &JoinType::LeftSemi); - - let plan = TestConfig::default() - .with_query_execution_partitions(4) - .to_plan(join, &DISTRIB_DISTRIB_SORT); - - println!("{}", displayable(plan.as_ref()).indent(true)); + assert_plan!( + plan, + @r" + HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(a@0, a@0)] + DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC], [(10), (20), (30)], 4), file_type=parquet + DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC], [(10), (20), (30)], 4), file_type=parquet + " + ); Ok(()) } #[test] -fn left_hash_join_range_rehash_incompatible_sort_options() -> Result<()> { +fn range_left_anti_hash_join_rehashes_incompatible_sort_options() -> Result<()> { let left = parquet_exec_with_output_partitioning(range_partitioning( "a", [10, 20, 30], @@ -1105,7 +977,10 @@ fn left_hash_join_range_rehash_incompatible_sort_options() -> Result<()> { let right = parquet_exec_with_output_partitioning(range_partitioning( "a", [30, 20, 10], - SortOptions::default().with_descending(true), + SortOptions { + descending: true, + nulls_first: true, + }, )?); let join_on = vec![( Arc::new(Column::new_with_schema("a", &left.schema())?) as _, @@ -1117,34 +992,16 @@ fn left_hash_join_range_rehash_incompatible_sort_options() -> Result<()> { .with_query_execution_partitions(4) .to_plan(join, &DISTRIB_DISTRIB_SORT); - println!("{}", displayable(plan.as_ref()).indent(true)); - - Ok(()) -} - -#[test] -fn left_hash_join_range_rehash_incompatible_key_expressions() -> Result<()> { - let left = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let right = parquet_exec_with_output_partitioning(range_partitioning( - "a", - [10, 20, 30], - SortOptions::default(), - )?); - let join_on = vec![( - Arc::new(Column::new_with_schema("b", &left.schema())?) as _, - Arc::new(Column::new_with_schema("b", &right.schema())?) as _, - )]; - let join = hash_join_exec(left, right, &join_on, &JoinType::LeftMark); - - let plan = TestConfig::default() - .with_query_execution_partitions(4) - .to_plan(join, &DISTRIB_DISTRIB_SORT); - - println!("{}", displayable(plan.as_ref()).indent(true)); + assert_plan!( + plan, + @r" + HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(a@0, a@0)] + RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4 + DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC], [(10), (20), (30)], 4), file_type=parquet + RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4 + DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 DESC], [(30), (20), (10)], 4), file_type=parquet + " + ); Ok(()) } diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 6f60155ea34a4..4426a53bcc26c 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -1293,7 +1293,16 @@ impl ExecutionPlan for HashJoinExec { ]), }; - if self.mode == PartitionMode::Partitioned && self.join_type == JoinType::Inner { + if self.mode == PartitionMode::Partitioned + && matches!( + self.join_type, + JoinType::Inner + | JoinType::Left + | JoinType::LeftSemi + | JoinType::LeftAnti + | JoinType::LeftMark + ) + { requirements.allow_range_satisfaction_for_key_partitioning() } else { requirements diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rs index 4c8545fecaa16..8c2142568cedf 100644 --- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs +++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs @@ -121,7 +121,7 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) { "range_partitioned_shifted", Path::new(env!("CARGO_MANIFEST_DIR")) .join("test_files/scratch_range_partitioning/range_partitioned_shifted"), - schema, + Arc::clone(&schema), [ "1,1,10\n5,2,50\n10,1,100\n", "15,2,150\n", @@ -130,6 +130,34 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) { ], Some(shifted_output_partitioning), ); + + // Same rows as `range_partitioned` but split into only three range + // partitions on `range_key`. Used to exercise the co-partition check when + // two Range inputs disagree on partition count. + let narrow_output_partitioning = Partitioning::Range( + RangePartitioning::try_new( + vec![col("range_key").sort(true, true)], + vec![ + SplitPoint::new(vec![ScalarValue::Int32(Some(10))]), + SplitPoint::new(vec![ScalarValue::Int32(Some(20))]), + ], + ) + .expect("range partitioning should be valid"), + ); + + register_csv_listing_table( + ctx, + "range_partitioned_narrow", + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("test_files/scratch_range_partitioning/range_partitioned_narrow"), + schema, + [ + "1,1,10\n5,2,50\n", + "10,1,100\n15,2,150\n", + "20,1,200\n25,2,250\n30,1,300\n35,2,350\n", + ], + Some(narrow_output_partitioning), + ); } fn register_csv_listing_table( diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt index 0bef0f0adaed1..ace752db686fd 100644 --- a/datafusion/sqllogictest/test_files/range_partitioning.slt +++ b/datafusion/sqllogictest/test_files/range_partitioning.slt @@ -395,28 +395,123 @@ ORDER BY l.non_range_key, l.value, r.value; 2 350 350 ########## -# TEST 12: Left Range Join Repartitions -# Only inner and right-side (Right/RightSemi/RightAnti/RightMark) partitioned -# hash joins opt in to Range satisfying KeyPartitioned requirements. Other -# join types, such as Left, keep using Hash repartitioning. +# TEST 12: Left-Side Range Hash Joins +# Left-side partitioned hash joins also opt in to Range satisfying +# KeyPartitioned requirements. Compatible Range layouts satisfy both the +# per-child key requirements and the cross-child layout requirement, so no +# Hash repartitioning is inserted. Incompatible split points, partition counts, +# or join key expressions still use Hash repartitioning to repair the inputs. ########## query TT EXPLAIN SELECT l.range_key, l.value, r.value FROM range_partitioned l -LEFT JOIN range_partitioned r ON l.range_key = r.range_key; +LEFT JOIN (SELECT range_key, value FROM range_partitioned WHERE value <= 150) r +ON l.range_key = r.range_key; +---- +physical_plan +01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)--FilterExec: value@1 <= 150 +04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false + +query III +SELECT l.range_key, l.value, r.value +FROM range_partitioned l +LEFT JOIN (SELECT range_key, value FROM range_partitioned WHERE value <= 150) r +ON l.range_key = r.range_key +ORDER BY l.range_key; +---- +1 10 10 +5 50 50 +10 100 100 +15 150 150 +20 200 NULL +25 250 NULL +30 300 NULL +35 350 NULL + +query TT +EXPLAIN SELECT l.range_key, l.value +FROM range_partitioned l +LEFT SEMI JOIN (SELECT range_key FROM range_partitioned WHERE value <= 150) r +ON l.range_key = r.range_key; +---- +physical_plan +01)HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(range_key@0, range_key@0)] +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)--FilterExec: value@1 <= 150, projection=[range_key@0] +04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false + +query II +SELECT l.range_key, l.value +FROM range_partitioned l +LEFT SEMI JOIN (SELECT range_key FROM range_partitioned WHERE value <= 150) r +ON l.range_key = r.range_key +ORDER BY l.range_key; +---- +1 10 +5 50 +10 100 +15 150 + +query TT +EXPLAIN SELECT l.range_key, l.value +FROM range_partitioned l +LEFT ANTI JOIN (SELECT range_key FROM range_partitioned WHERE value <= 150) r +ON l.range_key = r.range_key; +---- +physical_plan +01)HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(range_key@0, range_key@0)] +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +03)--FilterExec: value@1 <= 150, projection=[range_key@0] +04)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false + +query II +SELECT l.range_key, l.value +FROM range_partitioned l +LEFT ANTI JOIN (SELECT range_key FROM range_partitioned WHERE value <= 150) r +ON l.range_key = r.range_key +ORDER BY l.range_key; +---- +20 200 +25 250 +30 300 +35 350 + +# Range([range_key]) is only a subset of the composite join key, so the +# co-partitioned hash join requirement is repaired with Hash repartitioning. +query TT +EXPLAIN SELECT l.range_key, l.non_range_key, l.value, r.value +FROM range_partitioned l +LEFT JOIN (SELECT range_key, non_range_key, value FROM range_partitioned WHERE value <= 150) r +ON l.range_key = r.range_key AND l.non_range_key = r.non_range_key; +---- +physical_plan +01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0), (non_range_key@1, non_range_key@1)], projection=[range_key@0, non_range_key@1, value@2, value@5] +02)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4 +03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4 +05)----FilterExec: value@2 <= 150 +06)------DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false + +# Different split points do not satisfy the co-partitioned layout requirement. +query TT +EXPLAIN SELECT l.range_key, l.value, r.value +FROM range_partitioned l +LEFT JOIN range_partitioned_shifted r ON l.range_key = r.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] 02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false 04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 -05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false query III SELECT l.range_key, l.value, r.value FROM range_partitioned l -LEFT JOIN range_partitioned r ON l.range_key = r.range_key +LEFT JOIN range_partitioned_shifted r ON l.range_key = r.range_key ORDER BY l.range_key; ---- 1 10 10 @@ -428,6 +523,77 @@ ORDER BY l.range_key; 30 300 300 35 350 350 +# Different partition counts do not satisfy the co-partitioned layout +# requirement. +query TT +EXPLAIN SELECT l.range_key, l.value, r.value +FROM range_partitioned l +LEFT JOIN range_partitioned_narrow r ON l.range_key = r.range_key; +---- +physical_plan +01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] +02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4 +03)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=3 +05)----DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20)], 3), file_type=csv, has_header=false + +# Range([range_key]) does not satisfy a join keyed on non_range_key. +query TT +EXPLAIN SELECT l.range_key, l.non_range_key, l.value, r.value +FROM range_partitioned l +LEFT JOIN range_partitioned r ON l.non_range_key = r.non_range_key; +---- +physical_plan +01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(non_range_key@1, non_range_key@0)], projection=[range_key@0, non_range_key@1, value@2, value@4] +02)--RepartitionExec: partitioning=Hash([non_range_key@1], 4), input_partitions=4 +03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4 +05)----DataSourceExec: file_groups=, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false + +# SQL IN subqueries decorrelate to LeftMark joins. These queries pin matched, +# unmatched, and NULL marker behavior over compatible Range inputs. +query TT +EXPLAIN SELECT l.range_key, l.value +FROM range_partitioned l +WHERE l.non_range_key = 2 OR l.range_key IN ( + SELECT range_key FROM range_partitioned WHERE value <= 150); +---- +physical_plan +01)FilterExec: non_range_key@1 = 2 OR mark@3, projection=[range_key@0, value@2] +02)--HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(range_key@0, range_key@0)] +03)----DataSourceExec: file_groups=, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false +04)----FilterExec: value@1 <= 150, projection=[range_key@0] +05)------DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false + +query II +SELECT l.range_key, l.value +FROM range_partitioned l +WHERE l.non_range_key = 2 OR l.range_key IN ( + SELECT range_key FROM range_partitioned WHERE value <= 150) +ORDER BY l.range_key; +---- +1 10 +5 50 +10 100 +15 150 +25 250 +35 350 + +query II +SELECT l.range_key, l.value +FROM range_partitioned l +WHERE l.non_range_key = 2 OR l.range_key IN ( + SELECT CASE WHEN value <= 150 THEN range_key ELSE NULL END + FROM range_partitioned) +ORDER BY l.range_key; +---- +1 10 +5 50 +10 100 +15 150 +25 250 +35 350 + ########## # TEST 13: Compatible Range Join Repartitions to Increase Parallelism # Co-partitioning satisfaction does not prevent a repartition that increases