Skip to content

feat: support WindowGroupLimitExec #4870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
comphead wants to merge 13 commits into
base: main
Choose a base branch
from
Draft

feat: support WindowGroupLimitExec #4870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
8de60ac
feat: support `WindowGroupLimit`
comphead Jul 8, 2026
5e17824
feat: support `WindowGroupLimit`
comphead Jul 9, 2026
c844121
feat: support `WindowGroupLimit`
comphead Jul 23, 2026
f314a1a
feat: support `WindowGroupLimit`
comphead Jul 23, 2026
612b2e7
feat: support `WindowGroupLimit`
comphead Jul 23, 2026
5eb5b41
feat: support `WindowGroupLimit`
comphead Jul 23, 2026
3d56501
feat: support `WindowGroupLimit`
comphead Jul 24, 2026
7ad467f
feat: support `WindowGroupLimit`
comphead Jul 24, 2026
acf57e6
feat: support `WindowGroupLimit`
comphead Jul 24, 2026
95dd96e
feat: support `WindowGroupLimit`
comphead Jul 24, 2026
035a53c
feat: support `WindowGroupLimit`
comphead Jul 24, 2026
f3c2878
feat: support `WindowGroupLimit`
comphead Jul 25, 2026
d781362
feat: support `WindowGroupLimit`
comphead Jul 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions dev/diffs/3.5.8.diff
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1426,6 +1426,30 @@ index 005e764cc30..92ec088efab 100644
}

private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
index 46ed8fdfd21..4585dbab5b8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.execution

import org.apache.spark.sql.{DataFrame, QueryTest}
+import org.apache.spark.sql.comet.CometWindowGroupLimitExec
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
import org.apache.spark.sql.execution.window.WindowGroupLimitExec
import org.apache.spark.sql.functions.lit
@@ -30,7 +31,10 @@ abstract class RemoveRedundantWindowGroupLimitsSuiteBase

private def checkNumWindowGroupLimits(df: DataFrame, count: Int): Unit = {
val plan = df.queryExecution.executedPlan
- assert(collectWithSubqueries(plan) { case exec: WindowGroupLimitExec => exec }.length == count)
+ assert(collectWithSubqueries(plan) {
+ case exec: WindowGroupLimitExec => exec
+ case exec: CometWindowGroupLimitExec => exec
+ }.length == count)
}

private def checkWindowGroupLimits(query: String, count: Int): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
index 47679ed7865..9ffbaecb98e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
Expand Down
24 changes: 24 additions & 0 deletions dev/diffs/4.0.2.diff
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1828,6 +1828,30 @@ index 005e764cc30..92ec088efab 100644
}

private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
index 46ed8fdfd21..4585dbab5b8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.execution

import org.apache.spark.sql.{DataFrame, QueryTest}
+import org.apache.spark.sql.comet.CometWindowGroupLimitExec
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
import org.apache.spark.sql.execution.window.WindowGroupLimitExec
import org.apache.spark.sql.functions.lit
@@ -30,7 +31,10 @@ abstract class RemoveRedundantWindowGroupLimitsSuiteBase

private def checkNumWindowGroupLimits(df: DataFrame, count: Int): Unit = {
val plan = df.queryExecution.executedPlan
- assert(collectWithSubqueries(plan) { case exec: WindowGroupLimitExec => exec }.length == count)
+ assert(collectWithSubqueries(plan) {
+ case exec: WindowGroupLimitExec => exec
+ case exec: CometWindowGroupLimitExec => exec
+ }.length == count)
}

private def checkWindowGroupLimits(query: String, count: Int): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
index 47679ed7865..9ffbaecb98e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
Expand Down
24 changes: 24 additions & 0 deletions dev/diffs/4.1.2.diff
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1951,6 +1951,30 @@ index 005e764cc30..92ec088efab 100644
}

private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
index 46ed8fdfd21..4585dbab5b8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantWindowGroupLimitsSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.execution

import org.apache.spark.sql.{DataFrame, QueryTest}
+import org.apache.spark.sql.comet.CometWindowGroupLimitExec
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
import org.apache.spark.sql.execution.window.WindowGroupLimitExec
import org.apache.spark.sql.functions.lit
@@ -30,7 +31,10 @@ abstract class RemoveRedundantWindowGroupLimitsSuiteBase

private def checkNumWindowGroupLimits(df: DataFrame, count: Int): Unit = {
val plan = df.queryExecution.executedPlan
- assert(collectWithSubqueries(plan) { case exec: WindowGroupLimitExec => exec }.length == count)
+ assert(collectWithSubqueries(plan) {
+ case exec: WindowGroupLimitExec => exec
+ case exec: CometWindowGroupLimitExec => exec
+ }.length == count)
}

private def checkWindowGroupLimits(query: String, count: Int): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
index 47679ed7865..9ffbaecb98e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReplaceHashWithSortAggSuite.scala
Expand Down
1 change: 1 addition & 0 deletions native/core/src/execution/jni_api.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,7 @@ fn op_name(op: &OpStruct) -> &'static str {
OpStruct::CsvScan(_) => "CsvScan",
OpStruct::ShuffleScan(_) => "ShuffleScan",
OpStruct::BroadcastNestedLoopJoin(_) => "BroadcastNestedLoopJoin",
OpStruct::WindowGroupLimit(_) => "WindowGroupLimit",
}
}

Expand Down
2 changes: 2 additions & 0 deletions native/core/src/execution/operators/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ mod parquet_writer;
pub use parquet_writer::{ParquetCompression, ParquetWriterExec};
mod csv_scan;
pub mod projection;
mod rank_limit;
pub use rank_limit::{PartitionedRankLimitExec, WindowFnKind};
mod scan;
mod shuffle_scan;
pub use csv_scan::init_csv_datasource_exec;
Expand Down
Loading
Loading