diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index a4aedfc991..197adc8fe3 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -391,9 +391,12 @@ impl PhysicalPlanner { DataType::Timestamp(TimeUnit::Microsecond, Some(tz)) => { ScalarValue::TimestampMicrosecond(Some(*value), Some(tz)) } + DataType::Time64(TimeUnit::Nanosecond) => { + ScalarValue::Time64Nanosecond(Some(*value)) + } dt => { return Err(GeneralError(format!( - "Expected either 'Int64' or 'Timestamp' for LongVal, but found {dt:?}" + "Expected 'Int64', 'Timestamp', or 'Time64(Nanosecond)' for LongVal, but found {dt:?}" ))) } }, @@ -4262,6 +4265,7 @@ mod tests { }; use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema}; use datafusion::catalog::memory::DataSourceExec; + use datafusion::common::ScalarValue; use datafusion::config::TableParquetOptions; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::object_store::ObjectStoreUrl; @@ -4293,6 +4297,33 @@ mod tests { }; use datafusion_comet_spark_expr::EvalMode; + #[test] + fn create_time64_nanosecond_literal() { + let nanos = 45_296_123_456_000; + let expr = Expr { + expr_struct: Some(Literal(spark_expression::Literal { + value: Some(literal::Value::LongVal(nanos)), + datatype: Some(spark_expression::DataType { + type_id: 17, + type_info: None, + }), + is_null: false, + })), + query_context: None, + expr_id: None, + }; + + let planner = PhysicalPlanner::default(); + let physical_expr = planner + .create_expr(&expr, Arc::new(Schema::empty())) + .unwrap(); + let literal = physical_expr + .downcast_ref::() + .unwrap(); + + assert_eq!(literal.value(), &ScalarValue::Time64Nanosecond(Some(nanos))); + } + #[test] fn test_unpack_dictionary_primitive() { let op_scan = Operator { diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index f62db397cb..2b0cc02ad0 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -1111,6 +1111,8 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { _: DoubleType | _: DecimalType | _: DateType | _: TimestampType | _: TimestampNTZType | _: BooleanType | _: BinaryType | _: StringType => true + case dt if isTimeType(dt) => + true case _ => false } diff --git a/spark/src/main/scala/org/apache/comet/serde/aggregates.scala b/spark/src/main/scala/org/apache/comet/serde/aggregates.scala index 588d6a5907..991db1fb75 100644 --- a/spark/src/main/scala/org/apache/comet/serde/aggregates.scala +++ b/spark/src/main/scala/org/apache/comet/serde/aggregates.scala @@ -31,7 +31,7 @@ import org.apache.comet.CometConf.COMET_EXEC_STRICT_FLOATING_POINT import org.apache.comet.CometSparkSessionExtensions.{isSpark41Plus, withFallbackReason} import org.apache.comet.expressions.CometEvalMode import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto, exprToProto, serializeDataType} -import org.apache.comet.shims.CometEvalModeUtil +import org.apache.comet.shims.{CometEvalModeUtil, CometTypeShim} object CometMin extends CometAggregateExpressionSerde[Min] { @@ -960,7 +960,7 @@ object CometApproxCountDistinct extends CometAggregateExpressionSerde[HyperLogLo } } -object AggSerde { +object AggSerde extends CometTypeShim { import org.apache.spark.sql.types._ def minMaxDataTypeSupported(dt: DataType): Boolean = { @@ -970,6 +970,7 @@ object AggSerde { case FloatType | DoubleType => true case _: DecimalType => true case DateType | TimestampType => true + case dt if isTimeType(dt) => true case _ => false } } diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_min_max.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_min_max.sql new file mode 100644 index 0000000000..c6ee6f553f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_min_max.sql @@ -0,0 +1,55 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 4.1 +-- Config: spark.sql.timeType.enabled=true + +statement +CREATE TABLE test_make_time_min_max(grp string, hours int, minutes int, secs decimal(16,6)) USING parquet + +statement +INSERT INTO test_make_time_min_max VALUES + ('a', 12, 30, 45.123456), + ('a', 0, 0, 0.000000), + ('a', NULL, NULL, NULL), + ('b', 23, 59, 59.999999), + ('b', 1, 2, 3.500000), + ('c', NULL, 0, 0.000000), + ('c', 12, NULL, 0.000000), + ('c', 12, 30, NULL) + +query +SELECT + min(make_time(hours, minutes, secs)) AS min_time, + max(make_time(hours, minutes, secs)) AS max_time +FROM test_make_time_min_max + +query +SELECT + grp, + min(make_time(hours, minutes, secs)) AS min_time, + max(make_time(hours, minutes, secs)) AS max_time +FROM test_make_time_min_max +GROUP BY grp +ORDER BY grp + +query +SELECT + min(make_time(hours, minutes, secs)) AS min_time, + max(make_time(hours, minutes, secs)) AS max_time +FROM test_make_time_min_max +WHERE grp = 'c' diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_sort.sql b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_sort.sql new file mode 100644 index 0000000000..7d7edc580b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_sort.sql @@ -0,0 +1,81 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 4.1 +-- Config: spark.sql.timeType.enabled=true + +statement +CREATE TABLE test_make_time_sort(id int, hours int, minutes int, secs decimal(16,6)) USING parquet + +statement +INSERT INTO test_make_time_sort VALUES + (1, 12, 30, 45.123456), + (2, 0, 0, 0.000000), + (3, 23, 59, 59.999999), + (4, 1, 2, 3.500000), + (5, 0, 0, 0.000001), + (6, NULL, NULL, NULL), + (7, 0, 0, 0.000000) + +query +SELECT id, t +FROM ( + SELECT id, make_time(hours, minutes, secs) AS t + FROM test_make_time_sort +) +SORT BY t ASC NULLS FIRST + +query +SELECT id, t +FROM ( + SELECT id, make_time(hours, minutes, secs) AS t + FROM test_make_time_sort +) +SORT BY t DESC NULLS LAST + +query +SELECT id, t +FROM ( + SELECT id, make_time(hours, minutes, secs) AS t + FROM test_make_time_sort +) +SORT BY t ASC NULLS LAST + +query +SELECT id, t +FROM ( + SELECT id, make_time(hours, minutes, secs) AS t + FROM test_make_time_sort +) +SORT BY t DESC NULLS FIRST + +query +SELECT id, t +FROM ( + SELECT id, make_time(hours, minutes, secs) AS t + FROM test_make_time_sort +) +SORT BY t ASC NULLS LAST, id DESC + +query +SELECT id, t +FROM ( + SELECT id, make_time(hours, minutes, secs) AS t + FROM test_make_time_sort +) +WHERE t >= TIME '12:30:45.123456' +SORT BY t ASC