From 0f132a9bd7cff5056602a54e167f8f9463c2738b Mon Sep 17 00:00:00 2001 From: linfeng Date: Thu, 16 Jul 2026 21:45:57 +0800 Subject: [PATCH 1/2] handle null date and timestamp format arguments --- datafusion/functions/src/datetime/common.rs | 34 +++++++++++++------ .../test_files/datetime/dates.slt | 21 ++++++++++++ .../test_files/datetime/timestamps.slt | 21 ++++++++++++ 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/datafusion/functions/src/datetime/common.rs b/datafusion/functions/src/datetime/common.rs index 2db64beafa9b7..9a7f94bd5973f 100644 --- a/datafusion/functions/src/datetime/common.rs +++ b/datafusion/functions/src/datetime/common.rs @@ -32,7 +32,7 @@ use chrono::{DateTime, TimeZone, Utc}; use datafusion_common::cast::as_generic_string_array; use datafusion_common::{ DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, - internal_datafusion_err, unwrap_or_internal_err, + internal_datafusion_err, }; use datafusion_expr::ColumnarValue; @@ -353,9 +353,9 @@ where // if the first argument is a scalar utf8 all arguments are expected to be scalar utf8 ColumnarValue::Scalar(scalar) => match scalar.try_as_str() { Some(a) => { - let a = a.as_ref(); - // ASK: Why do we trust `a` to be non-null at this point? - let a = unwrap_or_internal_err!(a); + let Some(a) = a.as_ref() else { + return Ok(ColumnarValue::Scalar(scalar_value(dt, None)?)); + }; let mut ret = None; @@ -384,7 +384,10 @@ where } } - unwrap_or_internal_err!(ret) + match ret { + Some(ret) => ret, + None => Ok(ColumnarValue::Scalar(scalar_value(dt, None)?)), + } } other => { exec_err!("Unsupported data type {other:?} for function {name}") @@ -483,12 +486,21 @@ where if let Some(x) = x { for arg in args { let v = match arg { - ColumnarValue::Array(a) => match a.data_type() { - DataType::Utf8View => Ok(a.as_string_view().value(pos)), - DataType::LargeUtf8 => Ok(a.as_string::().value(pos)), - DataType::Utf8 => Ok(a.as_string::().value(pos)), - other => exec_err!("Unexpected type encountered '{other}'"), - }, + ColumnarValue::Array(a) => { + if a.is_null(pos) { + continue; + } + match a.data_type() { + DataType::Utf8View => Ok(a.as_string_view().value(pos)), + DataType::LargeUtf8 => { + Ok(a.as_string::().value(pos)) + } + DataType::Utf8 => Ok(a.as_string::().value(pos)), + other => { + exec_err!("Unexpected type encountered '{other}'") + } + } + } ColumnarValue::Scalar(s) => match s.try_as_str() { Some(Some(v)) => Ok(v), Some(None) => continue, // null string diff --git a/datafusion/sqllogictest/test_files/datetime/dates.slt b/datafusion/sqllogictest/test_files/datetime/dates.slt index d2a7360b120c6..68d87eceed99e 100644 --- a/datafusion/sqllogictest/test_files/datetime/dates.slt +++ b/datafusion/sqllogictest/test_files/datetime/dates.slt @@ -298,6 +298,27 @@ SELECT to_date('2020-09-08 12/00/00+00:00', '%c', '%+') query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input SELECT to_date('2020-09-08 12/00/00+00:00', '%q') +# NULL string scalar inputs and all-NULL scalar formats return NULL +query DD +SELECT + to_date(NULL::VARCHAR, '%Y-%m-%d'), + to_date('2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +---- +NULL NULL + +# NULL array formats are skipped for each row; rows with no usable format return NULL +query ID +SELECT id, to_date(value, format1, format2) +FROM ( + VALUES + (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'), + (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +) AS t(id, value, format1, format2) +ORDER BY id +---- +1 2020-09-08 +2 NULL + statement ok create table ts_utf8_data(ts varchar(100), format varchar(100)) as values ('2020-09-08 12/00/00+00:00', '%Y-%m-%d %H/%M/%S%#z'), diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index 8ba095ef934dd..463693f3f2ff1 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -518,6 +518,27 @@ SELECT COUNT(*) FROM ts_data_secs where ts > to_timestamp_seconds('2020-09-08 12 ---- 2 +# NULL string scalar inputs and all-NULL scalar formats return NULL +query PP +SELECT + to_timestamp(NULL::VARCHAR, '%Y-%m-%d'), + to_timestamp('2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +---- +NULL NULL + +# NULL array formats are skipped for each row; rows with no usable format return NULL +query IP +SELECT id, to_timestamp(value, format1, format2) +FROM ( + VALUES + (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'), + (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +) AS t(id, value, format1, format2) +ORDER BY id +---- +1 2020-09-08T00:00:00 +2 NULL + # to_timestamp float inputs query PPP From 9620484706c34d638791c7db066304f48e5e2219 Mon Sep 17 00:00:00 2001 From: linfeng Date: Thu, 16 Jul 2026 21:57:15 +0800 Subject: [PATCH 2/2] cover null format arguments in to_unixtime --- .../sqllogictest/test_files/datetime/timestamps.slt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index 463693f3f2ff1..9ac00e72b47e6 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -539,6 +539,14 @@ ORDER BY id 1 2020-09-08T00:00:00 2 NULL +# to_unixtime uses the same formatted string parsing path +query II +SELECT + to_unixtime(NULL::VARCHAR, '%Y-%m-%d'), + to_unixtime('2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +---- +NULL NULL + # to_timestamp float inputs query PPP