diff --git a/datafusion/common/src/scalar/consts.rs b/datafusion/common/src/scalar/consts.rs index 599c2523cd2c7..df12265a3723c 100644 --- a/datafusion/common/src/scalar/consts.rs +++ b/datafusion/common/src/scalar/consts.rs @@ -17,6 +17,9 @@ // Constants defined for scalar construction. +use arrow::datatypes::{Decimal32Type, Decimal64Type, Decimal128Type, DecimalType}; +use arrow::datatypes::{Decimal256Type, i256}; + // Next F16 value above π (upper bound) pub(super) const PI_UPPER_F16: half::f16 = half::f16::from_bits(0x4249); @@ -54,3 +57,63 @@ pub(super) const NEGATIVE_FRAC_PI_2_LOWER_F32: f32 = // Next f64 value below -π/2 (lower bound) pub(super) const NEGATIVE_FRAC_PI_2_LOWER_F64: f64 = (-std::f64::consts::FRAC_PI_2).next_down(); + +// Generate lookup table for 1 values of decimals (1, 10, 100, etc.) +macro_rules! decimal_ones_lut { + () => {{ + let mut values = [1; _]; + let mut i = 1; + while i < values.len() { + values[i] = values[i - 1] * 10; + i += 1; + } + values + }}; +} + +// 1, 10, 100 values meant to be indexed by scale. We omit handling for MAX_SCALE +// itself (we don't go to MAX_SCALE + 1) since we can't represent a 1 value at +// that scale. +pub(super) const DECIMAL32_ONES: [i32; Decimal32Type::MAX_SCALE as usize] = + decimal_ones_lut!(); +pub(super) const DECIMAL64_ONES: [i64; Decimal64Type::MAX_SCALE as usize] = + decimal_ones_lut!(); +pub(super) const DECIMAL128_ONES: [i128; Decimal128Type::MAX_SCALE as usize] = + decimal_ones_lut!(); +pub(super) const DECIMAL256_ONES: [i256; Decimal256Type::MAX_SCALE as usize] = { + // This code was generated by codex and frankly I don't know how it works, + // but the test below verifies it outputs the correct values so ¯\_(ツ)_/¯ + // + // This is mainly a shortcut for not needing to manually list out each value + // anyway. + // + // TODO: simplify this after https://github.com/apache/arrow-rs/pull/10363 + // lands upstream + let mut values = [i256::ONE; _]; + let mut i = 1; + while i < values.len() { + let (low, high) = values[i - 1].to_parts(); + let low_product = (low as u64 as u128) * 10; + let high_product = (low >> 64) * 10 + (low_product >> 64); + let low = ((high_product as u64 as u128) << 64) | low_product as u64 as u128; + let carry = (high_product >> 64) as i128; + values[i] = i256::from_parts(low, high * 10 + carry); + i += 1; + } + values +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ensure_correct_decimal256_ones() { + for (scale, val) in DECIMAL256_ONES.iter().enumerate() { + let zeros = "0".repeat(scale); + let num = "1".to_string() + &zeros; + let num = i256::from_string(&num).unwrap(); + assert_eq!(num, *val, "{scale}"); + } + } +} diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs index ddfe32edd41cc..924620a930869 100644 --- a/datafusion/common/src/scalar/mod.rs +++ b/datafusion/common/src/scalar/mod.rs @@ -54,6 +54,9 @@ use crate::cast::{ use crate::error::{_exec_err, _internal_err, _not_impl_err, DataFusionError, Result}; use crate::format::DEFAULT_CAST_OPTIONS; use crate::hash_utils::create_hashes; +use crate::scalar::consts::{ + DECIMAL32_ONES, DECIMAL64_ONES, DECIMAL128_ONES, DECIMAL256_ONES, +}; use crate::utils::SingleRowListArrayBuilder; use crate::{_internal_datafusion_err, arrow_datafusion_err}; use arrow::array::{ @@ -83,10 +86,14 @@ use arrow::datatypes::{ Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType, Field, FieldRef, Float32Type, Int8Type, Int16Type, Int32Type, Int64Type, IntervalDayTime, IntervalDayTimeType, IntervalMonthDayNano, IntervalMonthDayNanoType, IntervalUnit, - IntervalYearMonthType, RunEndIndexType, TimeUnit, TimestampMicrosecondType, - TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt8Type, - UInt16Type, UInt32Type, UInt64Type, UnionFields, UnionMode, i256, - validate_decimal_precision_and_scale, + IntervalYearMonthType, MAX_DECIMAL32_FOR_EACH_PRECISION, + MAX_DECIMAL64_FOR_EACH_PRECISION, MAX_DECIMAL128_FOR_EACH_PRECISION, + MAX_DECIMAL256_FOR_EACH_PRECISION, MIN_DECIMAL32_FOR_EACH_PRECISION, + MIN_DECIMAL64_FOR_EACH_PRECISION, MIN_DECIMAL128_FOR_EACH_PRECISION, + MIN_DECIMAL256_FOR_EACH_PRECISION, RunEndIndexType, TimeUnit, + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, UnionFields, + UnionMode, i256, validate_decimal_precision_and_scale, }; use arrow::util::display::{ArrayFormatter, FormatOptions, array_value_to_string}; use cache::{get_or_create_cached_key_array, get_or_create_cached_null_array}; @@ -1804,48 +1811,56 @@ impl ScalarValue { *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match 10_i32.checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal32(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL32_ONES[*scale as usize]; + ScalarValue::Decimal32(Some(one), *precision, *scale) } DataType::Decimal64(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i64::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal64(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL64_ONES[*scale as usize]; + ScalarValue::Decimal64(Some(one), *precision, *scale) } DataType::Decimal128(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i128::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal128(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL128_ONES[*scale as usize]; + ScalarValue::Decimal128(Some(one), *precision, *scale) } DataType::Decimal256(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i256::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal256(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL256_ONES[*scale as usize]; + ScalarValue::Decimal256(Some(one), *precision, *scale) } _ => { return _not_impl_err!( @@ -1858,10 +1873,10 @@ impl ScalarValue { /// Create a negative one value in the given type. pub fn new_negative_one(datatype: &DataType) -> Result { Ok(match datatype { - DataType::Int8 | DataType::UInt8 => ScalarValue::Int8(Some(-1)), - DataType::Int16 | DataType::UInt16 => ScalarValue::Int16(Some(-1)), - DataType::Int32 | DataType::UInt32 => ScalarValue::Int32(Some(-1)), - DataType::Int64 | DataType::UInt64 => ScalarValue::Int64(Some(-1)), + DataType::Int8 => ScalarValue::Int8(Some(-1)), + DataType::Int16 => ScalarValue::Int16(Some(-1)), + DataType::Int32 => ScalarValue::Int32(Some(-1)), + DataType::Int64 => ScalarValue::Int64(Some(-1)), DataType::Float16 => ScalarValue::Float16(Some(f16::NEG_ONE)), DataType::Float32 => ScalarValue::Float32(Some(-1.0)), DataType::Float64 => ScalarValue::Float64(Some(-1.0)), @@ -1870,48 +1885,56 @@ impl ScalarValue { *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match 10_i32.checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal32(Some(-value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent negative one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL32_ONES[*scale as usize]; + ScalarValue::Decimal32(Some(-one), *precision, *scale) } DataType::Decimal64(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i64::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal64(Some(-value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent negative one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL64_ONES[*scale as usize]; + ScalarValue::Decimal64(Some(-one), *precision, *scale) } DataType::Decimal128(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i128::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal128(Some(-value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent negative one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL128_ONES[*scale as usize]; + ScalarValue::Decimal128(Some(-one), *precision, *scale) } DataType::Decimal256(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i256::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal256(Some(-value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL256_ONES[*scale as usize]; + ScalarValue::Decimal256(Some(-one), *precision, *scale) } _ => { return _not_impl_err!( @@ -1939,48 +1962,64 @@ impl ScalarValue { *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match 10_i32.checked_pow((*scale + 1) as u32) { - Some(value) => { - ScalarValue::Decimal32(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + (*precision - *scale as u8) > 1, + "Can't represent ten at scale {} with precision {}", + *scale, + *precision + ); + // +1 safe since we validate above that scale must be less than + // the max possible scale + let ten = DECIMAL32_ONES[*scale as usize + 1]; + ScalarValue::Decimal32(Some(ten), *precision, *scale) } DataType::Decimal64(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i64::from(10).checked_pow((*scale + 1) as u32) { - Some(value) => { - ScalarValue::Decimal64(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + (*precision - *scale as u8) > 1, + "Can't represent ten at scale {} with precision {}", + *scale, + *precision + ); + // +1 safe since we validate above that scale must be less than + // the max possible scale + let ten = DECIMAL64_ONES[*scale as usize + 1]; + ScalarValue::Decimal64(Some(ten), *precision, *scale) } DataType::Decimal128(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i128::from(10).checked_pow((*scale + 1) as u32) { - Some(value) => { - ScalarValue::Decimal128(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + (*precision - *scale as u8) > 1, + "Can't represent ten at scale {} with precision {}", + *scale, + *precision + ); + // +1 safe since we validate above that scale must be less than + // the max possible scale + let ten = DECIMAL128_ONES[*scale as usize + 1]; + ScalarValue::Decimal128(Some(ten), *precision, *scale) } DataType::Decimal256(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i256::from(10).checked_pow((*scale + 1) as u32) { - Some(value) => { - ScalarValue::Decimal256(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + (*precision - *scale as u8) > 1, + "Can't represent ten at scale {} with precision {}", + *scale, + *precision + ); + // +1 safe since we validate above that scale must be less than + // the max possible scale + let ten = DECIMAL256_ONES[*scale as usize + 1]; + ScalarValue::Decimal256(Some(ten), *precision, *scale) } _ => { return _not_impl_err!( @@ -2299,7 +2338,18 @@ impl ScalarValue { | ScalarValue::Int64(None) | ScalarValue::Float16(None) | ScalarValue::Float32(None) - | ScalarValue::Float64(None) => Ok(self.clone()), + | ScalarValue::Float64(None) + | ScalarValue::IntervalYearMonth(None) + | ScalarValue::IntervalDayTime(None) + | ScalarValue::IntervalMonthDayNano(None) + | ScalarValue::Decimal32(None, _, _) + | ScalarValue::Decimal64(None, _, _) + | ScalarValue::Decimal128(None, _, _) + | ScalarValue::Decimal256(None, _, _) + | ScalarValue::TimestampSecond(None, _) + | ScalarValue::TimestampMillisecond(None, _) + | ScalarValue::TimestampMicrosecond(None, _) + | ScalarValue::TimestampNanosecond(None, _) => Ok(self.clone()), ScalarValue::Float16(Some(v)) => Ok(ScalarValue::Float16(Some(-v))), ScalarValue::Float64(Some(v)) => Ok(ScalarValue::Float64(Some(-v))), ScalarValue::Float32(Some(v)) => Ok(ScalarValue::Float32(Some(-v))), @@ -5053,28 +5103,21 @@ impl ScalarValue { DataType::Float16 => Some(ScalarValue::Float16(Some(f16::NEG_INFINITY))), DataType::Float32 => Some(ScalarValue::Float32(Some(f32::NEG_INFINITY))), DataType::Float64 => Some(ScalarValue::Float64(Some(f64::NEG_INFINITY))), + DataType::Decimal32(precision, scale) => { + let min = MIN_DECIMAL32_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal32(Some(min), *precision, *scale)) + } + DataType::Decimal64(precision, scale) => { + let min = MIN_DECIMAL64_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal64(Some(min), *precision, *scale)) + } DataType::Decimal128(precision, scale) => { - // For decimal, min is -10^(precision-scale) + 10^(-scale) - // But for simplicity, we use the minimum i128 value that fits the precision - let max_digits = 10_i128.pow(*precision as u32) - 1; - Some(ScalarValue::Decimal128( - Some(-max_digits), - *precision, - *scale, - )) + let min = MIN_DECIMAL128_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal128(Some(min), *precision, *scale)) } DataType::Decimal256(precision, scale) => { - // Similar to Decimal128 but with i256 - // For now, use a large negative value - let max_digits = i256::from_i128(10_i128) - .checked_pow(*precision as u32) - .and_then(|v| v.checked_sub(i256::from_i128(1))) - .unwrap_or(i256::MAX); - Some(ScalarValue::Decimal256( - Some(max_digits.neg_wrapping()), - *precision, - *scale, - )) + let min = MIN_DECIMAL256_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal256(Some(min), *precision, *scale)) } DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MIN))), DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MIN))), @@ -5149,27 +5192,21 @@ impl ScalarValue { DataType::Float16 => Some(ScalarValue::Float16(Some(f16::INFINITY))), DataType::Float32 => Some(ScalarValue::Float32(Some(f32::INFINITY))), DataType::Float64 => Some(ScalarValue::Float64(Some(f64::INFINITY))), + DataType::Decimal32(precision, scale) => { + let max = MAX_DECIMAL32_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal32(Some(max), *precision, *scale)) + } + DataType::Decimal64(precision, scale) => { + let max = MAX_DECIMAL64_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal64(Some(max), *precision, *scale)) + } DataType::Decimal128(precision, scale) => { - // For decimal, max is 10^(precision-scale) - 10^(-scale) - // But for simplicity, we use the maximum i128 value that fits the precision - let max_digits = 10_i128.pow(*precision as u32) - 1; - Some(ScalarValue::Decimal128( - Some(max_digits), - *precision, - *scale, - )) + let max = MAX_DECIMAL128_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal128(Some(max), *precision, *scale)) } DataType::Decimal256(precision, scale) => { - // Similar to Decimal128 but with i256 - let max_digits = i256::from_i128(10_i128) - .checked_pow(*precision as u32) - .and_then(|v| v.checked_sub(i256::from_i128(1))) - .unwrap_or(i256::MAX); - Some(ScalarValue::Decimal256( - Some(max_digits), - *precision, - *scale, - )) + let max = MAX_DECIMAL256_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal256(Some(max), *precision, *scale)) } DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MAX))), DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MAX))), @@ -11470,4 +11507,41 @@ mod tests { .unwrap(); assert_eq!(s.to_string(), "[]"); } + + #[test] + fn test_decimal_value_bounds() { + fn run_tests() { + // 0.1111, 0.2222, etc. + let max_scale = D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE); + // 1.111, 2.222, etc. + let max_scale_less_one = + D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE - 1); + // 11.11, 22.22, etc. + let max_scale_less_two = + D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE - 2); + + // Invalid (can't represent the value) + assert!(ScalarValue::new_one(&max_scale).is_err()); + assert!(ScalarValue::new_negative_one(&max_scale).is_err()); + assert!(ScalarValue::new_ten(&max_scale).is_err()); + assert!(ScalarValue::new_ten(&max_scale_less_one).is_err()); + + // Valid + let one = ScalarValue::Int32(Some(1)); + let neg_one = ScalarValue::Int32(Some(-1)); + let ten = ScalarValue::Int32(Some(10)); + + let num = ScalarValue::new_one(&max_scale_less_one).unwrap(); + assert_eq!(num.cast_to(&DataType::Int32).unwrap(), one); + let num = ScalarValue::new_negative_one(&max_scale_less_one).unwrap(); + assert_eq!(num.cast_to(&DataType::Int32).unwrap(), neg_one); + let num = ScalarValue::new_ten(&max_scale_less_two).unwrap(); + assert_eq!(num.cast_to(&DataType::Int32).unwrap(), ten); + } + + run_tests::(); + run_tests::(); + run_tests::(); + run_tests::(); + } } diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 39c8541b51b2f..9abc3faa2b32b 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -3062,17 +3062,6 @@ mod tests { #[test] fn test_simplify_negated_bitwise_and() { - // !c4 & c4 --> 0 - let expr = (-col("c4_non_null")) & col("c4_non_null"); - let expected = lit(0u32); - - assert_eq!(simplify(expr), expected); - // c4 & !c4 --> 0 - let expr = col("c4_non_null") & (-col("c4_non_null")); - let expected = lit(0u32); - - assert_eq!(simplify(expr), expected); - // !c3 & c3 --> 0 let expr = (-col("c3_non_null")) & col("c3_non_null"); let expected = lit(0i64); @@ -3087,18 +3076,6 @@ mod tests { #[test] fn test_simplify_negated_bitwise_or() { - // !c4 | c4 --> -1 - let expr = (-col("c4_non_null")) | col("c4_non_null"); - let expected = lit(-1i32); - - assert_eq!(simplify(expr), expected); - - // c4 | !c4 --> -1 - let expr = col("c4_non_null") | (-col("c4_non_null")); - let expected = lit(-1i32); - - assert_eq!(simplify(expr), expected); - // !c3 | c3 --> -1 let expr = (-col("c3_non_null")) | col("c3_non_null"); let expected = lit(-1i64); @@ -3114,18 +3091,6 @@ mod tests { #[test] fn test_simplify_negated_bitwise_xor() { - // !c4 ^ c4 --> -1 - let expr = (-col("c4_non_null")) ^ col("c4_non_null"); - let expected = lit(-1i32); - - assert_eq!(simplify(expr), expected); - - // c4 ^ !c4 --> -1 - let expr = col("c4_non_null") ^ (-col("c4_non_null")); - let expected = lit(-1i32); - - assert_eq!(simplify(expr), expected); - // !c3 ^ c3 --> -1 let expr = (-col("c3_non_null")) ^ col("c3_non_null"); let expected = lit(-1i64); diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs index b0908b47602f7..89bb762d59ce2 100644 --- a/datafusion/optimizer/src/simplify_expressions/utils.rs +++ b/datafusion/optimizer/src/simplify_expressions/utils.rs @@ -17,7 +17,6 @@ //! Utility functions for expression simplification -use arrow::datatypes::i256; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ Case, Expr, Like, Operator, @@ -25,47 +24,6 @@ use datafusion_expr::{ expr_fn::{and, bitwise_and, bitwise_or, or}, }; -pub static POWS_OF_TEN: [i128; 38] = [ - 1, - 10, - 100, - 1000, - 10000, - 100000, - 1000000, - 10000000, - 100000000, - 1000000000, - 10000000000, - 100000000000, - 1000000000000, - 10000000000000, - 100000000000000, - 1000000000000000, - 10000000000000000, - 100000000000000000, - 1000000000000000000, - 10000000000000000000, - 100000000000000000000, - 1000000000000000000000, - 10000000000000000000000, - 100000000000000000000000, - 1000000000000000000000000, - 10000000000000000000000000, - 100000000000000000000000000, - 1000000000000000000000000000, - 10000000000000000000000000000, - 100000000000000000000000000000, - 1000000000000000000000000000000, - 10000000000000000000000000000000, - 100000000000000000000000000000000, - 1000000000000000000000000000000000, - 10000000000000000000000000000000000, - 100000000000000000000000000000000000, - 1000000000000000000000000000000000000, - 10000000000000000000000000000000000000, -]; - /// returns true if `needle` is found in a chain of search_op /// expressions. Such as: (A AND B) AND C fn expr_contains_inner(expr: &Expr, needle: &Expr, search_op: Operator) -> bool { @@ -139,54 +97,26 @@ pub fn delete_xor_in_complex_expr(expr: &Expr, needle: &Expr, is_left: bool) -> } pub fn is_zero(s: &Expr) -> bool { - match s { - Expr::Literal(ScalarValue::Int8(Some(0)), _) - | Expr::Literal(ScalarValue::Int16(Some(0)), _) - | Expr::Literal(ScalarValue::Int32(Some(0)), _) - | Expr::Literal(ScalarValue::Int64(Some(0)), _) - | Expr::Literal(ScalarValue::UInt8(Some(0)), _) - | Expr::Literal(ScalarValue::UInt16(Some(0)), _) - | Expr::Literal(ScalarValue::UInt32(Some(0)), _) - | Expr::Literal(ScalarValue::UInt64(Some(0)), _) => true, - Expr::Literal(ScalarValue::Float32(Some(v)), _) if *v == 0. => true, - Expr::Literal(ScalarValue::Float64(Some(v)), _) if *v == 0. => true, - Expr::Literal(ScalarValue::Decimal128(Some(v), _p, _s), _) if *v == 0 => true, - Expr::Literal(ScalarValue::Decimal256(Some(v), _p, _s), _) - if *v == i256::ZERO => - { - true - } - _ => false, + if let Expr::Literal(sv, _) = s + && sv.data_type().is_numeric() + { + // unwrap safe since numeric types always have a 0 value + sv == &ScalarValue::new_zero(&sv.data_type()).unwrap() + } else { + false } } pub fn is_one(s: &Expr) -> bool { - match s { - Expr::Literal(ScalarValue::Int8(Some(1)), _) - | Expr::Literal(ScalarValue::Int16(Some(1)), _) - | Expr::Literal(ScalarValue::Int32(Some(1)), _) - | Expr::Literal(ScalarValue::Int64(Some(1)), _) - | Expr::Literal(ScalarValue::UInt8(Some(1)), _) - | Expr::Literal(ScalarValue::UInt16(Some(1)), _) - | Expr::Literal(ScalarValue::UInt32(Some(1)), _) - | Expr::Literal(ScalarValue::UInt64(Some(1)), _) => true, - Expr::Literal(ScalarValue::Float32(Some(v)), _) if *v == 1. => true, - Expr::Literal(ScalarValue::Float64(Some(v)), _) if *v == 1. => true, - Expr::Literal(ScalarValue::Decimal128(Some(v), _p, s), _) => { - *s >= 0 - && POWS_OF_TEN - .get(*s as usize) - .map(|x| x == v) - .unwrap_or_default() - } - Expr::Literal(ScalarValue::Decimal256(Some(v), _p, s), _) => { - *s >= 0 - && match i256::from(10).checked_pow(*s as u32) { - Some(res) => res == *v, - None => false, - } - } - _ => false, + if let Expr::Literal(sv, _) = s + && sv.data_type().is_numeric() + // there are edge cases like negative scale decimals not being able to + // create a one value so this can fail + && let Ok(one) = ScalarValue::new_one(&sv.data_type()) + { + sv == &one + } else { + false } }