diff --git a/be/src/exprs/function/cast/cast_base.h b/be/src/exprs/function/cast/cast_base.h index cc58e29d4acd11..a34acbcf527a59 100644 --- a/be/src/exprs/function/cast/cast_base.h +++ b/be/src/exprs/function/cast/cast_base.h @@ -157,7 +157,7 @@ WrapperType create_identity_wrapper(const DataTypePtr&); } // namespace CastWrapper -enum class CastModeType { StrictMode, NonStrictMode }; +enum class CastModeType { StrictMode, NonStrictMode, LosslessMode, StrictLosslessMode }; inline std::string cast_mode_type_to_string(CastModeType cast_mode) { switch (cast_mode) { @@ -165,6 +165,10 @@ inline std::string cast_mode_type_to_string(CastModeType cast_mode) { return "StrictMode"; case CastModeType::NonStrictMode: return "NonStrictMode"; + case CastModeType::LosslessMode: + return "LosslessMode"; + case CastModeType::StrictLosslessMode: + return "StrictLosslessMode"; default: return "Unknown"; } diff --git a/be/src/exprs/function/cast/cast_to_decimal.h b/be/src/exprs/function/cast/cast_to_decimal.h index 8dc4c6b4aeac93..a35c1bbc47d92d 100644 --- a/be/src/exprs/function/cast/cast_to_decimal.h +++ b/be/src/exprs/function/cast/cast_to_decimal.h @@ -17,7 +17,11 @@ #pragma once +#include +#include #include +#include +#include #include #include "common/status.h" @@ -35,45 +39,174 @@ namespace doris { value, from_type_name, precision, scale)) struct CastToDecimal { + static bool is_lossless_decimal_string(const char* data, size_t size, UInt32 precision, + UInt32 target_scale) { + size_t begin = 0; + size_t end = size; + while (begin < end && std::isspace(static_cast(data[begin]))) { + ++begin; + } + while (begin < end && std::isspace(static_cast(data[end - 1]))) { + --end; + } + if (begin == end) { + return false; + } + + size_t pos = begin; + if (data[pos] == '+' || data[pos] == '-') { + ++pos; + } + + bool seen_dot = false; + bool seen_digit = false; + size_t digits_before_dot = 0; + size_t digits = 0; + size_t first_non_zero = 0; + size_t last_non_zero = 0; + while (pos < end && data[pos] != 'e' && data[pos] != 'E') { + unsigned char ch = static_cast(data[pos]); + if (std::isdigit(ch)) { + seen_digit = true; + ++digits; + if (!seen_dot) { + ++digits_before_dot; + } + if (ch != '0') { + if (first_non_zero == 0) { + first_non_zero = digits; + } + last_non_zero = digits; + } + } else if (data[pos] == '.' && !seen_dot) { + seen_dot = true; + } else { + return false; + } + ++pos; + } + if (!seen_digit) { + return false; + } + + int64_t exponent = 0; + if (pos < end) { + ++pos; + bool negative_exponent = false; + if (pos < end && (data[pos] == '+' || data[pos] == '-')) { + negative_exponent = data[pos] == '-'; + ++pos; + } + if (pos == end) { + return false; + } + + int64_t exponent_magnitude = 0; + bool exponent_overflow = false; + while (pos < end) { + unsigned char ch = static_cast(data[pos]); + if (!std::isdigit(ch)) { + return false; + } + if (!exponent_overflow) { + if (exponent_magnitude > + (std::numeric_limits::max() - (ch - '0')) / 10) { + exponent_overflow = true; + } else { + exponent_magnitude = exponent_magnitude * 10 + (ch - '0'); + } + } + ++pos; + } + if (exponent_overflow) { + return first_non_zero == 0; + } + exponent = negative_exponent ? -exponent_magnitude : exponent_magnitude; + } + + // Zero is exactly representable at every decimal precision and scale. + if (first_non_zero == 0) { + return true; + } + + size_t first_non_zero_offset = first_non_zero - 1; + if (digits_before_dot > std::numeric_limits::max() || + first_non_zero_offset > std::numeric_limits::max()) { + return false; + } + int64_t base_decimal_point = static_cast(digits_before_dot) - + static_cast(first_non_zero_offset); + + size_t significant_digits = last_non_zero - first_non_zero + 1; + if (significant_digits > precision) { + return false; + } + + if ((exponent > 0 && base_decimal_point > std::numeric_limits::max() - exponent) || + (exponent < 0 && base_decimal_point < std::numeric_limits::min() - exponent)) { + return false; + } + int64_t decimal_point = base_decimal_point + exponent; + int64_t max_integer_digits = precision - target_scale; + if (decimal_point > max_integer_digits || + decimal_point < -static_cast(target_scale)) { + return false; + } + int64_t required_scale = + std::max(static_cast(significant_digits) - decimal_point, int64_t {0}); + return required_scale <= target_scale; + } + template requires(IsDecimalNumber) - static inline bool from_string(const StringRef& from, ToCppT& to, UInt32 precision, - UInt32 scale, CastParameters& params) { + static inline StringParser::ParseResult parse_string(const StringRef& from, ToCppT& to, + UInt32 precision, UInt32 scale) { + StringParser::ParseResult result = StringParser::PARSE_SUCCESS; if constexpr (IsDecimalV2) { - StringParser::ParseResult result = StringParser::PARSE_SUCCESS; to = DecimalV2Value(StringParser::string_to_decimal( - from.data, (int)from.size, DecimalV2Value::PRECISION, DecimalV2Value::SCALE, + from.data, from.size, DecimalV2Value::PRECISION, DecimalV2Value::SCALE, &result)); - return result == StringParser::PARSE_SUCCESS; } if constexpr (IsDecimal32) { - StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - to.value = StringParser::string_to_decimal(from.data, (int)from.size, + to.value = StringParser::string_to_decimal(from.data, from.size, precision, scale, &result); - return result == StringParser::PARSE_SUCCESS; } if constexpr (IsDecimal64) { - StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - to.value = StringParser::string_to_decimal(from.data, (int)from.size, + to.value = StringParser::string_to_decimal(from.data, from.size, precision, scale, &result); - return result == StringParser::PARSE_SUCCESS; } if constexpr (IsDecimal128V3) { - StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - to.value = StringParser::string_to_decimal(from.data, (int)from.size, + to.value = StringParser::string_to_decimal(from.data, from.size, precision, scale, &result); - return result == StringParser::PARSE_SUCCESS; } if constexpr (IsDecimal256) { - StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - to.value = StringParser::string_to_decimal(from.data, (int)from.size, + to.value = StringParser::string_to_decimal(from.data, from.size, precision, scale, &result); - return result == StringParser::PARSE_SUCCESS; } + return result; + } + + template + requires(IsDecimalNumber) + static inline bool from_string(const StringRef& from, ToCppT& to, UInt32 precision, + UInt32 scale, CastParameters& params) { + return parse_string(from, to, precision, scale) == StringParser::PARSE_SUCCESS; + } + + template + requires(IsDecimalNumber) + static inline bool from_string_lossless(const StringRef& from, ToCppT& to, UInt32 precision, + UInt32 scale, CastParameters& params) { + if (!is_lossless_decimal_string(from.data, from.size, precision, scale)) { + return false; + } + StringParser::ParseResult result = parse_string(from, to, precision, scale); + return result == StringParser::PARSE_SUCCESS || result == StringParser::PARSE_OVERFLOW || + result == StringParser::PARSE_UNDERFLOW; } // cast int to decimal @@ -704,6 +837,49 @@ class CastToImpl : public CastToBase { RETURN_IF_ERROR( serde->from_string_strict_mode_batch(*col_from, *column_to, {}, null_map)); block.get_by_position(result).column = std::move(column_to); + } else if constexpr (Mode == CastModeType::LosslessMode || + Mode == CastModeType::StrictLosslessMode) { + auto nullable_col_to = create_empty_nullable_column(nested_to_type); + auto& nullable_to = *nullable_col_to; + nullable_to.resize(col_from->size()); + auto& column_to = + assert_cast(nullable_to.get_nested_column()); + auto& values_to = column_to.get_data(); + auto& null_map_to = nullable_to.get_null_map_data(); + auto* decimal_to_type = assert_cast(nested_to_type.get()); + const auto& chars = col_from->get_chars(); + const auto& offsets = col_from->get_offsets(); + size_t current_offset = 0; + UInt32 precision = decimal_to_type->get_precision(); + UInt32 scale = decimal_to_type->get_scale(); + [[maybe_unused]] CastParameters params; + params.is_strict = false; + for (size_t i = 0; i < col_from->size(); ++i) { + size_t next_offset = offsets[i]; + size_t string_size = next_offset - current_offset; + bool source_is_null = null_map != nullptr && null_map[i]; + values_to[i] = {}; + StringRef source(reinterpret_cast(&chars[current_offset]), + string_size); + bool parsed = false; + if constexpr (Mode == CastModeType::StrictLosslessMode) { + if (!source_is_null && + CastToDecimal::parse_string(source, values_to[i], precision, scale) != + StringParser::PARSE_SUCCESS) { + return Status::InvalidArgument("parse number fail, string: '{}'", + std::string(source.data, source.size)); + } + parsed = !source_is_null && CastToDecimal::is_lossless_decimal_string( + source.data, source.size, precision, scale); + } else { + parsed = !source_is_null && + CastToDecimal::from_string_lossless(source, values_to[i], precision, + scale, params); + } + null_map_to[i] = !parsed; + current_offset = next_offset; + } + block.get_by_position(result).column = std::move(nullable_col_to); } else { return Status::InternalError("Unsupported cast mode"); } diff --git a/be/src/exprs/function/cast/function_cast_decimal.cpp b/be/src/exprs/function/cast/function_cast_decimal.cpp index 797bebb5f07595..cd71bf71540ac5 100644 --- a/be/src/exprs/function/cast/function_cast_decimal.cpp +++ b/be/src/exprs/function/cast/function_cast_decimal.cpp @@ -32,7 +32,23 @@ WrapperType create_decimal_wrapper(FunctionContext* context, const DataTypePtr& using Types = std::decay_t; using FromDataType = typename Types::LeftType; if constexpr (type_allow_cast_to_decimal) { - if (context->enable_strict_mode()) { + if constexpr (std::is_same_v) { + if (context->enable_lossless_decimal_cast()) { + if (context->enable_strict_mode()) { + cast_impl = std::make_shared>(); + } else { + cast_impl = std::make_shared< + CastToImpl>(); + } + } else if (context->enable_strict_mode()) { + cast_impl = std::make_shared< + CastToImpl>(); + } else { + cast_impl = std::make_shared< + CastToImpl>(); + } + } else if (context->enable_strict_mode()) { cast_impl = std::make_shared< CastToImpl>(); } else { diff --git a/be/src/exprs/function_context.cpp b/be/src/exprs/function_context.cpp index 3592bcf4be8725..4284e062e1ccd5 100644 --- a/be/src/exprs/function_context.cpp +++ b/be/src/exprs/function_context.cpp @@ -56,6 +56,7 @@ std::unique_ptr FunctionContext::clone() { new_context->_fragment_local_fn_state = _fragment_local_fn_state; new_context->_check_overflow_for_decimal = _check_overflow_for_decimal; new_context->_enable_strict_mode = _enable_strict_mode; + new_context->_enable_lossless_decimal_cast = _enable_lossless_decimal_cast; new_context->_string_as_jsonb_string = _string_as_jsonb_string; new_context->_jsonb_string_as_string = _jsonb_string_as_string; return new_context; diff --git a/be/src/exprs/function_context.h b/be/src/exprs/function_context.h index ca0f1ed461038f..608826fff88b13 100644 --- a/be/src/exprs/function_context.h +++ b/be/src/exprs/function_context.h @@ -84,6 +84,8 @@ class FunctionContext { bool enable_strict_mode() const { return _enable_strict_mode; } + bool enable_lossless_decimal_cast() const { return _enable_lossless_decimal_cast; } + bool set_check_overflow_for_decimal(bool check_overflow_for_decimal) { return _check_overflow_for_decimal = check_overflow_for_decimal; } @@ -92,6 +94,10 @@ class FunctionContext { return _enable_strict_mode = enable_strict_mode; } + bool set_enable_lossless_decimal_cast(bool enable_lossless_decimal_cast) { + return _enable_lossless_decimal_cast = enable_lossless_decimal_cast; + } + void set_string_as_jsonb_string(bool string_as_jsonb_string) { _string_as_jsonb_string = string_as_jsonb_string; } @@ -198,6 +204,7 @@ class FunctionContext { RuntimeProfile::Counter* _udf_execute_timer = nullptr; bool _check_overflow_for_decimal = false; bool _enable_strict_mode = false; + bool _enable_lossless_decimal_cast = false; bool _string_as_jsonb_string = false; bool _jsonb_string_as_string = false; diff --git a/be/src/exprs/vcast_expr.cpp b/be/src/exprs/vcast_expr.cpp index 1435b40c58c58d..9435e3927c12e2 100644 --- a/be/src/exprs/vcast_expr.cpp +++ b/be/src/exprs/vcast_expr.cpp @@ -74,6 +74,8 @@ doris::Status VCastExpr::prepare(doris::RuntimeState* state, const doris::RowDes child->data_type()->get_name(), _target_data_type_name); } VExpr::register_function_context(state, context); + context->fn_context(_fn_context_index) + ->set_enable_lossless_decimal_cast(_lossless_decimal_cast); _expr_name = fmt::format("({} {}({}) TO {})", cast_name(), child_name, child->data_type()->get_name(), _target_data_type_name); _prepare_finished = true; diff --git a/be/src/exprs/vcast_expr.h b/be/src/exprs/vcast_expr.h index 4ac5539ff28224..5d2196f9e56f54 100644 --- a/be/src/exprs/vcast_expr.h +++ b/be/src/exprs/vcast_expr.h @@ -44,7 +44,8 @@ class VCastExpr : public VExpr { #ifdef BE_TEST VCastExpr() = default; #endif - VCastExpr(const TExprNode& node) : VExpr(node) {} + VCastExpr(const TExprNode& node) + : VExpr(node), _lossless_decimal_cast(node.lossless_decimal_cast) {} ~VCastExpr() override = default; Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const override; @@ -55,20 +56,24 @@ class VCastExpr : public VExpr { const std::string& expr_name() const override; std::string debug_string() const override; const DataTypePtr& get_target_type() const; + bool is_lossless_decimal_cast() const { return _lossless_decimal_cast; } bool is_safe_to_execute_on_selected_rows() const override { return false; } virtual std::string cast_name() const { return "CAST"; } Status clone_node(VExprSPtr* cloned_expr) const override { DORIS_CHECK(cloned_expr != nullptr); - *cloned_expr = VCastExpr::create_shared(clone_texpr_node()); + auto node = clone_texpr_node(); + node.__set_lossless_decimal_cast(_lossless_decimal_cast); + *cloned_expr = VCastExpr::create_shared(node); return Status::OK(); } uint64_t get_digest(uint64_t seed) const override { auto res = VExpr::get_digest(seed); if (res) { - return HashUtil::hash64(_target_data_type_name.data(), _target_data_type_name.size(), - res); + res = HashUtil::hash64(_target_data_type_name.data(), _target_data_type_name.size(), + res); + return HashUtil::hash64(&_lossless_decimal_cast, sizeof(_lossless_decimal_cast), res); } return 0; } @@ -82,6 +87,7 @@ class VCastExpr : public VExpr { std::string _target_data_type_name; DataTypePtr _cast_param_data_type; + bool _lossless_decimal_cast = false; static const constexpr char* function_name = "CAST"; }; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 313bbf376f860e..a5624e49273f2e 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -482,7 +482,8 @@ static Status clone_table_expr_node(const VExpr& expr, VExprSPtr* cloned_expr) { split_literal->original_type(), split_literal->original_field()); } else if (const auto* vcast_expr = dynamic_cast(&expr); vcast_expr != nullptr && vcast_expr->node_type() == TExprNodeType::CAST_EXPR) { - *cloned_expr = Cast::create_shared(vcast_expr->data_type()); + *cloned_expr = Cast::create_shared(vcast_expr->data_type(), + vcast_expr->is_lossless_decimal_cast()); } return Status::OK(); } diff --git a/be/src/format_v2/expr/cast.cpp b/be/src/format_v2/expr/cast.cpp index efeb9d851deb22..66c662df54b08a 100644 --- a/be/src/format_v2/expr/cast.cpp +++ b/be/src/format_v2/expr/cast.cpp @@ -62,6 +62,8 @@ Status Cast::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContex return Status::InternalError("Could not find function {} ", _expr_name); } VExpr::register_function_context(state, context); + context->fn_context(_fn_context_index) + ->set_enable_lossless_decimal_cast(_lossless_decimal_cast); _prepare_finished = true; return Status::OK(); } diff --git a/be/src/format_v2/expr/cast.h b/be/src/format_v2/expr/cast.h index 1dc06bcf07f2bc..71878743221df3 100644 --- a/be/src/format_v2/expr/cast.h +++ b/be/src/format_v2/expr/cast.h @@ -38,7 +38,8 @@ class Cast final : public VExpr { ENABLE_FACTORY_CREATOR(Cast); public: - Cast(const DataTypePtr& type) { + Cast(const DataTypePtr& type, bool lossless_decimal_cast = false) + : _lossless_decimal_cast(lossless_decimal_cast) { _node_type = TExprNodeType::CAST_EXPR; _opcode = TExprOpcode::CAST; _data_type = type; @@ -55,7 +56,7 @@ class Cast final : public VExpr { const std::string& expr_name() const override { return _expr_name; } Status clone_node(VExprSPtr* cloned_expr) const override { DORIS_CHECK(cloned_expr != nullptr); - *cloned_expr = Cast::create_shared(_data_type); + *cloned_expr = Cast::create_shared(_data_type, _lossless_decimal_cast); return Status::OK(); } @@ -64,5 +65,6 @@ class Cast final : public VExpr { size_t count, ColumnPtr& result_column) const; std::string _expr_name; FunctionBasePtr _function; + bool _lossless_decimal_cast = false; }; } // namespace doris::format diff --git a/be/test/exprs/function/cast/lossless_decimal_cast_test.cpp b/be/test/exprs/function/cast/lossless_decimal_cast_test.cpp new file mode 100644 index 00000000000000..0153ed6602247c --- /dev/null +++ b/be/test/exprs/function/cast/lossless_decimal_cast_test.cpp @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "exprs/function/cast/cast_to_decimal.h" + +namespace doris { + +TEST(LosslessDecimalCastTest, StringMustBeExactlyRepresentable) { + auto lossless = [](const char* value, UInt32 precision, UInt32 scale) { + return CastToDecimal::is_lossless_decimal_string(value, std::strlen(value), precision, + scale); + }; + + EXPECT_TRUE(lossless("9223372036854775808", 38, 0)); + EXPECT_TRUE(lossless("100.000", 38, 0)); + EXPECT_TRUE(lossless("1.25e2", 38, 0)); + EXPECT_TRUE(lossless("0.0100", 5, 2)); + EXPECT_TRUE(lossless("999.99", 5, 2)); + + EXPECT_FALSE(lossless("100.4", 38, 0)); + EXPECT_FALSE(lossless("1.001e2", 38, 0)); + EXPECT_FALSE(lossless("1000", 5, 2)); + EXPECT_FALSE(lossless("0.001", 5, 2)); + EXPECT_FALSE(lossless("100abc", 38, 0)); + EXPECT_FALSE(lossless("+.", 38, 0)); + EXPECT_FALSE(lossless("999999999999999999999999999999999999999", 38, 0)); + EXPECT_TRUE(lossless("0e9223372036854775808", 38, 0)); + EXPECT_FALSE(lossless("0e9223372036854775808x", 38, 0)); +} + +TEST(LosslessDecimalCastTest, ParseOnlyExactValues) { + CastParameters params; + Decimal128V3 value; + + EXPECT_TRUE(CastToDecimal::from_string_lossless(StringRef("100.000", 7), value, 38, 0, params)); + EXPECT_EQ(value.value, 100); + + EXPECT_TRUE(CastToDecimal::from_string_lossless(StringRef("1.25e2", 6), value, 38, 0, params)); + EXPECT_EQ(value.value, 125); + + EXPECT_FALSE(CastToDecimal::from_string_lossless(StringRef("100.4", 5), value, 38, 0, params)); + EXPECT_FALSE(CastToDecimal::from_string_lossless(StringRef("100abc", 6), value, 38, 0, params)); + EXPECT_TRUE(CastToDecimal::from_string_lossless(StringRef("0e9223372036854775808", 21), value, + 38, 0, params)); + EXPECT_FALSE(CastToDecimal::from_string_lossless(StringRef("0e9223372036854775808x", 22), value, + 38, 0, params)); +} + +TEST(LosslessDecimalCastTest, HandlesCancellingExponent) { + std::string value = "0."; + value.append(1024, '0'); + value.append("1e1025"); + + EXPECT_TRUE(CastToDecimal::is_lossless_decimal_string(value.data(), value.size(), 38, 0)); + + CastParameters params; + Decimal128V3 decimal; + EXPECT_TRUE(CastToDecimal::from_string_lossless(StringRef(value.data(), value.size()), decimal, + 38, 0, params)); + EXPECT_EQ(decimal.value, 1); +} + +} // namespace doris diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/CastExpr.java index 1aad243b1f2fcb..0b14adbf0d714b 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -33,15 +33,24 @@ public class CastExpr extends Expr { @SerializedName("noOp") protected boolean noOp = false; + // True if this implicit string-to-decimal cast must reject rounding. + @SerializedName("ldc") + protected boolean losslessDecimalCast = false; + // only used restore from readFields. private CastExpr() { } public CastExpr(Type targetType, Expr e, boolean nullable) { + this(targetType, e, nullable, false); + } + + public CastExpr(Type targetType, Expr e, boolean nullable, boolean losslessDecimalCast) { type = targetType; isImplicit = true; children.add(e); + this.losslessDecimalCast = losslessDecimalCast; noOp = Type.matchExactType(e.type, type, true); if (noOp) { @@ -63,6 +72,7 @@ protected CastExpr(CastExpr other) { super(other); isImplicit = other.isImplicit; noOp = other.noOp; + losslessDecimalCast = other.losslessDecimalCast; } @Override @@ -87,14 +97,19 @@ public boolean isNoOp() { return noOp; } + public boolean isLosslessDecimalCast() { + return losslessDecimalCast; + } + @Override public int hashCode() { - return super.hashCode(); + return 31 * super.hashCode() + Boolean.hashCode(losslessDecimalCast); } @Override public boolean equals(Object obj) { - return super.equals(obj); + return super.equals(obj) && obj instanceof CastExpr + && losslessDecimalCast == ((CastExpr) obj).losslessDecimalCast; } public boolean canHashPartition() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToExternalSqlVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToExternalSqlVisitor.java index 572ae0c3508bfa..f351795b24a4dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToExternalSqlVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToExternalSqlVisitor.java @@ -57,6 +57,10 @@ public String visitSlotRef(SlotRef expr, ToSqlParams context) { @Override public String visitCastExpr(CastExpr expr, ToSqlParams context) { + if (expr.isLosslessDecimalCast()) { + throw new UnsupportedOperationException( + "Lossless decimal comparison casts cannot be translated to external SQL"); + } return expr.getChild(0).accept(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java index 3df252e8fed56b..67f2c2cf19c640 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java @@ -433,6 +433,7 @@ public Void visitArithmeticExpr(ArithmeticExpr expr, TExprNode msg) { public Void visitCastExpr(CastExpr expr, TExprNode msg) { msg.node_type = TExprNodeType.CAST_EXPR; msg.setOpcode(TExprOpcode.CAST); + msg.setLosslessDecimalCast(expr.isLosslessDecimalCast()); return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExprToConnectorExpressionConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExprToConnectorExpressionConverter.java index 021782e0e53058..4056c0465474b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExprToConnectorExpressionConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExprToConnectorExpressionConverter.java @@ -106,6 +106,10 @@ public static ConnectorExpression convert(Expr expr) { } else if (expr instanceof LiteralExpr) { return convertLiteral((LiteralExpr) expr); } else if (expr instanceof CastExpr) { + if (((CastExpr) expr).isLosslessDecimalCast()) { + throw new UnsupportedOperationException( + "Lossless decimal comparison casts cannot be pushed to connectors"); + } return convert(expr.getChild(0)); } else if (expr instanceof ArithmeticExpr) { return convertArithmeticExpr((ArithmeticExpr) expr); @@ -146,6 +150,17 @@ public static ConnectorExpression convertConjuncts(List conjuncts) { return new ConnectorAnd(converted); } + public static boolean containsLosslessDecimalCast(Expr expr) { + List castExprs = new ArrayList<>(); + expr.collect(CastExpr.class, castExprs); + return castExprs.stream().anyMatch(CastExpr::isLosslessDecimalCast); + } + + public static boolean canPushDownLimit(List conjuncts) { + return conjuncts.stream().noneMatch( + ExprToConnectorExpressionConverter::containsLosslessDecimalCast); + } + /** * Converts a Doris {@link Type} to a {@link ConnectorType}. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/PluginDrivenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/PluginDrivenScanNode.java index d0875e6f32bf90..2c6254d30a0648 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/PluginDrivenScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/PluginDrivenScanNode.java @@ -285,7 +285,13 @@ protected void convertPredicate() { return; } ConnectorMetadata metadata = connector.getMetadata(connectorSession); - ConnectorFilterConstraint constraint = buildFilterConstraint(conjuncts); + List pushableConjuncts = filterPushableConjuncts( + metadata.supportsCastPredicatePushdown(connectorSession)); + if (pushableConjuncts.isEmpty()) { + filteredToOriginalIndex = null; + return; + } + ConnectorFilterConstraint constraint = buildFilterConstraint(pushableConjuncts); Optional> result = metadata.applyFilter(connectorSession, currentHandle, constraint); if (result.isPresent()) { @@ -297,8 +303,20 @@ protected void convertPredicate() { // - non-null means some/all predicates remain → keep conjuncts (conservative) ConnectorExpression remaining = filterResult.getRemainingFilter(); if (remaining == null) { - conjuncts.clear(); - LOG.debug("Filter fully pushed down for plugin-driven scan, cleared conjuncts"); + if (filteredToOriginalIndex == null) { + conjuncts.clear(); + } else { + Set pushedIndices = new HashSet<>(filteredToOriginalIndex); + List remainingConjuncts = new ArrayList<>(); + for (int i = 0; i < conjuncts.size(); i++) { + if (!pushedIndices.contains(i)) { + remainingConjuncts.add(conjuncts.get(i)); + } + } + conjuncts.clear(); + conjuncts.addAll(remainingConjuncts); + } + LOG.debug("Filter fully pushed down for plugin-driven scan"); } else { // Partial or full remaining: keep all conjuncts for BE-side evaluation. // Fine-grained conjunct removal (matching individual remaining sub-expressions @@ -320,16 +338,16 @@ protected void convertPredicate() { * The limit is still passed to planScan() as a parameter for * connectors that handle limit directly in planScan().

*/ - private void tryPushDownLimit() { - if (limit <= 0) { + private void tryPushDownLimit(long connectorLimit) { + if (connectorLimit <= 0) { return; } ConnectorMetadata metadata = connector.getMetadata(connectorSession); Optional> result = - metadata.applyLimit(connectorSession, currentHandle, limit); + metadata.applyLimit(connectorSession, currentHandle, connectorLimit); if (result.isPresent()) { currentHandle = result.get().getHandle(); - LOG.debug("Limit {} pushed down via applyLimit for plugin-driven scan", limit); + LOG.debug("Limit {} pushed down via applyLimit for plugin-driven scan", connectorLimit); } } @@ -354,8 +372,10 @@ private void tryPushDownProjection(List columns) { @Override public List getSplits(int numBackends) throws UserException { + long connectorLimit = ExprToConnectorExpressionConverter.canPushDownLimit(conjuncts) + ? limit : -1; // Attempt limit and projection pushdown via SPI protocol - tryPushDownLimit(); + tryPushDownLimit(connectorLimit); ConnectorScanPlanProvider scanProvider = connector.getScanPlanProvider(); if (scanProvider == null) { @@ -368,7 +388,7 @@ public List getSplits(int numBackends) throws UserException { Optional remainingFilter = buildRemainingFilter(); List ranges = scanProvider.planScan( - connectorSession, currentHandle, columns, remainingFilter, limit); + connectorSession, currentHandle, columns, remainingFilter, connectorLimit); List splits = new ArrayList<>(ranges.size()); for (ConnectorScanRange range : ranges) { @@ -583,30 +603,34 @@ private Optional buildRemainingFilter() { filteredToOriginalIndex = null; return Optional.empty(); } - List pushableConjuncts = conjuncts; ConnectorMetadata metadata = connector.getMetadata(connectorSession); - if (!metadata.supportsCastPredicatePushdown(connectorSession)) { - filteredToOriginalIndex = new ArrayList<>(); - pushableConjuncts = new ArrayList<>(); - for (int i = 0; i < conjuncts.size(); i++) { - if (!containsCastExpr(conjuncts.get(i))) { - pushableConjuncts.add(conjuncts.get(i)); - filteredToOriginalIndex.add(i); - } - } - // If no filtering occurred, clear the mapping (1:1) - if (filteredToOriginalIndex.size() == conjuncts.size()) { - filteredToOriginalIndex = null; - } - } else { - filteredToOriginalIndex = null; - } + List pushableConjuncts = filterPushableConjuncts( + metadata.supportsCastPredicatePushdown(connectorSession)); if (pushableConjuncts.isEmpty()) { return Optional.empty(); } return Optional.of(ExprToConnectorExpressionConverter.convertConjuncts(pushableConjuncts)); } + private List filterPushableConjuncts(boolean supportsCastPredicatePushdown) { + filteredToOriginalIndex = new ArrayList<>(); + List pushableConjuncts = new ArrayList<>(); + for (int i = 0; i < conjuncts.size(); i++) { + Expr conjunct = conjuncts.get(i); + boolean losslessDecimalCast = + ExprToConnectorExpressionConverter.containsLosslessDecimalCast(conjunct); + if (!losslessDecimalCast + && (supportsCastPredicatePushdown || !containsCastExpr(conjunct))) { + pushableConjuncts.add(conjunct); + filteredToOriginalIndex.add(i); + } + } + if (filteredToOriginalIndex.size() == conjuncts.size()) { + filteredToOriginalIndex = null; + } + return pushableConjuncts; + } + private static boolean containsCastExpr(Expr expr) { List castExprs = new ArrayList<>(); expr.collect(CastExpr.class, castExprs); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/source/RemoteDorisScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/source/RemoteDorisScanNode.java index 644c00028e3c97..dbf178c4a2e750 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/source/RemoteDorisScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/source/RemoteDorisScanNode.java @@ -32,6 +32,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.ExprToConnectorExpressionConverter; import org.apache.doris.datasource.FileQueryScanNode; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; @@ -233,7 +234,7 @@ private String getQueryStr() { sql.append(")"); } - if (limit != -1) { + if (limit != -1 && ExprToConnectorExpressionConverter.canPushDownLimit(conjuncts)) { sql.append(" LIMIT ").append(limit); } @@ -241,6 +242,7 @@ private String getQueryStr() { } private void createFilters() { + filters.clear(); if (conjuncts.isEmpty()) { return; } @@ -257,6 +259,9 @@ private void createFilters() { ArrayList conjunctsList = Expr.cloneList(conjuncts, sMap); for (Expr expr : conjunctsList) { + if (ExprToConnectorExpressionConverter.containsLosslessDecimalCast(expr)) { + continue; + } String filter = conjunctExprToString(expr, desc.getTable()); filters.add(filter); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 84aea35f2c5001..4da339fc284ccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -449,7 +449,7 @@ public Expr visitCaseWhen(CaseWhen caseWhen, PlanTranslatorContext context) { public Expr visitCast(Cast cast, PlanTranslatorContext context) { // left child of cast is expression, right child of cast is target type CastExpr castExpr = new CastExpr(cast.getDataType().toCatalogDataType(), - cast.child().accept(this, context), cast.nullable()); + cast.child().accept(this, context), cast.nullable(), cast.isLosslessDecimalCast()); castExpr.setImplicit(!cast.isExplicitType()); return castExpr; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index 5338bf7df9b2ef..7053e16b9c7ea6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -495,6 +495,9 @@ public Expression visitOr(Or or, ExpressionRewriteContext context) { @Override public Expression visitCast(Cast cast, ExpressionRewriteContext context) { cast = rewriteChildren(cast, context); + if (cast.isLosslessDecimalCast()) { + return cast; + } Optional checkedExpr = preProcess(cast); if (checkedExpr.isPresent()) { return checkedExpr.get(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRule.java index 9baabf9f58437e..9012393d4adafb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRule.java @@ -69,6 +69,10 @@ public List> buildRules() { public static Expression simplifyCast(Cast cast) { Expression child = cast.child(); + if (cast.isLosslessDecimalCast()) { + return cast; + } + // remove redundant cast // CAST(value as type), value is type if (cast.getDataType().equals(child.getDataType())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java index dfcd2ea289cc6f..a71e366072f3d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LogicalResultSinkToShortCircuitPointQuery.java @@ -50,10 +50,16 @@ private Expression removeCast(Expression expression) { return expression; } + private boolean isLosslessDecimalCast(Expression expression) { + return expression instanceof Cast && ((Cast) expression).isLosslessDecimalCast(); + } + private boolean filterMatchShortCircuitCondition(LogicalFilter filter) { return filter.getConjuncts().stream().allMatch( // all conjuncts match with pattern `key = ?` expression -> (expression instanceof EqualTo) + && !isLosslessDecimalCast(expression.child(0)) + && !isLosslessDecimalCast(expression.child(1)) && (removeCast(expression.child(0)).isKeyColumnFromTable() || (expression.child(0) instanceof SlotReference && ((SlotReference) expression.child(0)).getName().equals(Column.DELETE_SIGN))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java index 6a8fd28b902ba7..bf4f5f7e1a5482 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java @@ -618,7 +618,7 @@ private Expression rewriteCast(Cast cast, boolean fillAccessPath) { newType = prunedTree.pruneCastType(originTree, castTree); } - return new Cast(newChild, newType); + return new Cast(newChild, newType, cast.isExplicitType(), cast.isLosslessDecimalCast()); } private List replaceIcebergAccessPathToId( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java index 1e14d3284ea768..af539ad01abfae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java @@ -46,6 +46,9 @@ public class Cast extends Expression implements UnaryExpression, Monotonic { // CAST can be from SQL Query or Type Coercion. true for explicitly cast from SQL query. protected final boolean isExplicitType; //FIXME: now not useful + // Only used by implicit numeric-string equality comparisons. + protected final boolean losslessDecimalCast; + protected final DataType targetType; public Cast(Expression child, DataType targetType) { @@ -53,19 +56,29 @@ public Cast(Expression child, DataType targetType) { } public Cast(Expression child, DataType targetType, boolean isExplicitType) { - this(ImmutableList.of(child), targetType, isExplicitType); + this(child, targetType, isExplicitType, false); + } + + public Cast(Expression child, DataType targetType, boolean isExplicitType, boolean losslessDecimalCast) { + this(ImmutableList.of(child), targetType, isExplicitType, losslessDecimalCast); } - protected Cast(List child, DataType targetType, boolean isExplicitType) { + protected Cast(List child, DataType targetType, boolean isExplicitType, + boolean losslessDecimalCast) { super(child); this.targetType = Objects.requireNonNull(targetType, "targetType can not be null"); this.isExplicitType = isExplicitType; + this.losslessDecimalCast = losslessDecimalCast; } public boolean isExplicitType() { return isExplicitType; } + public boolean isLosslessDecimalCast() { + return losslessDecimalCast; + } + @Override public DataType getDataType() { return targetType; @@ -213,7 +226,7 @@ public static boolean castNullable(boolean srcNullable, DataType srcType, DataTy @Override public Cast withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new Cast(children, targetType, isExplicitType); + return new Cast(children, targetType, isExplicitType, losslessDecimalCast); } @Override @@ -248,12 +261,13 @@ public boolean equals(Object o) { return false; } Cast cast = (Cast) o; - return Objects.equals(targetType, cast.targetType); + return Objects.equals(targetType, cast.targetType) + && losslessDecimalCast == cast.losslessDecimalCast; } @Override public int computeHashCode() { - return Objects.hash(super.computeHashCode(), targetType); + return Objects.hash(super.computeHashCode(), targetType, losslessDecimalCast); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TryCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TryCast.java index 2398d7d49e3e0b..6314e3b8271d35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TryCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TryCast.java @@ -44,7 +44,7 @@ public TryCast(Expression child, DataType targetType, boolean isExplicitType) { } private TryCast(List child, DataType targetType, boolean isExplicitType) { - super(child, targetType, isExplicitType); + super(child, targetType, isExplicitType, false); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java index 6fd99573b805b5..8aad2f56bddfe2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CopyIntoInfo.java @@ -341,9 +341,10 @@ public Expr visitSlotReference(SlotReference slotReference, PlanTranslatorContex @Override public Expr visitCast(Cast cast, PlanTranslatorContext context) { - // left child of cast is target type, right child of cast is expression - return new CastExpr(cast.getDataType().toCatalogDataType(), - cast.child().accept(this, context), cast.nullable()); + CastExpr legacyCast = new CastExpr(cast.getDataType().toCatalogDataType(), + cast.child().accept(this, context), cast.nullable(), cast.isLosslessDecimalCast()); + legacyCast.setImplicit(!cast.isExplicitType()); + return legacyCast; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index c19250e4060f2f..c826aad4d11fd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IntegralDivide; @@ -474,6 +475,42 @@ public static Expression castIfNotSameType(Expression input, DataType targetType } } + private static Expression castComparisonOperand( + Expression input, DataType targetType, boolean losslessDecimalCast) { + if (losslessDecimalCast && input.getDataType().isStringLikeType() + && targetType.isDecimalV3Type()) { + checkCanCastTo(input.getDataType(), targetType); + return new Cast(input, targetType, false, true); + } + return castIfNotSameType(input, targetType); + } + + private static Optional getLosslessDecimalStringComparisonType( + ComparisonPredicate comparisonPredicate, Expression left, Expression right) { + if (!(comparisonPredicate instanceof EqualTo)) { + return Optional.empty(); + } + + DataType decimalType; + if (left.getDataType().isDecimalLikeType() && right.getDataType().isStringLikeType()) { + decimalType = left.getDataType(); + } else if (right.getDataType().isDecimalLikeType() && left.getDataType().isStringLikeType()) { + decimalType = right.getDataType(); + } else { + return Optional.empty(); + } + + DecimalV3Type decimalV3Type = DecimalV3Type.forType(decimalType); + int maxPrecision = SessionVariable.getEnableDecimal256() + ? DecimalV3Type.MAX_DECIMAL256_PRECISION : DecimalV3Type.MAX_DECIMAL128_PRECISION; + int integerPart = Math.max(decimalV3Type.getPrecision() - decimalV3Type.getScale(), 0); + int maxScale = Math.max(maxPrecision - integerPart, 0); + int targetScale = Math.max(decimalV3Type.getScale(), + Math.min(SessionVariable.getDecimalOverFlowScale(), maxScale)); + targetScale = Math.min(targetScale, maxScale); + return Optional.of(DecimalV3Type.createDecimalV3Type(maxPrecision, targetScale)); + } + /** * Wrap {@code expression} in a cast to {@code targetType} when the source type can * already be resolved (Literal, or any expression whose {@code getDataType()} does @@ -1299,22 +1336,25 @@ public static Expression processComparisonPredicate(ComparisonPredicate comparis left = comparisonPredicate.left(); right = comparisonPredicate.right(); - Optional commonType; - if (GlobalVariable.enableNewTypeCoercionBehavior) { - commonType = findWiderTypeForTwo(left.getDataType(), right.getDataType(), false, false); - } else { - commonType = findWiderTypeForTwoForComparison(left.getDataType(), right.getDataType(), false); - } + Optional commonType = getLosslessDecimalStringComparisonType(comparisonPredicate, left, right); + boolean losslessDecimalCast = commonType.isPresent(); + if (!losslessDecimalCast) { + if (GlobalVariable.enableNewTypeCoercionBehavior) { + commonType = findWiderTypeForTwo(left.getDataType(), right.getDataType(), false, false); + } else { + commonType = findWiderTypeForTwoForComparison(left.getDataType(), right.getDataType(), false); + } - if (commonType.isPresent()) { - commonType = Optional.of(downgradeDecimalAndDateLikeType( - commonType.get(), - left, - right)); - commonType = Optional.of(downgradeDecimalAndDateLikeType( - commonType.get(), - right, - left)); + if (commonType.isPresent()) { + commonType = Optional.of(downgradeDecimalAndDateLikeType( + commonType.get(), + left, + right)); + commonType = Optional.of(downgradeDecimalAndDateLikeType( + commonType.get(), + right, + left)); + } } if (commonType.isPresent()) { @@ -1322,8 +1362,8 @@ public static Expression processComparisonPredicate(ComparisonPredicate comparis throw new AnalysisException("data type " + commonType.get() + " could not used in ComparisonPredicate " + comparisonPredicate.toSql()); } - left = castIfNotSameType(left, commonType.get()); - right = castIfNotSameType(right, commonType.get()); + left = castComparisonOperand(left, commonType.get(), losslessDecimalCast); + right = castComparisonOperand(right, commonType.get(), losslessDecimalCast); } else { throw new AnalysisException("unsupported comparison predicate " + comparisonPredicate.toSql()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToSqlTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToSqlTest.java index 24fceb2c901259..b80166c7d3fad4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToSqlTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToSqlTest.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.ExprToConnectorExpressionConverter; import org.apache.doris.nereids.util.MoreFieldsThread; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -597,6 +598,19 @@ public void testCastExprExternal() { Assertions.assertEquals("1", sql); } + @Test + public void testLosslessDecimalCastIsNotPushedDown() { + CastExpr expr = new CastExpr(ScalarType.createDecimalV3Type(38, 0), + new StringLiteral("100.4"), true, true); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> expr.accept(ExprToExternalSqlVisitor.INSTANCE, + new ToSqlParams(false, true, null, null))); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> ExprToConnectorExpressionConverter.convert(expr)); + Assertions.assertFalse(ExprToConnectorExpressionConverter.canPushDownLimit( + Collections.singletonList(expr))); + } + @Test public void testTryCastExprDefault() { TryCastExpr expr = new TryCastExpr(Type.INT, new IntLiteral(1L), false, false); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToThriftBehaviorTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToThriftBehaviorTest.java index ac40a4f174f682..91bb7ab597dd79 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToThriftBehaviorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToThriftBehaviorTest.java @@ -258,6 +258,16 @@ public void testCastExprNoOpTrue() { Assertions.assertEquals(TExprNodeType.INT_LITERAL, nodes.get(0).node_type); } + @Test + public void testLosslessDecimalCastExpr() { + IntLiteral child = new IntLiteral(42); + CastExpr expr = new CastExpr(Type.BIGINT, child, false, true); + + TExprNode node = firstNode(expr); + Assertions.assertTrue(node.isSetLosslessDecimalCast()); + Assertions.assertTrue(node.isLosslessDecimalCast()); + } + // ======================== treesToThrift ======================== @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java index d2ead4e326d667..fc79eb60402d7d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java @@ -23,8 +23,10 @@ import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.Subtract; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; @@ -69,6 +71,7 @@ import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.IntegralType; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; @@ -250,6 +253,69 @@ public void testProcessComparisonPredicateDowngrade() { Assertions.assertEquals(DateTimeType.INSTANCE, datetimeDowngrade.left().getDataType()); } + @Test + public void testLosslessDecimalStringComparisonCoercion() { + boolean originalNewTypeCoercionBehavior = GlobalVariable.enableNewTypeCoercionBehavior; + try { + for (boolean newTypeCoercionBehavior : new boolean[] {true, false}) { + GlobalVariable.enableNewTypeCoercionBehavior = newTypeCoercionBehavior; + DecimalV3Type decimal380 = DecimalV3Type.createDecimalV3Type(38, 0); + EqualTo decimalString = new EqualTo( + new SlotReference("decimal_col", decimal380), + new SlotReference("string_col", StringType.INSTANCE)); + decimalString = (EqualTo) TypeCoercionUtils.processComparisonPredicate(decimalString); + Assertions.assertEquals(decimal380, decimalString.left().getDataType()); + Cast decimalStringCast = (Cast) decimalString.right(); + Assertions.assertEquals(decimal380, decimalStringCast.getDataType()); + Assertions.assertTrue(decimalStringCast.isLosslessDecimalCast()); + + DecimalV3Type decimal54 = DecimalV3Type.createDecimalV3Type(5, 4); + EqualTo decimalChar = new EqualTo( + new SlotReference("decimal_col", decimal54), + new SlotReference("char_col", CharType.createCharType(255))); + decimalChar = (EqualTo) TypeCoercionUtils.processComparisonPredicate(decimalChar); + DecimalV3Type decimal386 = DecimalV3Type.createDecimalV3Type(38, 6); + Assertions.assertEquals(decimal386, decimalChar.left().getDataType()); + Assertions.assertEquals(decimal386, decimalChar.right().getDataType()); + Assertions.assertTrue(((Cast) decimalChar.right()).isLosslessDecimalCast()); + + NullSafeEqual nullSafeDecimalString = new NullSafeEqual( + new SlotReference("decimal_col", decimal380), + new SlotReference("string_col", StringType.INSTANCE)); + nullSafeDecimalString = (NullSafeEqual) TypeCoercionUtils + .processComparisonPredicate(nullSafeDecimalString); + Assertions.assertFalse(((Cast) nullSafeDecimalString.right()).isLosslessDecimalCast()); + + EqualTo bigintString = new EqualTo( + new SlotReference("bigint_col", BigIntType.INSTANCE), + new SlotReference("string_col", StringType.INSTANCE)); + bigintString = (EqualTo) TypeCoercionUtils.processComparisonPredicate(bigintString); + Assertions.assertFalse(((Cast) bigintString.right()).isLosslessDecimalCast()); + + EqualTo floatString = new EqualTo( + new SlotReference("float_col", FloatType.INSTANCE), + new SlotReference("string_col", StringType.INSTANCE)); + floatString = (EqualTo) TypeCoercionUtils.processComparisonPredicate(floatString); + Assertions.assertEquals(DoubleType.INSTANCE, floatString.left().getDataType()); + Assertions.assertFalse(((Cast) floatString.right()).isLosslessDecimalCast()); + + GreaterThan rangeComparison = new GreaterThan( + new SlotReference("decimal_col", decimal380), + new SlotReference("string_col", StringType.INSTANCE)); + rangeComparison = (GreaterThan) TypeCoercionUtils.processComparisonPredicate(rangeComparison); + Assertions.assertFalse(((Cast) rangeComparison.right()).isLosslessDecimalCast()); + + InPredicate inPredicate = new InPredicate( + new SlotReference("decimal_col", decimal380), + ImmutableList.of(new SlotReference("string_col", StringType.INSTANCE))); + inPredicate = (InPredicate) TypeCoercionUtils.processInPredicate(inPredicate); + Assertions.assertFalse(((Cast) inPredicate.getOptions().get(0)).isLosslessDecimalCast()); + } + } finally { + GlobalVariable.enableNewTypeCoercionBehavior = originalNewTypeCoercionBehavior; + } + } + @Test public void testProcessInStringCoercion() { // BigInt slot vs String literal diff --git a/gensrc/thrift/Exprs.thrift b/gensrc/thrift/Exprs.thrift index 812a65740a77c2..653c1b56fffc83 100644 --- a/gensrc/thrift/Exprs.thrift +++ b/gensrc/thrift/Exprs.thrift @@ -333,6 +333,8 @@ struct TExprNode { // distinguish current-scope lambda arguments from captured outer lambda // arguments when nested lambda expressions contain duplicated column ids. 42: optional list lambda_argument_names + // Use lossless parsing for implicit string-to-decimal equality comparisons. + 43: optional bool lossless_decimal_cast } // A flattened representation of a tree of Expr nodes, obtained by depth-first diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out index e352e38da3ae18..4eb65b96a23de7 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out @@ -1137,7 +1137,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 1.000000)) +--------filter((cast(t2.d_char as DECIMALV3(38, 12)) = 1.000000000003)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type124 -- @@ -1149,7 +1149,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 1.000000)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 12)) = 1.000000000003)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type125 -- @@ -1161,7 +1161,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 1.000000)) +--------filter((cast(t2.d_string as DECIMALV3(38, 12)) = 1.000000000003)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type128 -- @@ -1257,7 +1257,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 12232.239827)) +--------filter((cast(t2.d_char as DECIMALV3(38, 16)) = 12232.2398272342335234)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type140 -- @@ -1269,7 +1269,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 12232.239827)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 16)) = 12232.2398272342335234)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type141 -- @@ -1281,7 +1281,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 12232.239827)) +--------filter((cast(t2.d_string as DECIMALV3(38, 16)) = 12232.2398272342335234)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type144 -- diff --git a/regression-test/suites/correctness_p0/test_join_decimal_string_lossless_cast.groovy b/regression-test/suites/correctness_p0/test_join_decimal_string_lossless_cast.groovy new file mode 100644 index 00000000000000..58485302d42bbd --- /dev/null +++ b/regression-test/suites/correctness_p0/test_join_decimal_string_lossless_cast.groovy @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_join_decimal_string_lossless_cast") { + def originStrictCast = sql("select @@enable_strict_cast") + + sql """drop table if exists test_join_decimal_string_numeric""" + sql """drop table if exists test_join_decimal_string_value""" + + sql """ + create table test_join_decimal_string_numeric ( + id int, + k decimal(38, 0) + ) engine=olap + duplicate key(id) + distributed by hash(id) buckets 1 + properties ("replication_allocation" = "tag.location.default: 1") + """ + sql """ + create table test_join_decimal_string_value ( + k varchar(64) + ) engine=olap + duplicate key(k) + distributed by hash(k) buckets 1 + properties ("replication_allocation" = "tag.location.default: 1") + """ + + sql """ + insert into test_join_decimal_string_numeric values + (1, 100), + (2, 9007199254740992), + (3, 9007199254740993), + (4, 99999999999999999999999999999999999999), + (5, 0) + """ + sql """ + insert into test_join_decimal_string_value values + ('100'), + ('100.000'), + ('100.4'), + ('100.5'), + ('100abc'), + ('9007199254740992'), + ('9007199254740993'), + ('99999999999999999999999999999999999999'), + ('999999999999999999999999999999999999999'), + ('0e9223372036854775808'), + ('0e9223372036854775808x') + """ + + sql """set enable_strict_cast = false""" + + test { + sql """ + select count(*) + from test_join_decimal_string_numeric n + join test_join_decimal_string_value s on n.k = s.k + where n.id = 1 + """ + result([[2L]]) + } + + test { + sql """ + select count(*) + from test_join_decimal_string_numeric n + join test_join_decimal_string_value s on n.k = s.k + """ + result([[6L]]) + } + + sql """set enable_strict_cast = true""" + + test { + sql """ + select count(*) + from test_join_decimal_string_numeric n + join test_join_decimal_string_value s on n.k = s.k + where n.id = 1 and s.k = '100abc' + """ + exception "parse number fail" + } + + test { + sql """ + select count(*) + from test_join_decimal_string_numeric n + join test_join_decimal_string_value s on n.k = s.k + where n.id = 4 and s.k = '999999999999999999999999999999999999999' + """ + exception "parse number fail" + } + + sql """set enable_strict_cast = ${originStrictCast[0][0]}""" + sql """drop table if exists test_join_decimal_string_numeric""" + sql """drop table if exists test_join_decimal_string_value""" +}