diff --git a/be/src/core/column/column_string.h b/be/src/core/column/column_string.h index 9ca55caa9c6aa5..616d6ef9df39e1 100644 --- a/be/src/core/column/column_string.h +++ b/be/src/core/column/column_string.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -276,6 +277,56 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + template + void insert_many_parquet_plain_byte_arrays(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num, + const std::vector& value_spans) { + if (UNLIKELY(num == 0)) { + return; + } + const size_t old_chars_size = chars.size(); + const size_t bytes = value_offsets[num]; + check_chars_length(old_chars_size + bytes, offsets.size() + num); + chars.resize(old_chars_size + bytes); + + // Source payloads may be contiguous for decoder-owned buffers even though PLAIN normally + // separates them with length prefixes. Coalesce whenever the published layout permits it; + // the fallback remains one direct source-to-column copy without a StringRef staging array. + size_t covered_values = 0; + for (const auto& span : value_spans) { + DORIS_CHECK_EQ(span.first, covered_values); + DORIS_CHECK_LE(span.first + span.count, num); + size_t run_first = span.first; + const size_t span_end = span.first + span.count; + while (run_first < span_end) { + size_t run_end = run_first + 1; + while (run_end < span_end && + payload_offsets[run_end] == payload_offsets[run_first] + + value_offsets[run_end] - + value_offsets[run_first]) { + ++run_end; + } + const size_t run_bytes = value_offsets[run_end] - value_offsets[run_first]; + if (run_bytes != 0) { + memcpy(chars.data() + old_chars_size + value_offsets[run_first], + encoded_data + payload_offsets[run_first], run_bytes); + } + run_first = run_end; + } + covered_values = span_end; + } + DORIS_CHECK_EQ(covered_values, num); + + const size_t old_rows = offsets.size(); + const auto tail_offset = offsets.back(); + offsets.resize(old_rows + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_rows + row] = tail_offset + value_offsets[row + 1]; + } + sanity_check_simple(); + } + // Insert `num` string entries with real length information but no actual // character data. The `lengths` array provides the byte length of each // string. Offsets are built with correct cumulative sizes so that @@ -322,6 +373,35 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + void insert_many_fixed_length_data(const char* data, size_t value_length, size_t num) { + if (num == 0) { + return; + } + if (value_length > std::numeric_limits::max() / num) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + const size_t old_chars_size = chars.size(); + const size_t old_offsets_size = offsets.size(); + const size_t bytes = value_length * num; + if (bytes > std::numeric_limits::max() - old_chars_size || + num > std::numeric_limits::max() - old_offsets_size) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + check_chars_length(old_chars_size + bytes, old_offsets_size + num); + chars.resize(old_chars_size + bytes); + if (bytes != 0) { + memcpy(chars.data() + old_chars_size, data, bytes); + } + offsets.resize(old_offsets_size + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_offsets_size + row] = + static_cast(old_chars_size + (row + 1) * value_length); + } + sanity_check_simple(); + } + template void insert_many_strings_fixed_length(const StringRef* strings, size_t num) { size_t new_size = 0; diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 44402bb5b531cd..65153c2d8d0b49 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -31,10 +31,13 @@ #include "core/data_type/primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/data_type_serde/parquet_timestamp.h" #include "core/types.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" #include "exprs/function/cast/cast_to_string.h" +#include "util/unaligned.h" enum { DIVISOR_FOR_SECOND = 1, @@ -48,22 +51,6 @@ static const int64_t micro_to_nano_second = 1000; namespace { -#pragma pack(1) -struct DecodedInt96Timestamp { - int64_t nanos_of_day; - int32_t julian_day; - - int64_t to_timestamp_micros() const { - static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; - static constexpr int64_t MICROS_IN_DAY = 86400000000; - static constexpr int64_t NANOS_PER_MICROSECOND = 1000; - return (julian_day - JULIAN_EPOCH_OFFSET_DAYS) * MICROS_IN_DAY + - nanos_of_day / NANOS_PER_MICROSECOND; - } -}; -#pragma pack() -static_assert(sizeof(DecodedInt96Timestamp) == 12); - Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, int64_t timestamp_micros) { static constexpr int64_t MICROS_PER_SECOND = 1000000; @@ -105,9 +92,9 @@ Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, return Status::OK(); } -void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, - int64_t timestamp_micros, - const cctz::time_zone& timezone) { +Status append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, + int64_t timestamp_micros, + const cctz::time_zone& timezone) { static constexpr int64_t MICROS_PER_SECOND = 1000000; int64_t epoch_seconds = timestamp_micros / MICROS_PER_SECOND; @@ -120,19 +107,91 @@ void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, DateV2Value datetime_value; datetime_value.from_unixtime(epoch_seconds, timezone); datetime_value.set_microsecond(static_cast(micros_of_second)); + if (!datetime_value.is_valid_date()) { + return Status::DataQualityError( + "Decoded DATETIMEV2 timestamp is outside the target timezone range: micros={}", + timestamp_micros); + } data.push_back(datetime_value); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; + return ParquetTimeUnit::NANOS; } - return value; + return ParquetTimeUnit::MICROS; } +class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + DateTimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + static const auto utc_timezone = cctz::utc_time_zone(); + const auto& timezone = _context.timezone == nullptr ? utc_timezone : *_context.timezone; + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + Status status; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + status = _context.physical_type == ParquetPhysicalType::INT96 || + _context.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(_data, timestamp_micros); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectDateTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATETIMEV2"); + } +}; + } // namespace // NOLINTBEGIN(readability-function-size) @@ -559,7 +618,7 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( auto& data = assert_cast(column).get_data(); const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); static const auto utc_timezone = cctz::utc_time_zone(); const auto& timezone = view.timezone == nullptr ? utc_timezone : *view.timezone; for (int64_t row = 0; row < view.row_count; ++row) { @@ -567,8 +626,19 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - append_datetimev2_from_utc_epoch_micros(data, values[row].to_timestamp_micros(), - timezone); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -581,24 +651,63 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - const int64_t timestamp_micros = decoded_timestamp_micros(view, values[row]); - if (view.timestamp_is_adjusted_to_utc) { - append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); - } else { - auto st = append_datetimev2_from_epoch_micros(data, timestamp_micros); - if (!st.ok()) { - if (decoded_column_view_can_null_on_conversion_failure(view)) { - decoded_column_view_insert_null_on_conversion_failure(column, view, row); - continue; - } - data.resize(old_size); - return st; + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = view.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; } + data.resize(old_size); + return status; } } return Status::OK(); } +Status DataTypeDateTimeV2SerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateTimeV2ParquetConsumer consumer(column, context); + RejectDateTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateTimeV2SerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("DATETIMEV2 expects Parquet INT64 or INT96"); + } + DateTimeV2ParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateTimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectDateTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + Status DataTypeDateTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.h b/be/src/core/data_type_serde/data_type_datetimev2_serde.h index e089d789ccee7d..c3411ed4249843 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.h @@ -90,6 +90,11 @@ class DataTypeDateTimeV2SerDe : public DataTypeNumberSerDe(column).get_data()), _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + DateV2Value value; + const int64_t day_number = + static_cast(unaligned_load(values + row * sizeof(int32_t))) + + date_threshold; + if (day_number < 0 || !value.get_date_from_daynr(static_cast(day_number))) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = DateV2Value(); + continue; + } + _data.resize(old_size); + return Status::DataQualityError("Parquet DATE value is out of range"); + } + _data[old_size + row] = value; + } + return Status::OK(); + } + +private: + ColumnDateV2::Container& _data; + ParquetMaterializationState* _state; +}; + +class RejectDateV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATEV2"); + } +}; + +} // namespace Status DataTypeDateV2SerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, BufferWritable& bw, @@ -153,6 +197,40 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::OK(); } +Status DataTypeDateV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateV2ParquetConsumer consumer(column); + RejectDateV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT32 || + context.logical_type != ParquetLogicalType::DATE) { + return Status::NotSupported("DATEV2 expects Parquet DATE stored as INT32"); + } + DateV2ParquetConsumer consumer(column, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, &state); + RejectDateV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + Status DataTypeDateV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.h b/be/src/core/data_type_serde/data_type_datev2_serde.h index bc02b61b520193..39d44efcfa0652 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datev2_serde.h @@ -88,6 +88,11 @@ class DataTypeDateV2SerDe : public DataTypeNumberSerDe #include +#include #include #include @@ -36,6 +37,7 @@ #include "core/data_type/storage_field_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/types.h" #include "exec/common/arithmetic_overflow.h" #include "exprs/function/cast/cast_to_decimal.h" @@ -75,8 +77,13 @@ NativeType decode_big_endian_signed_integer(const uint8_t* data, int length) { template bool decoded_decimal_value_fits(const typename PrimitiveTypeTraits::CppType::NativeType& value, UInt32 precision) { - return value >= min_decimal_value(precision).value && - value <= max_decimal_value(precision).value; + if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; + } else { + return value >= min_decimal_value(precision).value && + value <= max_decimal_value(precision).value; + } } template @@ -85,6 +92,9 @@ bool decoded_decimal_int_value_fits(Int128 value, UInt32 precision) { if constexpr (std::is_same_v) { const auto wide_value = wide::Int256(value); return decoded_decimal_value_fits(wide_value, precision); + } else if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; } else { return value >= static_cast(min_decimal_value(precision).value) && value <= static_cast(max_decimal_value(precision).value); @@ -179,6 +189,214 @@ Status read_decimal_decoded_values(IColumn& column, const DecodedColumnView& vie return Status::OK(); } +template +wide::Int256 parquet_decimal_limit(UInt32 precision) { + if constexpr (T == TYPE_DECIMALV2) { + return wide::Int256(DataTypeDecimal::get_max_digits_number(precision)); + } else { + return wide::Int256(max_decimal_value(precision).value); + } +} + +template +Status scale_parquet_decimal(wide::Int256 value, int32_t source_scale, int32_t target_scale, + UInt32 target_precision, wide::Int256* result) { + DORIS_CHECK(result != nullptr); + const auto limit = parquet_decimal_limit(target_precision); + if (source_scale > target_scale) { + const int64_t scale_delta = static_cast(source_scale) - target_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // The precision bound above guarantees that narrowing the positive scale delta keeps + // the multiplier lookup in range. + const auto divisor = + decimal_scale_multiplier(static_cast(scale_delta)); + // Scale-down must be exact. Truncating before the target precision check silently + // changes source values and makes plain and dictionary decoding disagree with casts. + if (value % divisor != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + value /= divisor; + } + } else if (source_scale < target_scale) { + const int64_t scale_delta = static_cast(target_scale) - source_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // Keep the same checked narrowing invariant for scale-up as for exact scale-down. + const auto multiplier = + decimal_scale_multiplier(static_cast(scale_delta)); + if (value > limit / multiplier || value < -limit / multiplier) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + value *= multiplier; + } + } + if (value < -limit || value > limit) { + return Status::DataQualityError("Parquet decimal value is out of range"); + } + *result = value; + return Status::OK(); +} + +template +class DecimalParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + using FieldType = typename PrimitiveTypeTraits::CppType; + using NativeType = typename FieldType::NativeType; + + DecimalParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + UInt32 target_precision, int32_t target_scale, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast&>(column).get_data()), + _context(context), + _target_precision(target_precision), + _target_scale(target_scale), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_integers(values, num_values); + } + if (_context.physical_type == ParquetPhysicalType::INT64) { + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_integers(values, num_values); + } + if (_context.physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return Status::NotSupported("Unsupported Parquet physical type {} for decimal SerDe", + static_cast(_context.physical_type)); + } + DORIS_CHECK_EQ(value_width, static_cast(_context.type_length)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = + append_binary_value(values + row * value_width, value_width, old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = append_binary_value(reinterpret_cast(values[row].data), + values[row].size, old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const size_t length = value_offsets[row + 1] - value_offsets[row]; + auto status = append_binary_value( + reinterpret_cast(encoded_data + payload_offsets[row]), length, + old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + template + Status append_integers(const uint8_t* values, size_t num_values) { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + constexpr int32_t SOURCE_DIGITS = std::numeric_limits::digits10 + 1; + if (_context.decimal_scale == _target_scale && + SOURCE_DIGITS <= static_cast(_target_precision)) { + // The complete physical domain fits the target at the same scale, so narrowing cannot + // precede a failure check and the hot same-scale path needs no wide arithmetic. + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = + unaligned_load(values + row * sizeof(SourceType)); + _data[old_size + row] = FieldType {NativeType(source_value)}; + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = unaligned_load(values + row * sizeof(SourceType)); + auto status = append_wide_value(wide::Int256(source_value), old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status append_binary_value(const uint8_t* value, size_t length, size_t output_row) { + if (UNLIKELY(length > sizeof(wide::Int256))) { + return Status::DataQualityError("Parquet decimal binary value is too wide: {}", length); + } + return append_wide_value( + decode_big_endian_signed_integer(value, cast_set(length)), + output_row); + } + + Status append_wide_value(wide::Int256 value, size_t output_row) { + wide::Int256 scaled_value; + RETURN_IF_ERROR(scale_parquet_decimal(value, _context.decimal_scale, _target_scale, + _target_precision, &scaled_value)); + // Narrow only after scaling and target-precision validation. In particular, an INT64 or a + // sign-extended binary value must never wrap through Decimal32 before it can be rejected. + _data[output_row] = FieldType {static_cast(scaled_value)}; + return Status::OK(); + } + + typename ColumnDecimal::Container& _data; + const ParquetDecodeContext& _context; + UInt32 _target_precision; + int32_t _target_scale; + ParquetMaterializationState* _state; +}; + } // namespace template @@ -529,6 +747,49 @@ Status DataTypeDecimalSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeDecimalSerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeDecimalSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DecimalParquetConsumer dictionary_consumer(*state.typed_dictionary, context, + cast_set(precision), scale, &state); + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, dictionary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeDecimalSerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index 547488c9da4bc5..2a399ce83cc1a4 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -109,6 +109,11 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { int64_t end, const cctz::time_zone& ctz) const override; Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 6d589637f8a309..44f2cb2cbb1213 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -19,6 +19,8 @@ #include +#include +#include #include #include #include @@ -33,6 +35,7 @@ #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/packed_int128.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -51,6 +54,23 @@ namespace doris { namespace { +float parquet_half_to_float(uint16_t half) { + const uint32_t sign = (half & 0x8000U) << 16; + const uint32_t exponent = (half & 0x7C00U) >> 10; + const uint32_t mantissa = half & 0x03FFU; + if (exponent == 0) { + if (mantissa == 0) { + return std::bit_cast(sign); + } + const float value = std::ldexp(static_cast(mantissa), -24); + return sign == 0 ? value : -value; + } + if (exponent == 0x1FU) { + return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); + } + return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); +} + template const NativeType* decoded_values_as(const DecodedColumnView& view) { return reinterpret_cast(view.values); @@ -78,6 +98,22 @@ bool decoded_number_value_fits(SourceType value) { } } +template +constexpr bool parquet_number_conversion_always_fits() { + if constexpr (std::is_floating_point_v) { + return true; + } else if constexpr (!std::is_integral_v || !std::is_integral_v || + std::is_same_v) { + return false; + } else if constexpr (std::is_signed_v == std::is_signed_v) { + return sizeof(DorisCppType) >= sizeof(SourceType); + } else if constexpr (std::is_signed_v) { + return sizeof(DorisCppType) > sizeof(SourceType); + } else { + return false; + } +} + template Status read_number_decoded_values(IColumn& column, const DecodedColumnView& view) { if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { @@ -179,6 +215,212 @@ Status read_integer_decoded_values(IColumn& column, const DecodedColumnView& vie } } +template +Status append_parquet_number(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + if constexpr (std::is_same_v) { + // Identical fixed-width physical/logical types need no validation or conversion. Parquet + // PLAIN values and Doris POD columns share the byte representation on supported targets, + // so preserve one dense vector-at-a-time memcpy instead of converting value by value. + memcpy(data.data() + old_size, values, num_values * sizeof(SourceType)); + return Status::OK(); + } + if constexpr (parquet_number_conversion_always_fits()) { + // A widening conversion cannot fail, so keeping range checks in the row loop only blocks + // auto-vectorization. Input can be unaligned at a Parquet page boundary; load explicitly. + for (size_t row = 0; row < num_values; ++row) { + data[old_size + row] = static_cast( + unaligned_load(values + row * sizeof(SourceType))); + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load(values + row * sizeof(SourceType)); + if (!decoded_number_value_fits(value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError("Parquet value is out of range at row {}", row); + } + data[old_size + row] = static_cast(value); + } + return Status::OK(); +} + +template +Status append_parquet_logical_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values, ParquetMaterializationState* state) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + if constexpr (parquet_number_conversion_always_fits()) { + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = + unaligned_load(values + row * sizeof(SourceType)); + data[old_size + row] = + static_cast(static_cast(physical_value)); + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = unaligned_load(values + row * sizeof(SourceType)); + const auto logical_value = static_cast(physical_value); + if (!decoded_number_value_fits(logical_value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError("Parquet logical integer is out of range at row {}", + row); + } + data[old_size + row] = static_cast(logical_value); + } + return Status::OK(); +} + +template +Status append_parquet_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { + if (context.logical_integer_bit_width <= 0) { + return append_parquet_number(data, values, num_values, context, + state); + } + if (context.logical_integer_is_signed) { + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers( + data, values, num_values, state); + case 16: + return append_parquet_logical_integers( + data, values, num_values, state); + case 32: + return append_parquet_logical_integers( + data, values, num_values, state); + case 64: + return append_parquet_logical_integers( + data, values, num_values, state); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } + } + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers(data, values, + num_values, state); + case 16: + return append_parquet_logical_integers(data, values, + num_values, state); + case 32: + return append_parquet_logical_integers(data, values, + num_values, state); + case 64: + return append_parquet_logical_integers(data, values, + num_values, state); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } +} + +template +class NumberParquetConsumer final : public ParquetFixedValueConsumer { +public: + using DorisCppType = typename PrimitiveTypeTraits::CppType; + using ColumnType = typename PrimitiveTypeTraits::ColumnType; + + NumberParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return consume_impl(values, num_values, value_width); + } + + Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) override { + // Each selected PLAIN range is already contiguous in the encoded page. Append those spans + // directly so sparse reads never build an intermediate selected-values array. + for (const auto& range : ranges) { + RETURN_IF_ERROR( + consume_impl(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } + +private: + Status consume_impl(const uint8_t* values, size_t num_values, size_t value_width) { + if (_context.logical_float16) { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY); + DORIS_CHECK_EQ(value_width, sizeof(uint16_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const float value = parquet_half_to_float( + unaligned_load(values + row * sizeof(uint16_t))); + if (!decoded_number_value_fits(value)) { + if (_state != nullptr && _state->can_insert_null_on_conversion_failure()) { + _data[old_size + row] = DorisCppType(); + DORIS_CHECK(_state->mark_conversion_failure(old_size + row)); + continue; + } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet FLOAT16 value is out of range at row {}", row); + } + _data[old_size + row] = static_cast(value); + } + return Status::OK(); + } + switch (_context.physical_type) { + case ParquetPhysicalType::BOOLEAN: + DORIS_CHECK_EQ(value_width, sizeof(uint8_t)); + return append_parquet_number(_data, values, num_values, _context, + _state); + case ParquetPhysicalType::INT32: + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_parquet_integers(_data, values, num_values, + _context, _state); + case ParquetPhysicalType::INT64: + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_parquet_integers(_data, values, num_values, + _context, _state); + case ParquetPhysicalType::FLOAT: + DORIS_CHECK_EQ(value_width, sizeof(float)); + return append_parquet_number(_data, values, num_values, _context, + _state); + case ParquetPhysicalType::DOUBLE: + DORIS_CHECK_EQ(value_width, sizeof(double)); + return append_parquet_number(_data, values, num_values, _context, + _state); + default: + return Status::NotSupported("Unsupported Parquet physical type {} for numeric SerDe", + static_cast(_context.physical_type)); + } + } + + PaddedPODArray& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectParquetBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as a number"); + } +}; + } // namespace // Basic structure of the type map. template @@ -330,6 +572,52 @@ Status DataTypeNumberSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeNumberSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_parquet_dictionary(column, source, context); + } else { + NumberParquetConsumer consumer(column, context); + RejectParquetBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); + } +} + +template +Status DataTypeNumberSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_column_from_parquet(column, source, context, num_values, state); + } else { + NumberParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + NumberParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectParquetBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); + } +} + template Status DataTypeNumberSerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const { diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 4ced14ed858f94..49196d3956d01e 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -119,6 +119,11 @@ class DataTypeNumberSerDe : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index 6c41b4e9f141db..c84203b6d7f47b 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -52,6 +52,7 @@ #include "core/data_type_serde/data_type_number_serde.h" #include "core/data_type_serde/data_type_string_serde.h" #include "core/data_type_serde/data_type_timestamptz_serde.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/field.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -710,6 +711,18 @@ Status DataTypeSerDe::read_column_from_decoded_values(IColumn& column, get_name())); } +Status DataTypeSerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + return Status::NotSupported("read_column_from_parquet is not supported for {}", get_name()); +} + +Status DataTypeSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + return Status::NotSupported("read_parquet_dictionary is not supported for {}", get_name()); +} + Status DataTypeSerDe::read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const { return Status::NotSupported("read_column_from_orc is not supported for {}", get_name()); diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index edeb0b0de28781..49bfb73cbd854c 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -93,6 +93,9 @@ struct CastParameters; class DataTypeSerDe; using DataTypeSerDeSPtr = std::shared_ptr; using DataTypeSerDeSPtrs = std::vector; +class ParquetDecodeSource; +struct ParquetDecodeContext; +struct ParquetMaterializationState; /// Info that represents a scalar or array field in a decomposed view. /// It allows to recreate field with different number @@ -505,6 +508,18 @@ class DataTypeSerDe { // the Doris-type-specific materialization into IColumn. virtual Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const; + + // Read encoded Parquet values directly into the destination Doris column. ColumnReader owns + // levels/null/filter handling; the source owns only encoding-stream state; the target SerDe + // owns all physical/logical type interpretation and materialization. + virtual Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const; + // Decode one dictionary page into the selected Doris type without consuming data-page + // indices. Dictionary filters use this to keep type interpretation in SerDe instead of + // exposing dictionary bytes or decoder-owned strings to ColumnReader. + virtual Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const; virtual Status read_field_from_decoded_value(const IDataType& data_type, Field* field, const DecodedColumnView& view) const; diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index bd1c6e5b4f19e2..24f7215d42e044 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -17,14 +17,18 @@ #include "core/data_type_serde/data_type_string_serde.h" +#include #include #include +#include #include "common/config.h" #include "core/column/column_string.h" +#include "core/column/column_vector.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "util/jsonb_document_cast.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" @@ -57,6 +61,65 @@ Status read_string_decoded_values(IColumn& column, const DecodedColumnView& view return Status::OK(); } +template +class StringParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + explicit StringParquetConsumer(IColumn& column) : _column(assert_cast(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if constexpr (requires(ColumnType& column) { + column.insert_many_fixed_length_data(static_cast(nullptr), + size_t(), size_t()); + }) { + // FIXED_LEN_BYTE_ARRAY is already a dense byte span. Copy it once and synthesize + // offsets; StringRef batches add a second row loop and hundreds of tiny memcpy calls. + _column.insert_many_fixed_length_data(reinterpret_cast(values), + value_width, num_values); + } else { + static constexpr size_t BATCH_SIZE = 256; + std::array refs; + size_t offset = 0; + while (offset < num_values) { + const size_t batch_size = std::min(BATCH_SIZE, num_values - offset); + for (size_t row = 0; row < batch_size; ++row) { + refs[row] = StringRef( + reinterpret_cast(values + (offset + row) * value_width), + value_width); + } + _column.insert_many_strings(refs.data(), batch_size); + offset += batch_size; + } + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + _column.insert_many_strings(values, num_values); + return Status::OK(); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + if constexpr (requires(ColumnType& column) { + column.insert_many_parquet_plain_byte_arrays( + encoded_data, payload_offsets, value_offsets, num_values, + value_spans); + }) { + _column.insert_many_parquet_plain_byte_arrays(encoded_data, payload_offsets, + value_offsets, num_values, value_spans); + return Status::OK(); + } + return ParquetBinaryValueConsumer::consume_plain_byte_array( + encoded_data, payload_offsets, value_offsets, num_values, value_spans); + } + +private: + ColumnType& _column; +}; + } // namespace namespace { @@ -496,6 +559,57 @@ Status DataTypeStringSerDeBase::read_column_from_decoded_values( return read_string_decoded_values(column, view); } +template +Status DataTypeStringSerDeBase::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + StringParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeStringSerDeBase::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.dictionary_index_only) { + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return Status::IOError("Dictionary filter requested for a non-dictionary page"); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + auto& indices = assert_cast(column).get_data(); + const size_t old_size = indices.size(); + indices.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(state.dictionary_indices[row] > + static_cast(std::numeric_limits::max()))) { + indices.resize(old_size); + return Status::Corruption("Parquet dictionary index {} exceeds INT32", + state.dictionary_indices[row]); + } + indices[old_size + row] = static_cast(state.dictionary_indices[row]); + } + return Status::OK(); + } + StringParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type {} for string SerDe", + static_cast(context.physical_type)); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeStringSerDeBase::write_column_to_orc( const std::string& timezone, const IColumn& column, const NullMap* null_map, diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index ad9a05dc996358..c81f6d37aa4ffc 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -205,6 +205,11 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index c40e671793c848..52d387eda5ca6d 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -17,13 +17,17 @@ #include "core/data_type_serde/data_type_time_serde.h" +#include + #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/value/time_value.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_time_impl.hpp" +#include "util/unaligned.h" namespace doris { namespace { @@ -51,6 +55,74 @@ TimeValue::TimeType read_time_decoded_value(const DecodedColumnView& view, int64 abs_micros % TimeValue::ONE_SECOND_MICROSECONDS, negative); } +class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + int64_t raw_value; + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + raw_value = unaligned_load(values + row * sizeof(int32_t)); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + raw_value = unaligned_load(values + row * sizeof(int64_t)); + } + + int64_t units_per_day; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + units_per_day = 86400000; + } else if (_context.time_unit == ParquetTimeUnit::MICROS) { + units_per_day = 86400000000; + } else { + DORIS_CHECK(_context.time_unit == ParquetTimeUnit::NANOS); + units_per_day = 86400000000000; + } + // Validate the declared carrier before rescaling: truncating nanoseconds first could + // turn a value at or beyond 24:00:00 into an apparently valid TIMEV2 value. + if (raw_value < 0 || raw_value >= units_per_day) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = TimeValue::TimeType(); + continue; + } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet TIME value {} is outside the one-day domain", raw_value); + } + int64_t micros = raw_value; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + micros *= 1000; + } else if (_context.time_unit == ParquetTimeUnit::NANOS) { + micros /= 1000; + } + // Doris TIMEV2 stores signed microseconds in a double. Splitting into calendar fields + // and immediately recombining them is an identity operation with several divisions. + _data[old_size + row] = static_cast(micros); + } + return Status::OK(); + } + +private: + ColumnTimeV2::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMEV2"); + } +}; + } // namespace Status DataTypeTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, @@ -193,6 +265,41 @@ Status DataTypeTimeV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::OK(); } +Status DataTypeTimeV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + TimeV2ParquetConsumer consumer(column, context); + RejectTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if ((context.physical_type != ParquetPhysicalType::INT32 && + context.physical_type != ParquetPhysicalType::INT64) || + context.logical_type != ParquetLogicalType::TIME) { + return Status::NotSupported("TIMEV2 expects Parquet TIME stored as INT32 or INT64"); + } + TimeV2ParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeTimeV2SerDe::from_int_batch(const typename IntDataType::ColumnType& int_col, ColumnNullable& target_col) const { diff --git a/be/src/core/data_type_serde/data_type_time_serde.h b/be/src/core/data_type_serde/data_type_time_serde.h index e3fccf379c913a..20fa14a148854a 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.h +++ b/be/src/core/data_type_serde/data_type_time_serde.h @@ -69,6 +69,11 @@ class DataTypeTimeV2SerDe : public DataTypeNumberSerDe(micros_of_second)); + if (!timestamp_tz.is_valid_date()) { + return Status::DataQualityError( + "Decoded TIMESTAMPTZ is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } data.push_back(timestamp_tz); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; + return ParquetTimeUnit::NANOS; } - return value; + return ParquetTimeUnit::MICROS; } +class TimestampTzParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimestampTzParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + Status status; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + } + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(_data, timestamp_micros); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnTimeStampTz::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectTimestampTzBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMESTAMPTZ"); + } +}; + } // namespace // The implementation of these functions mainly refers to data_type_datetimev2_serde.cpp @@ -308,14 +355,27 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( } auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); for (int64_t row = 0; row < view.row_count; ++row) { if (decoded_column_view_row_is_null(view, row)) { data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, values[row].to_timestamp_micros()); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -326,11 +386,57 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, decoded_timestamp_micros(view, values[row])); + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } +Status DataTypeTimeStampTzSerDe::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + TimestampTzParquetConsumer consumer(column, context); + RejectTimestampTzBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeStampTzSerDe::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("TIMESTAMPTZ expects Parquet INT64 or INT96"); + } + TimestampTzParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimestampTzParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimestampTzBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + std::string DataTypeTimeStampTzSerDe::to_olap_string(const Field& field) const { return CastToString::from_timestamptz(field.get(), 6); } diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.h b/be/src/core/data_type_serde/data_type_timestamptz_serde.h index 23d57f57fc8dac..6777459909a090 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.h +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.h @@ -75,6 +75,11 @@ class DataTypeTimeStampTzSerDe : public DataTypeNumberSerDe(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(reinterpret_cast(values + row * value_width), + value_width); + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(values[row].data, values[row].size); + } + return Status::OK(); + } + + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return Status::OK(); + } + +private: + ColumnVarbinary& _column; +}; + +} // namespace + +Status DataTypeVarbinarySerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + VarbinaryParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +Status DataTypeVarbinarySerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + VarbinaryParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type for VARBINARY"); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} void DataTypeVarbinarySerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena& arena, int32_t col_id, int64_t row_num, diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.h b/be/src/core/data_type_serde/data_type_varbinary_serde.h index 41dfa3fce002f8..6d23159c8ea5fe 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.h +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.h @@ -81,6 +81,12 @@ class DataTypeVarbinarySerDe : public DataTypeSerDe { "read_column_from_arrow with type " + column.get_name()); } + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, const FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h new file mode 100644 index 00000000000000..e9762385689d07 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/check.h" +#include "common/status.h" +#include "core/column/column.h" +#include "core/string_ref.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris { + +// These enums deliberately do not expose parquet thrift classes to the core type system. The +// format reader translates the thrift metadata once when it creates a column reader. +enum class ParquetPhysicalType { + BOOLEAN, + INT32, + INT64, + INT96, + FLOAT, + DOUBLE, + BYTE_ARRAY, + FIXED_LEN_BYTE_ARRAY, +}; + +enum class ParquetValueEncoding { + PLAIN, + DICTIONARY, + RLE, + BIT_PACKED, + DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, + BYTE_STREAM_SPLIT, +}; + +enum class ParquetTimeUnit { + UNKNOWN, + MILLIS, + MICROS, + NANOS, +}; + +enum class ParquetLogicalType { + NONE, + STRING, + DECIMAL, + DATE, + TIME, + TIMESTAMP, + INTEGER, + UUID, + FLOAT16, +}; + +// Immutable metadata required to turn one Parquet physical value into the selected Doris type. +// Encoding describes how the value source is read; logical annotations describe its meaning. +struct ParquetDecodeContext { + ParquetPhysicalType physical_type = ParquetPhysicalType::INT32; + ParquetValueEncoding encoding = ParquetValueEncoding::PLAIN; + ParquetLogicalType logical_type = ParquetLogicalType::NONE; + ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; + + int32_t type_length = -1; + int32_t decimal_precision = -1; + int32_t decimal_scale = -1; + int32_t logical_integer_bit_width = -1; + bool logical_integer_is_signed = true; + bool timestamp_is_adjusted_to_utc = false; + bool logical_float16 = false; + bool logical_uuid = false; + bool dictionary_index_only = false; + + const cctz::time_zone* timezone = nullptr; +}; + +struct ParquetSelectionRange { + size_t first = 0; + size_t count = 0; +}; + +// A decoder may produce multiple contiguous spans for one request (for example delta encodings). +// Consumers are invoked per span, never per value, keeping virtual dispatch out of the row loop. +class ParquetFixedValueConsumer { +public: + virtual ~ParquetFixedValueConsumer() = default; + virtual Status consume(const uint8_t* values, size_t num_values, size_t value_width) = 0; + virtual Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) { + for (const auto& range : ranges) { + RETURN_IF_ERROR(consume(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } +}; + +class ParquetBinaryValueConsumer { +public: + virtual ~ParquetBinaryValueConsumer() = default; + virtual Status consume(const StringRef* values, size_t num_values) = 0; + + // PLAIN BYTE_ARRAY decoders already have to parse every length prefix. Publish the parsed + // source and destination offsets so string columns do not rebuild an equally large StringRef + // array and rescan all lengths before copying. Spans are expressed in output coordinates and + // preserve adjacent surviving runs for consumers that can amortize range setup. + virtual Status consume_plain_byte_array(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) { + std::vector values; + values.reserve(num_values); + for (size_t row = 0; row < num_values; ++row) { + values.emplace_back(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return consume(values.data(), values.size()); + } +}; + +// Dictionary decoders publish validated IDs without knowing the destination Doris type. A +// cache-resident dictionary can therefore fuse RLE decode with target-column gathering, while a +// large sparse dictionary can still choose a compact ID buffer before random dictionary access. +class ParquetDictionaryValueConsumer { +public: + virtual ~ParquetDictionaryValueConsumer() = default; + virtual Status consume_indices(const uint32_t* indices, size_t num_values) = 0; + virtual Status consume_repeated(uint32_t index, size_t num_values) { + std::vector indices(num_values, index); + return consume_indices(indices.data(), indices.size()); + } +}; + +// Physical value ranges selected from one page-bounded decode request. Definition-level NULLs do +// not occupy the encoded value stream, so the native ColumnReader merges the logical selection +// with definition levels before constructing this plan. The SerDe sees only selected non-NULL +// payload, while the reader restores selected NULL slots after the compact decode. Ranges are +// sorted, disjoint, and expressed in the physical value stream's coordinate space. +struct ParquetSelection { + size_t total_values = 0; + size_t selected_values = 0; + std::vector ranges; +}; + +// Encoding decoders implement this interface. They own encoded-stream cursors and dictionary +// storage, but they never know the destination Doris column type. DataTypeSerDe owns the consumer +// and therefore the physical/logical-to-Doris conversion. +class ParquetDecodeSource { +public: + virtual ~ParquetDecodeSource() = default; + + virtual Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) = 0; + virtual Status decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) = 0; + virtual Status skip_values(size_t num_values) = 0; + + // Batch-level sparse decode. The default implementation preserves every encoding's cursor + // semantics while moving SerDe dispatch and consumer construction out of the selection-run + // loop. Decoders with cheap random access or batch decode override these methods to remove the + // remaining per-range virtual calls as well. + virtual Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_fixed_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + virtual Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_binary_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + + virtual bool has_dictionary() const { return false; } + virtual uint64_t dictionary_generation() const { return 0; } + virtual size_t dictionary_size() const { return 0; } + virtual Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return Status::NotSupported("Parquet dictionary is not supported by this decoder"); + } + virtual Status decode_dictionary_indices(size_t num_values, std::vector* indices) { + return Status::NotSupported("Parquet dictionary indices are not supported by this decoder"); + } + virtual Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) { + DORIS_CHECK(indices != nullptr); + indices->clear(); + indices->reserve(selection.selected_values); + std::vector range_indices; + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_dictionary_indices(range.count, &range_indices)); + indices->insert(indices->end(), range_indices.begin(), range_indices.end()); + cursor = range.first + range.count; + } + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + DORIS_CHECK_EQ(indices->size(), selection.selected_values); + return Status::OK(); + } + virtual Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_dictionary_indices(num_values, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_selected_dictionary_indices(selection, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const { + return false; + } +}; + +enum class ParquetDictionaryMaterializationStrategy : uint8_t { DIRECT, INDICES }; + +// Dictionary values are materialized once into the selected Doris type. The state belongs to a +// column reader rather than DataTypeSerDe because a SerDe instance can be shared by many files. +struct ParquetMaterializationState { + MutableColumnPtr typed_dictionary; + std::vector dictionary_indices; + ParquetSelection selection; + uint64_t dictionary_generation = std::numeric_limits::max(); + bool enable_strict_mode = false; + IColumn::Filter* conversion_failure_null_map = nullptr; + IColumn::Filter dictionary_conversion_failures; + bool capturing_dictionary_conversion_failures = false; + bool dictionary_has_conversion_failures = false; + size_t dictionary_failure_scan_rows = 0; + ParquetDictionaryMaterializationStrategy dictionary_materialization_strategy = + ParquetDictionaryMaterializationStrategy::INDICES; + + void reset_dictionary() { + typed_dictionary.reset(); + dictionary_indices.clear(); + dictionary_conversion_failures.clear(); + capturing_dictionary_conversion_failures = false; + dictionary_has_conversion_failures = false; + dictionary_failure_scan_rows = 0; + dictionary_generation = std::numeric_limits::max(); + } + + bool can_insert_null_on_conversion_failure() const { + return conversion_failure_null_map != nullptr && + (!enable_strict_mode || capturing_dictionary_conversion_failures); + } + + bool mark_conversion_failure(size_t output_row) { + if (!can_insert_null_on_conversion_failure()) { + return false; + } + DORIS_CHECK_LT(output_row, conversion_failure_null_map->size()); + (*conversion_failure_null_map)[output_row] = 1; + if (capturing_dictionary_conversion_failures) { + dictionary_has_conversion_failures = true; + } + return true; + } + + IColumn::Filter* begin_dictionary_conversion(size_t dictionary_size) { + auto* output_null_map = conversion_failure_null_map; + dictionary_conversion_failures.resize_fill(dictionary_size, 0); + dictionary_has_conversion_failures = false; + conversion_failure_null_map = &dictionary_conversion_failures; + capturing_dictionary_conversion_failures = true; + return output_null_map; + } + + void end_dictionary_conversion(IColumn::Filter* output_null_map) { + conversion_failure_null_map = output_null_map; + capturing_dictionary_conversion_failures = false; + } + + Status materialize_dictionary(IColumn& column) { + const size_t old_size = column.size(); + dictionary_failure_scan_rows = 0; + if (UNLIKELY(dictionary_has_conversion_failures)) { + const bool insert_failure_as_null = can_insert_null_on_conversion_failure(); + if (!insert_failure_as_null) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] == 0) { + continue; + } + // A malformed dictionary entry is irrelevant until a selected row references + // it; failing while building the dictionary would reject valid pages. + return Status::DataQualityError( + "Parquet dictionary entry {} cannot be converted to the target type", + dictionary_id); + } + } + } + column.insert_indices_from(*typed_dictionary, dictionary_indices.data(), + dictionary_indices.data() + dictionary_indices.size()); + if (UNLIKELY(dictionary_has_conversion_failures && + can_insert_null_on_conversion_failure())) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] != 0) { + mark_conversion_failure(old_size + row); + } + } + } + return Status::OK(); + } + + Status materialize_dictionary(IColumn& column, ParquetDecodeSource& source, size_t num_values) { + DORIS_CHECK(typed_dictionary); + if (UNLIKELY(dictionary_has_conversion_failures) || + source.prefer_dictionary_index_materialization(typed_dictionary->byte_size())) { + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::INDICES; + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &dictionary_indices)); + DORIS_CHECK_EQ(dictionary_indices.size(), num_values); + return materialize_dictionary(column); + } + + class ColumnConsumer final : public ParquetDictionaryValueConsumer { + public: + ColumnConsumer(IColumn& destination, const IColumn& dictionary) + : _destination(destination), _dictionary(dictionary) {} + + Status consume_indices(const uint32_t* indices, size_t num_values) override { + _destination.insert_indices_from(_dictionary, indices, indices + num_values); + return Status::OK(); + } + + Status consume_repeated(uint32_t index, size_t num_values) override { + _destination.insert_many_from(_dictionary, index, num_values); + return Status::OK(); + } + + private: + IColumn& _destination; + const IColumn& _dictionary; + } consumer(column, *typed_dictionary); + + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::DIRECT; + const size_t old_size = column.size(); + const Status status = source.decode_dictionary_values(num_values, consumer); + if (!status.ok()) { + // Streaming direct gather may discover a corrupt late ID after earlier valid runs; + // restore the same all-or-nothing column invariant as the index-buffer path. + column.resize(old_size); + } + return status; + } +}; + +} // namespace doris diff --git a/be/src/core/data_type_serde/parquet_timestamp.h b/be/src/core/data_type_serde/parquet_timestamp.h new file mode 100644 index 00000000000000..87ff18d28ce331 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_timestamp.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { + +#pragma pack(1) +struct ParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(ParquetInt96Timestamp) == 12); + +inline constexpr int64_t MIN_DORIS_TIMESTAMP_MICROS = -62135596800000000LL; +inline constexpr int64_t MAX_DORIS_TIMESTAMP_MICROS = 253402300799999999LL; + +inline Status validate_parquet_timestamp_micros(int64_t timestamp_micros) { + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet timestamp is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } + return Status::OK(); +} + +inline Status parquet_timestamp_micros(ParquetTimeUnit unit, int64_t value, int64_t* result) { + if (unit == ParquetTimeUnit::MILLIS) { + // Validate in the source unit before scaling; signed overflow could otherwise turn an + // invalid file value into an in-range timestamp that survives later range checks. + if (value > std::numeric_limits::max() / 1000 || + value < std::numeric_limits::min() / 1000) { + return Status::DataQualityError("Parquet timestamp overflows microseconds"); + } + *result = value * 1000; + } else if (unit == ParquetTimeUnit::NANOS) { + *result = value / 1000; + } else { + *result = value; + } + return validate_parquet_timestamp_micros(*result); +} + +inline Status parquet_int96_timestamp_micros(const ParquetInt96Timestamp& value, int64_t* result) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_IN_DAY = 86400000000LL; + static constexpr int64_t NANOS_IN_DAY = 86400000000000LL; + static constexpr int64_t NANOS_PER_MICROSECOND = 1000; + + // INT96 nanos is a time-of-day, not a signed duration. Validate it before widened day + // arithmetic so corrupt input cannot wrap into a plausible Doris timestamp. + if (value.nanos_of_day < 0 || value.nanos_of_day >= NANOS_IN_DAY) { + return Status::DataQualityError("Invalid Parquet INT96 nanos-of-day: {}", + value.nanos_of_day); + } + const __int128 days = static_cast(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + const __int128 timestamp_micros = + days * MICROS_IN_DAY + value.nanos_of_day / NANOS_PER_MICROSECOND; + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet INT96 timestamp is outside the Doris 0001-9999 range"); + } + *result = static_cast(timestamp_micros); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index f06e9fb9533774..f2dd05854c2362 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -65,6 +65,7 @@ #include "io/io_common.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "service/backend_options.h" #include "storage/id_manager.h" @@ -324,26 +325,42 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { RETURN_IF_ERROR(Scanner::init(state, conjuncts)); - _get_block_timer = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerV2GetBlockTime", 1); - _empty_file_counter = - ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "EmptyFileNum", TUnit::UNIT, 1); - _not_found_file_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "NotFoundFileNum", TUnit::UNIT, 1); - _file_counter = - ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT, 1); - _file_read_bytes_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadBytes", TUnit::BYTES, 1); - _file_read_calls_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadCalls", TUnit::UNIT, 1); + auto* profile = _local_state->scanner_profile(); + const auto hierarchy = file_scan_profile::ensure_hierarchy(profile); + _scanner_total_timer = hierarchy.scanner; + _io_timer = hierarchy.io; + _init_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2InitTime", + file_scan_profile::SCANNER, 1); + _open_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2OpenTime", + file_scan_profile::SCANNER, 1); + _get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetBlockTime", + file_scan_profile::SCANNER, 1); + _prepare_split_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2PrepareSplitTime", + file_scan_profile::SCANNER, 1); + _get_next_range_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetNextRangeTime", + file_scan_profile::SCANNER, 1); + _close_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2CloseTime", + file_scan_profile::SCANNER, 1); + _empty_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptyFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _not_found_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NotFoundFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNumber", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_read_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadBytes", TUnit::BYTES, + file_scan_profile::IO, 1); + _file_read_calls_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadCalls", TUnit::UNIT, + file_scan_profile::IO, 1); _file_read_time_counter = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileReadTime", 1); - _adaptive_batch_predicted_rows_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchPredictedRows", TUnit::UNIT, 1); - _adaptive_batch_actual_bytes_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchActualBytes", TUnit::BYTES, 1); - _adaptive_batch_probe_count_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchProbeCount", TUnit::UNIT, 1); + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReadTime", file_scan_profile::IO, 1); + _adaptive_batch_predicted_rows_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchPredictedRows", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _adaptive_batch_actual_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchActualBytes", TUnit::BYTES, file_scan_profile::SCANNER, 1); + _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchProbeCount", TUnit::UNIT, file_scan_profile::SCANNER, 1); + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_init_timer); _file_cache_statistics = std::make_unique(); _file_reader_stats = std::make_unique(); RETURN_IF_ERROR(_init_io_ctx()); @@ -354,6 +371,8 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc } Status FileScannerV2::_open_impl(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_open_timer); RETURN_IF_CANCELLED(state); RETURN_IF_ERROR(Scanner::_open_impl(state)); RETURN_IF_ERROR(_get_next_scan_range(&_first_scan_range)); @@ -367,6 +386,7 @@ Status FileScannerV2::_open_impl(RuntimeState* state) { } Status FileScannerV2::_get_next_scan_range(bool* has_next) { + SCOPED_TIMER(_get_next_range_timer); DORIS_CHECK(has_next != nullptr); RETURN_IF_ERROR(_split_source->get_next(has_next, &_current_range)); if (*has_next) { @@ -376,6 +396,8 @@ Status FileScannerV2::_get_next_scan_range(bool* has_next) { } Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* eof) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_get_block_timer); while (true) { RETURN_IF_CANCELLED(state); if (!_has_prepared_split) { @@ -386,7 +408,6 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } { - SCOPED_TIMER(_get_block_timer); if (_should_run_adaptive_batch_size()) { _table_reader->set_batch_size(_predict_reader_batch_rows()); } @@ -428,6 +449,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } Status FileScannerV2::_prepare_next_split(bool* eos) { + SCOPED_TIMER(_prepare_split_timer); while (true) { bool has_next = _first_scan_range; if (!_first_scan_range) { @@ -906,6 +928,8 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) { } Status FileScannerV2::close(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_close_timer); if (!_try_close()) { return Status::OK(); } @@ -1045,14 +1069,31 @@ void FileScannerV2::_collect_profile_before_close() { Scanner::_collect_profile_before_close(); if (config::enable_file_cache && _state->query_options().enable_file_cache && _profile != nullptr) { - _report_file_cache_profile(_profile, *_file_cache_statistics); + auto file_cache_delta = io::diff_file_cache_statistics(*_file_cache_statistics, + _reported_file_cache_statistics); + // Profile collection can run more than once. Keep additive fields incremental while + // publishing high-water gauges and peer identities from the latest complete snapshot. + file_cache_delta.remote_only_on_miss_triggered = + _file_cache_statistics->remote_only_on_miss_triggered; + file_cache_delta.remote_only_on_miss_threshold_bytes = + _file_cache_statistics->remote_only_on_miss_threshold_bytes; + file_cache_delta.peer_hosts = _file_cache_statistics->peer_hosts; + _report_file_cache_profile(_profile, file_cache_delta); _state->get_query_ctx()->resource_ctx()->io_context()->update_bytes_write_into_cache( - _file_cache_statistics->bytes_write_into_cache); + file_cache_delta.bytes_write_into_cache); + _reported_file_cache_statistics = *_file_cache_statistics; } if (_file_reader_stats != nullptr) { COUNTER_SET(_file_read_bytes_counter, cast_set(_file_reader_stats->read_bytes)); COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); + const auto read_time = cast_set(_file_reader_stats->read_time_ns); + DORIS_CHECK(read_time >= _reported_io_read_time); + // Some transports (for example Arrow Flight) record directly into IO, while filesystem + // reads arrive through FileReaderStats. Add only the new traced delta so both paths remain + // visible without double counting repeated profile publication. + COUNTER_UPDATE(_io_timer, read_time - _reported_io_read_time); + _reported_io_read_time = read_time; } // Query profiles can be collected before Scanner::close() runs. Publish condition-cache // counters here as well, using deltas so this method and close() cannot double count. @@ -1061,7 +1102,8 @@ void FileScannerV2::_collect_profile_before_close() { void FileScannerV2::_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics) { - io::FileCacheProfileReporter cache_profile(profile); + file_scan_profile::ensure_hierarchy(profile); + io::FileCacheProfileReporter cache_profile(profile, file_scan_profile::IO); cache_profile.update(&file_cache_statistics); } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 0cd03030b56851..1d2a62f8f9af1a 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -185,12 +185,20 @@ class FileScannerV2 final : public Scanner { std::unordered_map _partition_slot_descs; std::unique_ptr _file_cache_statistics; + io::FileCacheStatistics _reported_file_cache_statistics; std::unique_ptr _file_reader_stats; std::shared_ptr _io_ctx; ShardedKVCache* _kv_cache = nullptr; + RuntimeProfile::Counter* _scanner_total_timer = nullptr; + RuntimeProfile::Counter* _init_timer = nullptr; + RuntimeProfile::Counter* _open_timer = nullptr; RuntimeProfile::Counter* _get_block_timer = nullptr; RuntimeProfile::Counter* _empty_file_counter = nullptr; + RuntimeProfile::Counter* _prepare_split_timer = nullptr; + RuntimeProfile::Counter* _get_next_range_timer = nullptr; + RuntimeProfile::Counter* _close_timer = nullptr; + RuntimeProfile::Counter* _io_timer = nullptr; RuntimeProfile::Counter* _not_found_file_counter = nullptr; RuntimeProfile::Counter* _file_counter = nullptr; RuntimeProfile::Counter* _file_read_bytes_counter = nullptr; @@ -207,6 +215,7 @@ class FileScannerV2 final : public Scanner { int64_t _last_read_rows = 0; int64_t _last_bytes_read_from_local = 0; int64_t _last_bytes_read_from_remote = 0; + int64_t _reported_io_read_time = 0; }; } // namespace doris diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index b6e5c13d111756..d1bdebe17a2251 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -24,9 +24,11 @@ #include #include +#include #include #include +#include "common/compare.h" #include "common/config.h" #include "common/exception.h" #include "common/logging.h" @@ -82,6 +84,82 @@ const static std::set DISTANCE_FUNCS = {L2DistanceApproximate::name const static std::set OPS_FOR_ANN_RANGE_SEARCH = { TExprOpcode::GE, TExprOpcode::LE, TExprOpcode::LE, TExprOpcode::GT, TExprOpcode::LT}; +namespace { + +enum class RawComparisonOp : uint8_t { EQ, NE, LT, LE, GT, GE }; + +std::optional raw_comparison_op(std::string_view function_name, bool reverse) { + RawComparisonOp op; + if (function_name == "eq") { + op = RawComparisonOp::EQ; + } else if (function_name == "ne") { + op = RawComparisonOp::NE; + } else if (function_name == "lt") { + op = RawComparisonOp::LT; + } else if (function_name == "le") { + op = RawComparisonOp::LE; + } else if (function_name == "gt") { + op = RawComparisonOp::GT; + } else if (function_name == "ge") { + op = RawComparisonOp::GE; + } else { + return std::nullopt; + } + if (!reverse || op == RawComparisonOp::EQ || op == RawComparisonOp::NE) { + return op; + } + switch (op) { + case RawComparisonOp::LT: + return RawComparisonOp::GT; + case RawComparisonOp::LE: + return RawComparisonOp::GE; + case RawComparisonOp::GT: + return RawComparisonOp::LT; + case RawComparisonOp::GE: + return RawComparisonOp::LE; + case RawComparisonOp::EQ: + case RawComparisonOp::NE: + break; + } + __builtin_unreachable(); +} + +template +void execute_raw_comparison(const uint8_t* values, size_t num_values, const Field& literal, + RawComparisonOp op, uint8_t* matches) { + const T rhs = literal.get(); + for (size_t row = 0; row < num_values; ++row) { + if (matches[row] == 0) { + continue; + } + const T lhs = unaligned_load(values + row * sizeof(T)); + bool keep = false; + switch (op) { + case RawComparisonOp::EQ: + keep = Compare::equal(lhs, rhs); + break; + case RawComparisonOp::NE: + keep = Compare::not_equal(lhs, rhs); + break; + case RawComparisonOp::LT: + keep = Compare::less(lhs, rhs); + break; + case RawComparisonOp::LE: + keep = Compare::less_equal(lhs, rhs); + break; + case RawComparisonOp::GT: + keep = Compare::greater(lhs, rhs); + break; + case RawComparisonOp::GE: + keep = Compare::greater_equal(lhs, rhs); + break; + } + matches[row] = static_cast(keep); + } +} + +} // namespace + VectorizedFnCall::VectorizedFnCall(const TExprNode& node) : VExpr(node) { _function_name = _fn.name.function_name; } @@ -339,6 +417,71 @@ Status VectorizedFnCall::execute_column_impl(VExprContext* context, const Block* return _do_execute(context, block, selector, count, result_column, nullptr); } +bool VectorizedFnCall::can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + if (data_type == nullptr || !raw_comparison_op(_function_name, false).has_value()) { + return false; + } + auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + if (!slot_literal.has_value() || slot_literal->slot_index != column_id || + slot_literal->literal.is_null()) { + return false; + } + const auto raw_type = remove_nullable(data_type); + if (!remove_nullable(slot_literal->slot_type)->equals(*raw_type) || + !remove_nullable(slot_literal->literal_type)->equals(*raw_type)) { + return false; + } + const auto primitive_type = raw_type->get_primitive_type(); + return primitive_type == TYPE_INT || primitive_type == TYPE_BIGINT || + primitive_type == TYPE_FLOAT || primitive_type == TYPE_DOUBLE; +} + +Status VectorizedFnCall::execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const { + if (!can_execute_on_raw_fixed_values(data_type, column_id)) { + return Status::NotSupported("Expression {} cannot evaluate raw fixed-width values", + expr_name()); + } + DORIS_CHECK(values != nullptr || num_values == 0); + DORIS_CHECK(matches != nullptr || num_values == 0); + const auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + DORIS_CHECK(slot_literal.has_value()); + const auto op = raw_comparison_op(_function_name, slot_literal->literal_on_left); + DORIS_CHECK(op.has_value()); + const auto primitive_type = remove_nullable(data_type)->get_primitive_type(); + const size_t expected_width = primitive_type == TYPE_INT || primitive_type == TYPE_FLOAT + ? sizeof(uint32_t) + : sizeof(uint64_t); + if (value_width != expected_width) { + return Status::Corruption("Raw expression width {} does not match expected {}", value_width, + expected_width); + } + switch (primitive_type) { + case TYPE_INT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_BIGINT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_FLOAT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_DOUBLE: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + default: + __builtin_unreachable(); + } + return Status::OK(); +} + const std::string& VectorizedFnCall::expr_name() const { return _expr_name; } diff --git a/be/src/exprs/vectorized_fn_call.h b/be/src/exprs/vectorized_fn_call.h index 8343242ce314fd..f4e0dcb8f527fe 100644 --- a/be/src/exprs/vectorized_fn_call.h +++ b/be/src/exprs/vectorized_fn_call.h @@ -57,6 +57,11 @@ class VectorizedFnCall : public VExpr { Status execute_runtime_filter(VExprContext* context, const Block* block, const uint8_t* __restrict filter, size_t count, ColumnPtr& result_column, ColumnPtr* arg_column) const override; + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override; + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const override; Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; bool can_evaluate_zonemap_filter() const override; diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index f54af6f7819acb..77520377af6f93 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -174,6 +174,19 @@ class VExpr { uint8_t* __restrict result_filter_data, size_t rows, bool accept_null, bool* can_filter_all) const; + // Raw fixed-width evaluation is an optional expression capability used before a storage reader + // materializes a column. `matches` is ANDed in place; callers handle NULL rows separately + // because raw value streams contain only non-NULL payloads. + virtual bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + return false; + } + virtual Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, const DataTypePtr& data_type, + int column_id, uint8_t* matches) const { + return Status::NotSupported("{} cannot evaluate raw fixed-width values", expr_name()); + } + // `is_blockable` means this expr will be blocked in `execute` (e.g. AI Function, Remote Function) [[nodiscard]] virtual bool is_blockable() const { return std::any_of(_children.begin(), _children.end(), diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index 377a26ce530db8..e417617cf5a64f 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -108,6 +108,107 @@ instructions as well; this file adds format-v2-specific review expectations. - For JNI readers, review local/global reference lifetime, exception propagation, type conversion, thread attachment assumptions, and cleanup on partial initialization. +### Parquet Native Decode Kernel + +- Keep new production integration under `be/src/format_v2/parquet/`. Doris v1 is the behavior and + performance baseline, but v2 owns an independent page/encoding reader and must not call the v1 + `ParquetColumnReader`. Do not modify `be/src/format/parquet/` for a v2 decoder change. Reimplement + the required behavior under the v2 tree and keep v1 unchanged so differential correctness and + performance results remain meaningful. +- Keep the native decode boundary independent of both Arrow descriptors/builders and table-schema + objects. A Column Chunk schema contract should contain only immutable physical type, fixed width, + and Dremel-level thresholds. Review constructor arguments and stored references for ownership and + lifetime; metadata owned by a temporary schema adapter must not escape into a persistent reader. +- Treat selection positions as logical Row Group rows, including null rows. Selection indices must + be sorted, unique, and bounded by the batch's logical row count. Dense identity, empty selection, + and fragmented selection must have explicit representations and tests; do not silently mix row + ordinals with non-null value ordinals or dictionary IDs. +- Verify the three decode counts separately: logical rows consumed, encoded non-null payload values + consumed, and output values materialized. Null and filtered-null runs consume no payload; + selected and filtered non-null runs both consume payload. Every page transition, skip, error, and + end-of-batch path must leave all three cursors aligned for the next call. +- A flat scalar fast path may run only when `max_repetition_level == 0`. Repeated leaves require a + definition/repetition-level plan that identifies parent-row boundaries, empty and null + collections, null ancestors, and rows spanning pages. Choose one physical leaf as the parent + shape owner; sibling leaf streams advance over the same parent-row range and validate their + payload counts instead of independently redefining the parent shape. +- The old Arrow value-reader hierarchy (`ParquetLeafBatch`, scalar/list/map/struct readers, and the + nested load/build/consume protocol) has been removed. Do not reintroduce an intermediate decoded + batch or a stateful load-before-build phase. Ordinary predicate/output scans construct + `NativeColumnReader`, consume compressed page data through the native decoder, and append directly + into the final Doris column. +- Keep logical schema changes distinct from physical decoding. The native reader materializes the + projected file type only; `ColumnMapper` and `TableReader` own every file-to-table cast. A type + mismatch at the native reader boundary is an invariant violation, not a reason to create a + reader-local conversion path or expose a decoder-owned value batch. +- `CountColumnReader` uses the v2 native `LevelReader`; it selects one representative leaf (the key + for MAP) and advances only definition/repetition levels. It exposes no value API and must not be + reused as a scan reader or expanded into a fallback path. +- Build ARRAY/MAP/STRUCT parent boundaries, offsets, nulls, and child payload spans in one traversal + of the owning leaf's levels. For example, `[[1, 2], NULL, []]` must yield entry counts + `[2, 0, 0]` and parent nulls `[0, 1, 0]` without rescanning that leaf. MAP key levels own entry + existence; value levels validate against that shape. STRUCT siblings validate parent-row + alignment. If every projected STRUCT child is missing, consume only a retained physical leaf's + levels and never materialize its payload into a temporary Doris column. +- Do not size level or selection scratch from a 16-bit batch-row assumption. A repeated parent row + can contain more level entries than the requested parent-row batch. Split large runs without + changing alternation or row-boundary semantics, and check overflow before narrowing counts. +- Decoder dispatch must reject an incompatible physical type, encoding, or type length explicitly. + Review PLAIN, dictionary/RLE, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, BYTE_STREAM_SPLIT, BOOLEAN RLE, and level RLE/bit-packed paths for identical + selection and malformed-input behavior. Page V1 and V2 must feed the same decoder contract after + their different level/decompression layouts are parsed. +- Keep every index coordinate domain explicit: table-local column ID, physical leaf-column ID, + Row Group ID, data-page ordinal, OffsetIndex row ordinal, logical batch-row ordinal, non-null + payload ordinal, and dictionary-entry ID are different types of identity. Data-page ordinals must + exclude dictionary pages consistently. Dictionary-entry bitmaps are local to one Column Chunk + dictionary and cannot be reused after a Row Group, dictionary, or encoding transition. +- Review index composition, not only each index in isolation. Row Group statistics, dictionary, + Bloom, ColumnIndex/OffsetIndex, page cache registration, page skip plans, SelectionVector, and + lazy column cursors must describe the same surviving logical rows. Missing or unusable optional + indexes retain candidates; structurally inconsistent indexes or out-of-range IDs return an + explicit corruption error. A mixed dictionary/plain Column Chunk must leave dictionary-ID + filtering before any cursor is consumed. +- Decoder owns encoded-stream parsing and cursor movement only. `DataTypeSerDe` owns Parquet + physical/logical interpretation and writes directly into Doris columns. Dictionary pages are + materialized once through the same SerDe and data pages expose only validated dictionary indices. + Decimal and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, + precision, and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. +- Do not add an Arrow runtime fallback. Once an ordinary v2 scan selects its native Parquet reader, + unsupported physical types, encodings, page layouts, or malformed inputs return an explicit + status. Arrow may be used by metadata planning and as a test oracle; no Arrow array, builder, + RecordReader, or metadata lifetime belongs in data-page value or level materialization. +- Reuse decoder, SerDe, null-map, selection-range, binary-value, level, and builder scratch across + batches. String-like decoders should gather selected `StringRef` values and append once per batch, + rather than allocate or grow the destination once per run. Scratch capacity may grow to a bounded + high-water mark but must be reset logically between pages, Row Groups, files, and errors. +- Adaptive batch sizing must measure completed Doris output rows/bytes and must not recreate native + readers, builders, or scratch when only the row cap changes. Compare the v1 and v2 lifecycle: + probe batches must not turn persistent setup into a per-batch cost or amplify highly fragmented + selection work. +- Footer and metadata caching must key on stable file identity and cache the serialized footer plus + parsed native metadata at the same lifecycle as v1. Never reuse metadata when path, file size, + modification/version identity, encryption state, or schema-affecting options differ. Cache misses + and uncacheable identities must remain correct without a fallback to stale entries. +- Page-cache behavior must match v1 for cache key, stable file identity, registered byte range, + compressed/decompressed entry kind, checksum/decompression ownership, subrange coverage, + invalidation, admission, and fallback I/O. Any intentional difference needs benchmark and memory + evidence showing it is no worse for v1 workloads. Cache lookup must never alter page ordinal, + decoder, level, or dictionary cursor state. +- Apply v1's MergeRange decision to the native data-page reader, after metadata/dictionary probes + finish. Predicate and lazy readers for one Row Group must share one ordered-range wrapper, and + native per-column prefetch must be disabled while that wrapper is active. Never allocate one + MergeRange buffer per leaf; wide complex projections would multiply its bounded scratch memory. +- Preserve observability inside aggregate counters. `TotalBatches` must be decomposable into probe, + dense, selected, empty, page-crossing, and nested/fragmented work where relevant; decode, level, + selection, conversion, allocation, and materialization time must remain attributable without + adding per-row timer overhead. +- Profile index attempts, successes, conservative fallbacks, and corrupt rejections separately for + statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skipping. Footer/page/file/ + condition-cache counters must expose requests, hits, misses, writes/admissions, bytes, wait/I/O + time, and bypass reasons with semantics aligned to v1. A lower total timer without its internal + work counters is not sufficient observability. + ## Detailed FileReader Review Guides - Before reviewing any FileReader implementation, index, predicate path, cache, or virtual column, @@ -174,6 +275,15 @@ instructions as well; this file adds format-v2-specific review expectations. malformed input. - For bug fixes, require a test that fails for the original reachable path and validates the result, row count, or explicit error after the fix. +- For native Parquet work, require a matrix across physical/logical types, every supported encoding, + Page V1/V2, required/optional/repeated levels, dictionary fallback, page and batch boundaries, + dense/empty/fragmented selections, null placement, malformed/truncated input, and files from + representative external writers. Include focused cursor-invariant tests and end-to-end nested + reconstruction tests; a scalar happy-path benchmark is not sufficient coverage. +- Performance-sensitive decoder changes need a reproducible comparison against v1 and a relevant + external baseline such as DuckDB. Report data shape, encoding, selectivity, null/cardinality + distribution, compression, storage path, batch policy, warm/cold cache state, CPU time, rows/s, + bytes/s, allocation behavior, and Profile counter deltas. ## Review Output diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 6713183c09d816..4b8f94ae2722db 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -2079,6 +2079,7 @@ Status TableColumnMapper::create_scan_request( // table-column to file-column conversion, so it also owns the file-local block positions. file_request->predicate_columns.clear(); file_request->non_predicate_columns.clear(); + file_request->predicate_only_columns.clear(); file_request->local_positions.clear(); file_request->conjuncts.clear(); file_request->delete_conjuncts.clear(); @@ -2110,6 +2111,27 @@ Status TableColumnMapper::create_scan_request( // Hidden filter mappings must be built before localizing filters, so that they can be localized together with visible mappings and referenced by localized filter expressions. RETURN_IF_ERROR(_build_hidden_filter_mappings(table_filters)); RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state)); + for (const auto& mapping : _hidden_mappings) { + if (!mapping.file_local_id.has_value()) { + continue; + } + const auto local_id = LocalColumnId(*mapping.file_local_id); + const bool is_visible_output = + std::ranges::any_of(_mappings, [local_id](const ColumnMapping& visible_mapping) { + return visible_mapping.file_local_id.has_value() && + LocalColumnId(*visible_mapping.file_local_id) == local_id; + }); + if (is_visible_output) { + continue; + } + if (std::ranges::any_of(file_request->predicate_columns, + [local_id](const LocalColumnIndex& projection) { + return projection.column_id() == local_id; + }) && + !file_request->is_predicate_only(local_id)) { + file_request->predicate_only_columns.push_back(local_id); + } + } // 3. Rebuild output projection expressions for projected columns. localize_filters() has // already applied the final scan projection to mapping.file_type/projected_file_children before // rewriting filter expressions. diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp b/be/src/format_v2/delimited_text/delimited_text_reader.cpp index f19d12c75714b9..bad7d9cf7d6d1b 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.cpp +++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp @@ -38,6 +38,7 @@ #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/string_util.h" @@ -180,7 +181,9 @@ void DelimitedTextReader::_init_profile() { return; } - ADD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, 1); + file_scan_profile::ensure_hierarchy(_profile); + _text_profile.total_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, + file_scan_profile::FILE_READER, 1); _text_profile.open_file_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OpenFileTime", DELIMITED_TEXT_PROFILE, 1); _text_profile.create_line_reader_time = @@ -199,8 +202,10 @@ void DelimitedTextReader::_init_profile() { _profile, "RawLinesRead", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_read_before_filter = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsReadBeforeFilter", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + // RuntimeProfile counter names are global within one profile, so the format prefix preserves + // independent ownership when Parquet and delimited readers are initialized in either order. _text_profile.rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _profile, "DelimitedRowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_filtered_by_delete_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsFilteredByDeleteConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_returned = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -215,6 +220,7 @@ void DelimitedTextReader::_init_profile() { Status DelimitedTextReader::init(RuntimeState* state) { _init_profile(); + SCOPED_TIMER(_text_profile.total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("{} v2 reader requires scan params", _reader_name); @@ -275,6 +281,7 @@ Status DelimitedTextReader::init(RuntimeState* state) { } Status DelimitedTextReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_text_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("{} v2 file_schema is null", _reader_name); } @@ -288,6 +295,7 @@ std::unique_ptr DelimitedTextReader::create_column_mapper( } Status DelimitedTextReader::open(std::shared_ptr request) { + SCOPED_TIMER(_text_profile.total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); @@ -307,6 +315,7 @@ Status DelimitedTextReader::open(std::shared_ptr request) { } Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -358,6 +367,7 @@ Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& request, FileAggregateResult* result) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(result != nullptr); if (request.agg_type != TPushAggOp::type::COUNT) { return Status::NotSupported("{} v2 reader only supports COUNT aggregate pushdown", @@ -396,6 +406,7 @@ Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& req } Status DelimitedTextReader::close() { + SCOPED_TIMER(_text_profile.total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.h b/be/src/format_v2/delimited_text/delimited_text_reader.h index daea3bc90942d8..dff27980c9cd12 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.h +++ b/be/src/format_v2/delimited_text/delimited_text_reader.h @@ -59,6 +59,7 @@ class DelimitedTextReader : public FileReader { protected: struct DelimitedTextProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* open_file_time = nullptr; RuntimeProfile::Counter* create_line_reader_time = nullptr; RuntimeProfile::Counter* read_line_time = nullptr; diff --git a/be/src/format_v2/expr/equality_delete_predicate.cpp b/be/src/format_v2/expr/equality_delete_predicate.cpp index 4d5c511abdb181..1d111aa7a74930 100644 --- a/be/src/format_v2/expr/equality_delete_predicate.cpp +++ b/be/src/format_v2/expr/equality_delete_predicate.cpp @@ -26,28 +26,63 @@ #include "core/assert_cast.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "util/hash_util.hpp" namespace doris::format { namespace { bool column_value_equal(const ColumnPtr& lhs, size_t lhs_row, const ColumnPtr& rhs, size_t rhs_row) { - if (lhs->is_nullable() && rhs->is_nullable()) { - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; - } + const IColumn* lhs_data = lhs.get(); + const IColumn* rhs_data = rhs.get(); if (lhs->is_nullable()) { const auto& nullable_lhs = assert_cast(*lhs); - return !nullable_lhs.is_null_at(lhs_row) && - nullable_lhs.get_nested_column().compare_at(lhs_row, rhs_row, *rhs, -1) == 0; + if (nullable_lhs.is_null_at(lhs_row)) { + return rhs->is_nullable() && + assert_cast(*rhs).is_null_at(rhs_row); + } + lhs_data = &nullable_lhs.get_nested_column(); } if (rhs->is_nullable()) { const auto& nullable_rhs = assert_cast(*rhs); - return !nullable_rhs.is_null_at(rhs_row) && - lhs->compare_at(lhs_row, rhs_row, nullable_rhs.get_nested_column(), -1) == 0; + if (nullable_rhs.is_null_at(rhs_row)) { + return false; + } + rhs_data = &nullable_rhs.get_nested_column(); + } + const bool lhs_binary = check_and_get_column(*lhs_data) != nullptr || + check_and_get_column(*lhs_data) != nullptr; + const bool rhs_binary = check_and_get_column(*rhs_data) != nullptr || + check_and_get_column(*rhs_data) != nullptr; + if (lhs_binary && rhs_binary) { + // Iceberg schema evolution may represent the same byte key as STRING in one file and + // VARBINARY in another. Equality-delete semantics compare bytes, not column storage classes. + return lhs_data->get_data_at(lhs_row) == rhs_data->get_data_at(rhs_row); + } + return lhs_data->compare_at(lhs_row, rhs_row, *rhs_data, -1) == 0; +} + +void update_varbinary_hashes(const ColumnWithTypeAndName& entry, uint64_t* hashes) { + const IColumn* data = entry.column.get(); + const uint8_t* null_map = nullptr; + if (entry.column->is_nullable()) { + const auto& nullable = assert_cast(*entry.column); + data = &nullable.get_nested_column(); + null_map = nullable.get_null_map_data().data(); + } + for (size_t row = 0; row < entry.column->size(); ++row) { + if (null_map != nullptr && null_map[row] != 0) { + hashes[row] = HashUtil::xxHash64NullWithSeed(hashes[row]); + continue; + } + const auto bytes = data->get_data_at(row); + hashes[row] = HashUtil::xxHash64WithSeed(bytes.data, bytes.size, hashes[row]); } - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; } } // namespace @@ -155,8 +190,14 @@ ColumnPtr EqualityDeletePredicate::_evaluate_key_block(const Block& data_key_blo std::vector EqualityDeletePredicate::_build_hashes(const Block& block) { std::vector hashes(block.rows(), 0); - for (const auto& column : block.get_columns()) { - column->update_hashes_with_value(hashes.data(), nullptr); + for (const auto& entry : block) { + if (remove_nullable(entry.type)->get_primitive_type() == TYPE_VARBINARY) { + // ColumnVarbinary intentionally lacks the generic column hash hook. Keep the V2 delete + // hash byte-identical to ColumnString so schema-mapped binary keys share hash buckets. + update_varbinary_hashes(entry, hashes.data()); + } else { + entry.column->update_hashes_with_value(hashes.data(), nullptr); + } } return hashes; } diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp index b2603a17958eae..9bbf7c3a8fc066 100644 --- a/be/src/format_v2/file_reader.cpp +++ b/be/src/format_v2/file_reader.cpp @@ -53,6 +53,10 @@ std::string FileScanRequest::debug_string() const { << join_debug_strings( non_predicate_columns, [](const LocalColumnIndex& projection) { return projection.debug_string(); }) + << ", predicate_only_columns=" + << join_debug_strings( + predicate_only_columns, + [](LocalColumnId column_id) { return std::to_string(column_id.value()); }) << ", local_positions={"; size_t position_idx = 0; for (const auto& [column_id, block_position] : local_positions) { diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 96d940702486b4..5f959c3e672dcd 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -70,6 +70,10 @@ struct FileScanRequest { // Columns read after row-level filtering. Predicate columns are also available for output and // should not be duplicated here. std::vector non_predicate_columns; + // Predicate columns introduced only to evaluate hidden filter slots. Their values are dead + // after all file-local predicates run, although the shared file block still needs row-shaped + // placeholders until TableReader finalizes projected columns. + std::vector predicate_only_columns; // file-local column id -> file-local output block position. std::map local_positions; // Row-level filters converted to file-local expressions from table-level predicates. @@ -89,6 +93,10 @@ struct FileScanRequest { return std::ranges::find(count_star_placeholder_columns, column_id) != count_star_placeholder_columns.end(); } + + bool is_predicate_only(LocalColumnId column_id) const { + return std::ranges::find(predicate_only_columns, column_id) != predicate_only_columns.end(); + } }; // Helper for constructing the scan-column layout in FileScanRequest. diff --git a/be/src/format_v2/jni/jdbc_reader.cpp b/be/src/format_v2/jni/jdbc_reader.cpp index e0391f3a13a8f0..7d28134db4d7e3 100644 --- a/be/src/format_v2/jni/jdbc_reader.cpp +++ b/be/src/format_v2/jni/jdbc_reader.cpp @@ -39,12 +39,19 @@ std::string JdbcJniReader::connector_class() const { } Status JdbcJniReader::prepare_split(const format::SplitReadOptions& options) { - _jdbc_params.clear(); - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.table_format_type == "jdbc") { - _jdbc_params = std::map( - options.current_range.table_format_params.jdbc_params.begin(), - options.current_range.table_format_params.jdbc_params.end()); + { + // End these scopes before JniTableReader enters the same counters; nested use would count + // this JDBC parameter preparation twice instead of extending the common lifecycle total. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + _jdbc_params.clear(); + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.table_format_type == "jdbc") { + _jdbc_params = std::map( + options.current_range.table_format_params.jdbc_params.begin(), + options.current_range.table_format_params.jdbc_params.end()); + } } return format::JniTableReader::prepare_split(options); } diff --git a/be/src/format_v2/jni/jni_table_reader.cpp b/be/src/format_v2/jni/jni_table_reader.cpp index 3cb105193b56ae..7658a52fc86e0f 100644 --- a/be/src/format_v2/jni/jni_table_reader.cpp +++ b/be/src/format_v2/jni/jni_table_reader.cpp @@ -24,6 +24,7 @@ #include "core/block/block.h" #include "exprs/vexpr_context.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/string_util.h" @@ -31,17 +32,31 @@ namespace doris::format { Status JniTableReader::init(TableReadOptions&& options) { RETURN_IF_ERROR(TableReader::init(std::move(options))); - _init_profile(); + { + // Base and derived scopes must not overlap on the same counter: RuntimeProfile timers add + // deltas, so nested use would double-count instead of extending lifecycle coverage. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + _init_profile(); + } + SCOPED_TIMER(_connector_total_time); return Status::OK(); } Status JniTableReader::prepare_split(const SplitReadOptions& options) { - // EOF belongs to the previous split. Keep it set after closing that split so repeated reads - // are idempotent, and clear it only when a new split is explicitly prepared. - _eof = false; - _current_range = options.current_range; - RETURN_IF_ERROR(validate_scan_range(options.current_range)); + SCOPED_TIMER(_connector_total_time); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + // EOF belongs to the previous split. Keep it set after closing that split so repeated reads + // are idempotent, and clear it only when a new split is explicitly prepared. + _eof = false; + _current_range = options.current_range; + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + } RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -63,6 +78,9 @@ Status JniTableReader::prepare_split(const SplitReadOptions& options) { } Status JniTableReader::get_block(Block* output_block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.exec_timer); + SCOPED_TIMER(_connector_total_time); DORIS_CHECK(output_block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(output_block->columns() == _projected_columns.size()); @@ -109,7 +127,11 @@ Status JniTableReader::get_block(Block* output_block, bool* eos) { } Status JniTableReader::abort_split() { - RETURN_IF_ERROR(_close_jni_scanner()); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + RETURN_IF_ERROR(_close_jni_scanner()); + } return TableReader::abort_split(); } @@ -342,10 +364,16 @@ void JniTableReader::_publish_split_profile(JNIEnv* env) { } Status JniTableReader::close() { + SCOPED_TIMER(_connector_total_time); if (_closed) { return Status::OK(); } - auto close_status = _close_jni_scanner(); + Status close_status; + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + close_status = _close_jni_scanner(); + } auto table_status = TableReader::close(); if (close_status.ok() && !table_status.ok()) { close_status = std::move(table_status); @@ -535,7 +563,9 @@ void JniTableReader::_init_profile() { return; } const auto connector_name = _connector_name(); - ADD_TIMER(_scanner_profile, connector_name); + file_scan_profile::ensure_hierarchy(_scanner_profile); + _connector_total_time = + ADD_CHILD_TIMER(_scanner_profile, connector_name, file_scan_profile::TABLE_READER); _open_scanner_time = ADD_CHILD_TIMER(_scanner_profile, "OpenScannerTime", connector_name); _java_scan_time = ADD_CHILD_TIMER(_scanner_profile, "JavaScanTime", connector_name); _java_append_data_time = diff --git a/be/src/format_v2/jni/jni_table_reader.h b/be/src/format_v2/jni/jni_table_reader.h index 24f84ca8d62756..5e48270515f996 100644 --- a/be/src/format_v2/jni/jni_table_reader.h +++ b/be/src/format_v2/jni/jni_table_reader.h @@ -84,6 +84,7 @@ class JniTableReader : public TableReader { virtual Status _open_jni_scanner(); bool _reserve_split_profile_publication(); const std::vector& jni_columns() const { return _jni_columns; } + RuntimeProfile::Counter* connector_total_timer() const { return _connector_total_time; } TFileRangeDesc _current_range; private: @@ -110,6 +111,7 @@ class JniTableReader : public TableReader { bool _eof = false; bool _split_profile_published = false; + RuntimeProfile::Counter* _connector_total_time = nullptr; RuntimeProfile::Counter* _open_scanner_time = nullptr; RuntimeProfile::Counter* _java_scan_time = nullptr; RuntimeProfile::Counter* _java_append_data_time = nullptr; diff --git a/be/src/format_v2/jni/trino_connector_jni_reader.cpp b/be/src/format_v2/jni/trino_connector_jni_reader.cpp index 11c9945c5dea16..4f0a3f55c3891e 100644 --- a/be/src/format_v2/jni/trino_connector_jni_reader.cpp +++ b/be/src/format_v2/jni/trino_connector_jni_reader.cpp @@ -84,8 +84,15 @@ Status TrinoConnectorJniReader::validate_scan_range(const TFileRangeDesc& range) } Status TrinoConnectorJniReader::prepare_split(const format::SplitReadOptions& options) { - RETURN_IF_ERROR(validate_scan_range(options.current_range)); - RETURN_IF_ERROR(_set_spi_plugins_dir()); + { + // Plugin discovery can dominate a cold split. Use non-overlapping common scopes because + // the JNI base method subsequently enters the same RuntimeProfile counters. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + RETURN_IF_ERROR(_set_spi_plugins_dir()); + } return format::JniTableReader::prepare_split(options); } diff --git a/be/src/format_v2/json/json_reader.cpp b/be/src/format_v2/json/json_reader.cpp index 313115e82ce744..1f42a2cd2d5e94 100644 --- a/be/src/format_v2/json/json_reader.cpp +++ b/be/src/format_v2/json/json_reader.cpp @@ -47,6 +47,7 @@ #include "io/fs/stream_load_pipe.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/slice.h" @@ -174,7 +175,26 @@ JsonReader::~JsonReader() { static_cast(close()); } +void JsonReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* json_profile = "JsonReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, json_profile, file_scan_profile::FILE_READER, 1); + _open_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonOpenTime", json_profile, 1); + _read_document_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonReadDocumentTime", json_profile, 1); + _parse_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonParseTime", json_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonMaterializeTime", json_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonFilterTime", json_profile, 1); +} + Status JsonReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("JSON v2 reader requires scan params"); @@ -230,6 +250,7 @@ Status JsonReader::init(RuntimeState* state) { } Status JsonReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("JSON v2 file_schema is null"); } @@ -243,6 +264,8 @@ std::unique_ptr JsonReader::create_column_mapper( } Status JsonReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); @@ -269,6 +292,7 @@ Status JsonReader::open(std::shared_ptr request) { } Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -297,7 +321,10 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { bool is_empty_row = false; Status st = Status::OK(); try { - st = _parse_next_json(&size, &_reader_eof); + { + SCOPED_TIMER(_parse_time); + st = _parse_next_json(&size, &_reader_eof); + } if (st.ok() && !_reader_eof) { if (size == 0) { is_empty_row = true; @@ -306,6 +333,7 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { } } if (st.ok() && !_reader_eof && !is_empty_row) { + SCOPED_TIMER(_materialize_time); st = _append_rows_from_current_value(file_block, &is_empty_row, &_reader_eof); } } catch (simdjson::simdjson_error& e) { @@ -324,13 +352,17 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } *eof = _reader_eof && *rows == 0; _eof = *eof; return Status::OK(); } Status JsonReader::close() { + SCOPED_TIMER(_total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); @@ -489,6 +521,7 @@ Status JsonReader::_parse_jsonpath_and_json_root() { } Status JsonReader::_read_one_document(size_t* size, bool* eof) { + SCOPED_TIMER(_read_document_time); DORIS_CHECK(size != nullptr); DORIS_CHECK(eof != nullptr); *size = 0; diff --git a/be/src/format_v2/json/json_reader.h b/be/src/format_v2/json/json_reader.h index 52cdfad6728d64..de3c084e681f23 100644 --- a/be/src/format_v2/json/json_reader.h +++ b/be/src/format_v2/json/json_reader.h @@ -75,6 +75,7 @@ class JsonReader final : public FileReader { Status close() override; private: + void _init_profile() override; // A requested column keeps both identities: // - `source_index`: index in FE file slots, used for jsonpaths and SerDe lookup. // - `block_position`: index in the caller's output block, used for materialization. @@ -137,6 +138,13 @@ class JsonReader final : public FileReader { std::unordered_map _slot_name_to_index; std::vector _previous_positions; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_time = nullptr; + RuntimeProfile::Counter* _read_document_time = nullptr; + RuntimeProfile::Counter* _parse_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + io::FileReaderSPtr _physical_file_reader; std::unique_ptr _decompressor; std::unique_ptr _line_reader; diff --git a/be/src/format_v2/native/native_reader.cpp b/be/src/format_v2/native/native_reader.cpp index 5d0984084a6d41..3d5429e0a9a262 100644 --- a/be/src/format_v2/native/native_reader.cpp +++ b/be/src/format_v2/native/native_reader.cpp @@ -29,6 +29,7 @@ #include "format_v2/materialized_reader_util.h" #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/slice.h" @@ -54,7 +55,26 @@ NativeReader::~NativeReader() { static_cast(close()); } +void NativeReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* native_profile = "NativeReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, native_profile, file_scan_profile::FILE_READER, 1); + _read_block_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeReadBlockTime", native_profile, 1); + _deserialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeDeserializeTime", native_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeMaterializeTime", native_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeFilterTime", native_profile, 1); +} + Status NativeReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_file_description == nullptr) { return Status::InvalidArgument("Native v2 reader requires file description"); @@ -65,6 +85,7 @@ Status NativeReader::init(RuntimeState* state) { } Status NativeReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("Native v2 file_schema is null"); } @@ -79,6 +100,7 @@ std::unique_ptr NativeReader::create_column_mapper( } Status NativeReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); _first_block_consumed = false; @@ -88,6 +110,7 @@ Status NativeReader::open(std::shared_ptr request) { } Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -108,6 +131,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { if (_first_block_loaded && !_first_block_consumed) { buffer = _first_block_buffer; } else { + SCOPED_TIMER(_read_block_time); RETURN_IF_ERROR(_read_next_pblock(&buffer, &local_eof)); } @@ -131,11 +155,20 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { Block source_block; size_t uncompressed_bytes = 0; int64_t decompress_time = 0; - RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); - RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + { + SCOPED_TIMER(_deserialize_time); + RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); + } + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + } *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } if (_first_block_loaded && !_first_block_consumed) { _first_block_consumed = true; @@ -149,6 +182,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { } Status NativeReader::close() { + SCOPED_TIMER(_total_time); _file_reader.reset(); _tracing_file_reader.reset(); _request.reset(); diff --git a/be/src/format_v2/native/native_reader.h b/be/src/format_v2/native/native_reader.h index 3719a6afd6c4f5..15a52fe6f8bc87 100644 --- a/be/src/format_v2/native/native_reader.h +++ b/be/src/format_v2/native/native_reader.h @@ -49,6 +49,7 @@ class NativeReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _validate_and_consume_header(); Status _ensure_schema_loaded() const; Status _read_next_pblock(std::string* buffer, bool* eof) const; @@ -65,6 +66,11 @@ class NativeReader final : public FileReader { mutable std::string _first_block_buffer; mutable bool _first_block_loaded = false; mutable bool _first_block_consumed = false; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _read_block_time = nullptr; + RuntimeProfile::Counter* _deserialize_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; }; } // namespace doris::format::native diff --git a/be/src/format_v2/orc/orc_file_input_stream.cpp b/be/src/format_v2/orc/orc_file_input_stream.cpp index a5144d7b2017b0..df0927aac3444a 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.cpp +++ b/be/src/format_v2/orc/orc_file_input_stream.cpp @@ -27,6 +27,7 @@ #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" #include "orc/Exceptions.hh" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "util/slice.h" @@ -54,24 +55,28 @@ class OrcMergedRangeFileReader final : public io::FileReader { _size(_file_reader->size()) { _statistics.apply_bytes += _range.end_offset - _range.start_offset; if (_profile != nullptr) { - const char* profile_name = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, profile_name, 1); - _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", profile_name, 1); - _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", profile_name, 1); - _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, + const char* profile_name = "OrcMergedSmallIO"; + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); + // RuntimeProfile counter lookup is flat, so every child must be format-qualified; + // a unique parent alone cannot prevent aliasing with Parquet's MergeRange reader. + _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedCopyTime", profile_name, 1); + _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedReadTime", profile_name, 1); + _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestIO", TUnit::UNIT, profile_name, 1); - _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedIO", TUnit::UNIT, + _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedIO", TUnit::UNIT, profile_name, 1); - _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, - profile_name, 1); - _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedBytes", TUnit::BYTES, + _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestBytes", + TUnit::BYTES, profile_name, 1); + _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedBytes", TUnit::BYTES, profile_name, 1); - _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ApplyBytes", TUnit::BYTES, - profile_name, 1); - _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OverReadBytes", TUnit::BYTES, - profile_name, 1); - _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ClusterNum", TUnit::UNIT, - profile_name, 1); + _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedApplyBytes", + TUnit::BYTES, profile_name, 1); + _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedOverReadBytes", + TUnit::BYTES, profile_name, 1); + _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedClusterNum", + TUnit::UNIT, profile_name, 1); } } @@ -113,6 +118,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { if (_profile == nullptr) { return; } + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -165,6 +171,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { OrcMergedRangeStatistics _statistics; RuntimeProfile::Counter* _copy_time = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; RuntimeProfile::Counter* _merged_io = nullptr; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 9a2aeb538977dc..49d878257aa0df 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -76,6 +76,7 @@ #include "format_v2/timestamp_statistics.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/segment/condition_cache.h" @@ -776,7 +777,9 @@ void OrcReader::_init_profile() { } static const char* orc_profile = "OrcReader"; - ADD_TIMER_WITH_LEVEL(_profile, orc_profile, 1); + file_scan_profile::ensure_hierarchy(_profile); + _orc_profile.total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, orc_profile, file_scan_profile::FILE_READER, 1); _orc_profile.reader_call = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReaderCall", TUnit::UNIT, orc_profile, 1); _orc_profile.reader_inclusive_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -803,20 +806,22 @@ void OrcReader::_init_profile() { _profile, "EvaluatedRowGroupCount", TUnit::UNIT, orc_profile, 1); _orc_profile.read_row_count = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReadRowCount", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsFiltered", - TUnit::UNIT, orc_profile, 1); + // RuntimeProfile counter names are flat; format-qualified names keep ORC ownership stable when + // one scan profile also initializes Parquet counters in either order. + _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "OrcRowGroupsFiltered", TUnit::UNIT, orc_profile, 1); _orc_profile.filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); - _orc_profile.read_row_groups = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsReadNum", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredRowsByGroup", - TUnit::UNIT, orc_profile, 1); + _profile, "OrcRowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); + _orc_profile.read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcRowGroupsReadNum", + TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "OrcFilteredRowsByGroup", TUnit::UNIT, orc_profile, 1); _orc_profile.lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "FilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_bytes = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredBytes", TUnit::BYTES, orc_profile, 1); + _profile, "OrcFilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFilteredBytes", + TUnit::BYTES, orc_profile, 1); _orc_profile.open_file_num = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, orc_profile, 1); + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFileNum", TUnit::UNIT, orc_profile, 1); } void OrcReader::_collect_profile() const { @@ -874,6 +879,8 @@ format::ColumnDefinition OrcReader::row_position_column_definition() { } Status OrcReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_orc_profile.total_time); RETURN_IF_ERROR(format::FileReader::init(state)); _state = std::make_unique(); TimezoneUtils::find_cctz_time_zone(_state->timezone, _state->timezone_obj); @@ -1162,6 +1169,7 @@ Status OrcReader::_fill_map_schema_children(const ::orc::Type& type, } Status OrcReader::get_schema(std::vector* const file_schema) const { + SCOPED_TIMER(_orc_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } @@ -1195,6 +1203,7 @@ std::unique_ptr OrcReader::create_column_mapper( } Status OrcReader::open(std::shared_ptr request) { + SCOPED_TIMER(_orc_profile.total_time); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); } @@ -1877,6 +1886,7 @@ Status OrcReader::_decode_column(const ::orc::Type& file_type, const ::orc::Type } Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -2000,6 +2010,7 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { // NOLINTNEXTLINE(readability-function-size) Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(result != nullptr); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); @@ -2455,6 +2466,7 @@ void OrcReader::_filter_requested_columns(Block* file_block, const IColumn::Filt } Status OrcReader::close() { + SCOPED_TIMER(_orc_profile.total_time); _collect_profile(); if (_state != nullptr) { _state = std::make_unique(); diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index adba20fbce0b10..a67dc5ad7f5e3e 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -72,6 +72,7 @@ class OrcReader final : public format::FileReader { private: struct OrcProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* reader_call = nullptr; // ReaderCall RuntimeProfile::Counter* reader_inclusive_latency_us = nullptr; // ReaderInclusiveLatencyUs RuntimeProfile::Counter* decompression_call = nullptr; // DecompressionCall diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp new file mode 100644 index 00000000000000..1b824a4efc8020 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -0,0 +1,825 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_desc.h" + +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/exception.h" +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "util/slice.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +static bool is_group_node(const tparquet::SchemaElement& schema) { + return schema.num_children > 0; +} + +static bool is_list_node(const tparquet::SchemaElement& schema) { + return schema.__isset.converted_type && schema.converted_type == tparquet::ConvertedType::LIST; +} + +static bool is_map_node(const tparquet::SchemaElement& schema) { + return schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::MAP || + schema.converted_type == tparquet::ConvertedType::MAP_KEY_VALUE); +} + +static bool is_repeated_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REPEATED; +} + +static bool is_required_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REQUIRED; +} + +static bool is_optional_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::OPTIONAL; +} + +static int num_children_node(const tparquet::SchemaElement& schema) { + return schema.__isset.num_children ? schema.num_children : 0; +} + +static Status validate_native_schema_structure( + const std::vector& schemas) { + if (schemas.empty()) { + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + const auto& root = schemas[0]; + if (root.__isset.type || !root.__isset.num_children || root.num_children <= 0 || + (root.__isset.repetition_type && + root.repetition_type != tparquet::FieldRepetitionType::REQUIRED)) { + // Writers may encode the root's implicit REQUIRED repetition explicitly, but an optional + // or repeated root would make every descendant's definition/repetition levels ambiguous. + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + struct PendingGroup { + size_t remaining_children; + size_t depth; + }; + const auto root_children = num_children_node(schemas[0]); + if (root_children < 0 || static_cast(root_children) > schemas.size() - 1) { + return Status::InvalidArgument("Invalid parquet root child count {}", root_children); + } + std::vector pending {{static_cast(root_children), 0}}; + for (size_t pos = 1; pos < schemas.size(); ++pos) { + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (pending.empty()) { + return Status::InvalidArgument("Schema element {} is not reachable from the root", pos); + } + + const size_t depth = pending.back().depth + 1; + --pending.back().remaining_children; + const auto& schema = schemas[pos]; + if (!schema.__isset.repetition_type) { + return Status::InvalidArgument("Schema element {} has no repetition type", pos); + } + const bool has_children = schema.__isset.num_children && schema.num_children > 0; + if (schema.__isset.type == has_children) { + // Some legacy parquet-cpp files explicitly encode num_children=0 on primitive nodes. + // Only a positive count denotes a group, preserving strict rejection of real dual-kind + // nodes without dropping those otherwise valid files. + return Status::InvalidArgument("Schema element {} has ambiguous primitive/group kind", + pos); + } + if (schema.__isset.num_children && schema.num_children < 0) { + return Status::InvalidArgument("Schema element {} has a negative child count", pos); + } + const int children = num_children_node(schemas[pos]); + if (children < 0 || static_cast(children) > schemas.size() - pos - 1) { + return Status::InvalidArgument("Invalid child count {} at schema element {}", children, + pos); + } + if (children > 0) { + // Bound the tree before any resize or recursion so an untrusted footer cannot + // turn a tiny schema into an unbounded allocation or parser stack. + if (depth > MAX_NATIVE_SCHEMA_DEPTH) { + return Status::InvalidArgument("Parquet schema depth {} exceeds limit {}", depth, + MAX_NATIVE_SCHEMA_DEPTH); + } + pending.push_back({static_cast(children), depth}); + } + } + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (!pending.empty()) { + return Status::InvalidArgument("Parquet schema ended before all children were parsed"); + } + return Status::OK(); +} + +/** + * `repeated_parent_def_level` is the definition level of the first ancestor node whose repetition_type equals REPEATED. + * Empty array/map values are not stored in doris columns, so have to use `repeated_parent_def_level` to skip the + * empty or null values in ancestor node. + * + * For instance, considering an array of strings with 3 rows like the following: + * null, [], [a, b, c] + * We can store four elements in data column: null, a, b, c + * and the offsets column is: 1, 1, 4 + * and the null map is: 1, 0, 0 + * For the i-th row in array column: range from `offsets[i - 1]` until `offsets[i]` represents the elements in this row, + * so we can't store empty array/map values in doris data column. + * As a comparison, spark does not require `repeated_parent_def_level`, + * because the spark column stores empty array/map values , and use anther length column to indicate empty values. + * Please reference: https://github.com/apache/spark/blob/master/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetColumnVector.java + * + * Furthermore, we can also avoid store null array/map values in doris data column. + * The same three rows as above, We can only store three elements in data column: a, b, c + * and the offsets column is: 0, 0, 3 + * and the null map is: 1, 0, 0 + * + * Inherit the repetition and definition level from parent node, if the parent node is repeated, + * we should set repeated_parent_def_level = definition_level, otherwise as repeated_parent_def_level. + * @param parent parent node + * @param repeated_parent_def_level the first ancestor node whose repetition_type equals REPEATED + */ +static void set_child_node_level(NativeFieldSchema* parent, int16_t repeated_parent_def_level) { + for (auto& child : parent->children) { + child.repetition_level = parent->repetition_level; + child.definition_level = parent->definition_level; + child.repeated_parent_def_level = repeated_parent_def_level; + } +} + +static bool is_struct_list_node(const tparquet::SchemaElement& schema, + const std::string& enclosing_list_name) { + // The legacy Parquet exception is exact: accepting every "*_tuple" wrapper changes a standard + // one-child LIST wrapper from ARRAY to ARRAY>. + return schema.name == "array" || schema.name == enclosing_list_name + "_tuple"; +} + +static bool has_logical_annotation(const tparquet::SchemaElement& schema) { + return schema.__isset.logicalType || schema.__isset.converted_type; +} + +std::string NativeFieldSchema::debug_string() const { + std::stringstream ss; + ss << "NativeFieldSchema(name=" << name << ", R=" << repetition_level + << ", D=" << definition_level; + if (children.size() > 0) { + ss << ", type=" << data_type->get_name() << ", children=["; + for (int i = 0; i < children.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << children[i].debug_string(); + } + ss << "]"; + } else { + ss << ", physical_type=" << physical_type; + ss << " , doris_type=" << data_type->get_name(); + } + ss << ")"; + return ss.str(); +} + +Status NativeFieldDescriptor::parse_from_thrift( + const std::vector& t_schemas) { + _fields.clear(); + _physical_fields.clear(); + _name_to_field.clear(); + RETURN_IF_ERROR(validate_native_schema_structure(t_schemas)); + const auto& root_schema = t_schemas[0]; + _fields.resize(root_schema.num_children); + _next_schema_pos = 1; + + for (int i = 0; i < root_schema.num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &_fields[i])); + if (_name_to_field.find(_fields[i].name) != _name_to_field.end()) { + return Status::InvalidArgument("Duplicated field name: {}", _fields[i].name); + } + _name_to_field.emplace(_fields[i].name, &_fields[i]); + } + + if (_next_schema_pos != t_schemas.size()) { + return Status::InvalidArgument("Remaining {} unparsed schema elements", + t_schemas.size() - _next_schema_pos); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_node_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field) { + if (curr_pos >= t_schemas.size()) { + return Status::InvalidArgument("Out-of-bounds index of schema elements"); + } + auto& t_schema = t_schemas[curr_pos]; + if (is_group_node(t_schema)) { + // nested structure or nullable list + return parse_group_field(t_schemas, curr_pos, node_field); + } + if (is_repeated_node(t_schema)) { + // repeated (LIST) + // produce required list + node_field->repetition_level++; + node_field->definition_level++; + node_field->children.resize(1); + set_child_node_level(node_field, node_field->definition_level); + auto child = &node_field->children[0]; + parse_physical_field(t_schema, false, child); + + node_field->name = t_schema.name; + node_field->lower_case_name = to_lower(t_schema.name); + node_field->data_type = std::make_shared(make_nullable(child->data_type)); + _next_schema_pos = curr_pos + 1; + node_field->field_id = t_schema.__isset.field_id ? t_schema.field_id : -1; + } else { + bool is_optional = is_optional_node(t_schema); + if (is_optional) { + node_field->definition_level++; + } + parse_physical_field(t_schema, is_optional, node_field); + _next_schema_pos = curr_pos + 1; + } + return Status::OK(); +} + +void NativeFieldDescriptor::parse_physical_field(const tparquet::SchemaElement& physical_schema, + bool is_nullable, + NativeFieldSchema* physical_field) { + physical_field->name = physical_schema.name; + physical_field->lower_case_name = to_lower(physical_field->name); + physical_field->parquet_schema = physical_schema; + physical_field->physical_type = physical_schema.type; + physical_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + _physical_fields.push_back(physical_field); + physical_field->physical_column_index = cast_set(_physical_fields.size() - 1); + auto type = get_doris_type(physical_schema, is_nullable); + physical_field->data_type = type.first; + physical_field->is_type_compatibility = type.second; + physical_field->field_id = physical_schema.__isset.field_id ? physical_schema.field_id : -1; +} + +std::pair NativeFieldDescriptor::get_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + try { + if (physical_schema.__isset.logicalType) { + ans = convert_to_doris_type(physical_schema.logicalType, nullable); + } else if (physical_schema.__isset.converted_type) { + ans = convert_to_doris_type(physical_schema, nullable); + } + } catch (...) { + // now the Not supported exception are ignored + // so those byte_array maybe be treated as varbinary(now) : string(before) + } + if (ans.first->get_primitive_type() == PrimitiveType::INVALID_TYPE) { + switch (physical_schema.type) { + case tparquet::Type::BOOLEAN: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BOOLEAN, nullable); + break; + case tparquet::Type::INT32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::Type::INT64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::Type::INT96: + if (_enable_mapping_timestamp_tz) { + // treat INT96 as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMPTZ, nullable, + 0, 6); + } else { + // in most cases, it's a nano timestamp + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, + 0, 6); + } + break; + case tparquet::Type::FLOAT: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + break; + case tparquet::Type::DOUBLE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, nullable); + break; + case tparquet::Type::BYTE_ARRAY: + if (_enable_mapping_varbinary) { + // if physical_schema not set logicalType and converted_type, + // we treat BYTE_ARRAY as VARBINARY by default, so that we can read all data directly. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet logicalType{}", + physical_schema.type)); + break; + } + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + tparquet::LogicalType logicalType, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + if (logicalType.__isset.STRING || logicalType.__isset.ENUM || logicalType.__isset.JSON || + logicalType.__isset.BSON) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } else if (logicalType.__isset.DECIMAL) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL128I, nullable, + logicalType.DECIMAL.precision, + logicalType.DECIMAL.scale); + } else if (logicalType.__isset.DATE) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + } else if (logicalType.__isset.INTEGER) { + if (logicalType.INTEGER.isSigned) { + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } + } else { + is_type_compatibility = true; + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + } + } + } else if (logicalType.__isset.TIME) { + const int scale = logicalType.TIME.unit.__isset.MILLIS ? 3 : 6; + // TIME stores an integer unit, so its Doris scale must preserve the footer unit or + // sub-second values are silently truncated by the target SerDe. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, scale); + } else if (logicalType.__isset.TIMESTAMP) { + if (_enable_mapping_timestamp_tz) { + if (logicalType.TIMESTAMP.isAdjustedToUTC) { + // treat TIMESTAMP with isAdjustedToUTC as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_TIMESTAMPTZ, nullable, 0, + logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + return ans; + } + } + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + } else if (logicalType.__isset.UUID) { + if (_enable_mapping_varbinary) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable, -1, + -1, 16); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + } else if (logicalType.__isset.FLOAT16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + } else { + throw Exception(Status::InternalError("Not supported parquet logicalType")); + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + switch (physical_schema.converted_type) { + case tparquet::ConvertedType::type::UTF8: + case tparquet::ConvertedType::type::ENUM: + case tparquet::ConvertedType::type::JSON: + case tparquet::ConvertedType::type::BSON: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + case tparquet::ConvertedType::type::DECIMAL: + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DECIMAL128I, nullable, physical_schema.precision, physical_schema.scale); + break; + case tparquet::ConvertedType::type::DATE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + break; + case tparquet::ConvertedType::type::TIME_MILLIS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 3); + break; + case tparquet::ConvertedType::type::TIME_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 6); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MILLIS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 3); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 6); + break; + case tparquet::ConvertedType::type::INT_8: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_8: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_16: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_16: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::ConvertedType::type::UINT_32: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_64: + is_type_compatibility = true; + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet ConvertedType: {}", + physical_schema.converted_type)); + } + return ans; +} + +Status NativeFieldDescriptor::parse_group_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field) { + auto& group_schema = t_schemas[curr_pos]; + if ((group_schema.__isset.logicalType && group_schema.logicalType.__isset.ENUM) || + (group_schema.__isset.converted_type && + group_schema.converted_type == tparquet::ConvertedType::ENUM)) { + // ENUM describes primitive bytes only. Rejecting it before recursive group parsing keeps + // the native metadata tree from silently accepting a schema the Parquet contract forbids. + return Status::InvalidArgument("Logical type Enum cannot be applied to group node"); + } + if (is_map_node(group_schema)) { + // the map definition: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + return parse_map_field(t_schemas, curr_pos, group_field); + } + if (is_list_node(group_schema)) { + // the list definition: + // optional group (LIST) { + // repeated group [bag | list] { // hive or spark + // optional [array_element | element]; // hive or spark + // } + // } + return parse_list_field(t_schemas, curr_pos, group_field); + } + + if (is_repeated_node(group_schema)) { + group_field->repetition_level++; + group_field->definition_level++; + group_field->children.resize(1); + set_child_node_level(group_field, group_field->definition_level); + auto struct_field = &group_field->children[0]; + // the list of struct: + // repeated group (LIST) { + // optional/required ; + // ... + // } + // produce a non-null list + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, struct_field)); + + group_field->name = group_schema.name; + group_field->lower_case_name = to_lower(group_field->name); + group_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + group_field->data_type = + std::make_shared(make_nullable(struct_field->data_type)); + group_field->field_id = group_schema.__isset.field_id ? group_schema.field_id : -1; + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, group_field)); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_list_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field, bool repeated_node_is_enclosing_list_element) { + // the list definition: + // spark and hive have three level schemas but with different schema name + // spark: - "list" - "element" + // hive: - "bag" - "array_element" + // parse three level schemas to two level primitive like: LIST, + // or nested structure like: LIST> + auto& first_level = t_schemas[curr_pos]; + if (first_level.num_children != 1) { + return Status::InvalidArgument("List element should have only one child"); + } + + if (curr_pos + 1 >= t_schemas.size()) { + return Status::InvalidArgument("List element should have the second level schema"); + } + + if (first_level.repetition_type == tparquet::FieldRepetitionType::REPEATED && + !repeated_node_is_enclosing_list_element) { + return Status::InvalidArgument("List element can't be a repeated schema"); + } + + // the repeated schema element + auto& second_level = t_schemas[curr_pos + 1]; + if (second_level.repetition_type != tparquet::FieldRepetitionType::REPEATED) { + return Status::InvalidArgument("The second level of list element should be repeated"); + } + + // This indicates if this list is nullable. + bool is_optional = is_optional_node(first_level); + if (is_optional) { + list_field->definition_level++; + } + list_field->repetition_level++; + list_field->definition_level++; + list_field->children.resize(1); + NativeFieldSchema* list_child = &list_field->children[0]; + + size_t num_children = num_children_node(second_level); + if (num_children > 0) { + const bool structural_wrapper = is_struct_list_node(second_level, first_level.name); + const auto& only_child = t_schemas[curr_pos + 2]; + if (num_children == 1 && !structural_wrapper && has_logical_annotation(second_level)) { + // The repeated node is already the outer LIST element. Preserve its own LIST/MAP + // annotation, but do not interpret its REPEATED marker as another outer array. + set_child_node_level(list_field, list_field->definition_level); + if (is_list_node(second_level)) { + RETURN_IF_ERROR(parse_list_field(t_schemas, curr_pos + 1, list_child, true)); + } else if (is_map_node(second_level)) { + RETURN_IF_ERROR(parse_map_field(t_schemas, curr_pos + 1, list_child, true)); + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 1 && !structural_wrapper && !is_repeated_node(only_child)) { + // optional field, and the third level element is the nested structure in list + // produce nested structure like: LIST, LIST, LIST> + // skip bag/list, it's a repeated element. + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_node_field(t_schemas, curr_pos + 2, list_child)); + } else { + // required field, produce the list of struct + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 0) { + // required two level list, for compatibility reason. + set_child_node_level(list_field, list_field->definition_level); + parse_physical_field(second_level, false, list_child); + _next_schema_pos = curr_pos + 2; + } + + list_field->name = first_level.name; + list_field->lower_case_name = to_lower(first_level.name); + list_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + list_field->data_type = + std::make_shared(make_nullable(list_field->children[0].data_type)); + if (is_optional) { + list_field->data_type = make_nullable(list_field->data_type); + } + list_field->field_id = first_level.__isset.field_id ? first_level.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_map_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element) { + // the map definition in parquet: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + // Map value can be optional, the map without values is a SET + if (curr_pos + 2 >= t_schemas.size()) { + return Status::InvalidArgument("Map element should have at least three levels"); + } + auto& map_schema = t_schemas[curr_pos]; + if (map_schema.num_children != 1) { + return Status::InvalidArgument( + "Map element should have only one child(name='map', type='MAP_KEY_VALUE')"); + } + if (is_repeated_node(map_schema) && !repeated_node_is_enclosing_list_element) { + return Status::InvalidArgument("Map element can't be a repeated schema"); + } + auto& map_key_value = t_schemas[curr_pos + 1]; + if (!is_group_node(map_key_value) || !is_repeated_node(map_key_value)) { + return Status::InvalidArgument( + "the second level in map must be a repeated group(key and value)"); + } + auto& map_key = t_schemas[curr_pos + 2]; + if (!is_required_node(map_key)) { + LOG(WARNING) << "Filed " << map_schema.name << " is map type, but with nullable key column"; + } + + if (map_key_value.num_children == 1) { + // The map with three levels is a SET + return parse_list_field(t_schemas, curr_pos, map_field, + repeated_node_is_enclosing_list_element); + } + if (map_key_value.num_children != 2) { + // A standard map should have four levels + return Status::InvalidArgument( + "the second level in map(MAP_KEY_VALUE) should have two children"); + } + // standard map + bool is_optional = is_optional_node(map_schema); + if (is_optional) { + map_field->definition_level++; + } + map_field->repetition_level++; + map_field->definition_level++; + + // Directly create key and value children instead of intermediate key_value node + map_field->children.resize(2); + // map is a repeated node, we should set the `repeated_parent_def_level` of its children as `definition_level` + set_child_node_level(map_field, map_field->definition_level); + + auto key_field = &map_field->children[0]; + auto value_field = &map_field->children[1]; + + // Parse key and value fields directly from the key_value group's children + _next_schema_pos = curr_pos + 2; // Skip key_value group, go directly to key + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, key_field)); + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, value_field)); + + map_field->name = map_schema.name; + map_field->lower_case_name = to_lower(map_field->name); + map_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + map_field->data_type = std::make_shared(make_nullable(key_field->data_type), + make_nullable(value_field->data_type)); + if (is_optional) { + map_field->data_type = make_nullable(map_field->data_type); + } + map_field->field_id = map_schema.__isset.field_id ? map_schema.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_struct_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* struct_field) { + // the nested column in parquet, parse group to struct. + auto& struct_schema = t_schemas[curr_pos]; + bool is_optional = is_optional_node(struct_schema); + if (is_optional) { + struct_field->definition_level++; + } + auto num_children = struct_schema.num_children; + struct_field->children.resize(num_children); + set_child_node_level(struct_field, struct_field->repeated_parent_def_level); + _next_schema_pos = curr_pos + 1; + for (int i = 0; i < num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &struct_field->children[i])); + } + struct_field->name = struct_schema.name; + struct_field->lower_case_name = to_lower(struct_field->name); + struct_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + + struct_field->field_id = struct_schema.__isset.field_id ? struct_schema.field_id : -1; + DataTypes res_data_types; + std::vector names; + for (int i = 0; i < num_children; ++i) { + res_data_types.push_back(make_nullable(struct_field->children[i].data_type)); + names.push_back(struct_field->children[i].name); + } + struct_field->data_type = std::make_shared(res_data_types, names); + if (is_optional) { + struct_field->data_type = make_nullable(struct_field->data_type); + } + return Status::OK(); +} + +int NativeFieldDescriptor::get_column_index(const std::string& column) const { + for (int32_t i = 0; i < _fields.size(); i++) { + if (_fields[i].name == column) { + return i; + } + } + return -1; +} + +NativeFieldSchema* NativeFieldDescriptor::get_column(const std::string& name) const { + auto it = _name_to_field.find(name); + if (it != _name_to_field.end()) { + return it->second; + } + throw Exception(Status::InternalError("Name {} not found in NativeFieldDescriptor!", name)); + return nullptr; +} + +void NativeFieldDescriptor::get_column_names(std::unordered_set* names) const { + names->clear(); + for (const NativeFieldSchema& f : _fields) { + names->emplace(f.name); + } +} + +std::string NativeFieldDescriptor::debug_string() const { + std::stringstream ss; + ss << "fields=["; + for (int i = 0; i < _fields.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << _fields[i].debug_string(); + } + ss << "]"; + return ss.str(); +} + +void NativeFieldDescriptor::assign_ids() { + uint64_t next_id = 1; + for (auto& field : _fields) { + field.assign_ids(next_id); + } +} + +const NativeFieldSchema* NativeFieldDescriptor::find_column_by_id(uint64_t column_id) const { + for (const auto& field : _fields) { + if (auto result = field.find_column_by_id(column_id)) { + return result; + } + } + return nullptr; +} + +void NativeFieldSchema::assign_ids(uint64_t& next_id) { + column_id = next_id++; + + for (auto& child : children) { + child.assign_ids(next_id); + } + + max_column_id = next_id - 1; +} + +const NativeFieldSchema* NativeFieldSchema::find_column_by_id(uint64_t target_id) const { + if (column_id == target_id) { + return this; + } + + for (const auto& child : children) { + if (auto result = child.find_column_by_id(target_id)) { + return result; + } + } + + return nullptr; +} + +uint64_t NativeFieldSchema::get_column_id() const { + return column_id; +} + +void NativeFieldSchema::set_column_id(uint64_t id) { + column_id = id; +} + +uint64_t NativeFieldSchema::get_max_column_id() const { + return max_column_id; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_desc.h b/be/src/format_v2/parquet/native_schema_desc.h new file mode 100644 index 00000000000000..a0bab48cc9e9c9 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.h @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nothing.h" +#include "util/slice.h" + +namespace doris::format::parquet { + +// Constant for unassigned column IDs +constexpr uint64_t NATIVE_UNASSIGNED_COLUMN_ID = UINT64_MAX; +constexpr size_t MAX_NATIVE_SCHEMA_DEPTH = 100; + +struct NativeFieldSchema { + std::string name; + std::string lower_case_name; // for hms column name case insensitive match + // the referenced parquet schema element + tparquet::SchemaElement parquet_schema; + + // Used to identify whether this field is a nested field. + DataTypePtr data_type; + + // Only valid when this field is a leaf node + tparquet::Type::type physical_type; + // The index order in NativeFieldDescriptor._physical_fields + int physical_column_index = -1; + int16_t definition_level = 0; + int16_t repetition_level = 0; + int16_t repeated_parent_def_level = 0; + std::vector children; + + //For UInt8 -> Int16,UInt16 -> Int32,UInt32 -> Int64,UInt64 -> Int128. + bool is_type_compatibility = false; + + NativeFieldSchema() + : data_type(std::make_shared()), + column_id(NATIVE_UNASSIGNED_COLUMN_ID) {} + ~NativeFieldSchema() = default; + NativeFieldSchema(const NativeFieldSchema& fieldSchema) = default; + std::string debug_string() const; + + int32_t field_id = -1; + uint64_t column_id = NATIVE_UNASSIGNED_COLUMN_ID; + uint64_t max_column_id = 0; // Maximum column ID for this field and its children + + // Column ID assignment and lookup methods + void assign_ids(uint64_t& next_id); + const NativeFieldSchema* find_column_by_id(uint64_t target_id) const; + uint64_t get_column_id() const; + void set_column_id(uint64_t id); + uint64_t get_max_column_id() const; +}; + +// V2 owns this schema tree and parser so footer/schema planning never invokes the V1 reader path. +class NativeFieldDescriptor { +private: + // Only the schema elements at the first level + std::vector _fields; + // The leaf node of schema elements + std::vector _physical_fields; + // Name to _fields, not all schema elements + std::unordered_map _name_to_field; + // Used in from_thrift, marking the next schema position that should be parsed + size_t _next_schema_pos; + // useful for parse_node_field to decide whether to convert byte_array to VARBINARY type + bool _enable_mapping_varbinary = false; + bool _enable_mapping_timestamp_tz = false; + +private: + void parse_physical_field(const tparquet::SchemaElement& physical_schema, bool is_nullable, + NativeFieldSchema* physical_field); + + Status parse_list_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field, + bool repeated_node_is_enclosing_list_element = false); + + Status parse_map_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element = false); + + Status parse_struct_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* struct_field); + + Status parse_group_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field); + + Status parse_node_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field); + + std::pair convert_to_doris_type(tparquet::LogicalType logicalType, + bool nullable); + std::pair convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable); + std::pair get_doris_type(const tparquet::SchemaElement& physical_schema, + bool nullable); + +public: + NativeFieldDescriptor() = default; + ~NativeFieldDescriptor() = default; + + /** + * Parse NativeFieldDescriptor from parquet thrift FileMetaData. + * @param t_schemas list of schema elements + */ + Status parse_from_thrift(const std::vector& t_schemas); + + int get_column_index(const std::string& column) const; + + /** + * Get the column(the first level schema element, maybe nested field) by index. + * @param index Column index in _fields + */ + const NativeFieldSchema* get_column(size_t index) const { return &_fields[index]; } + + /** + * Get the column(the first level schema element, maybe nested field) by name. + * @param name Column name + * @return NativeFieldSchema or nullptr if not exists + */ + NativeFieldSchema* get_column(const std::string& name) const; + + void get_column_names(std::unordered_set* names) const; + + std::string debug_string() const; + + int32_t size() const { return cast_set(_fields.size()); } + + size_t physical_fields_size() const { return _physical_fields.size(); } + + const NativeFieldSchema* get_physical_field(size_t index) const { + return _physical_fields[index]; + } + + const std::vector& get_fields_schema() const { return _fields; } + + /** + * Assign stable column IDs to schema fields. + * + * This uses an ORC-compatible encoding so that the results of + * create_column_ids() are consistent across formats. IDs start from 1 + * and are assigned in a pre-order traversal (parent before children). + * After calling this, each NativeFieldSchema will have column_id and + * max_column_id populated. + */ + void assign_ids(); + + const NativeFieldSchema* find_column_by_id(uint64_t column_id) const; + void set_enable_mapping_varbinary(bool enable) { _enable_mapping_varbinary = enable; } + void set_enable_mapping_timestamp_tz(bool enable) { _enable_mapping_timestamp_tz = enable; } +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.cpp b/be/src/format_v2/parquet/native_schema_node.cpp new file mode 100644 index 00000000000000..7539fb45efb46d --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.cpp @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_node.h" + +#include +#include + +#include "core/assert_cast.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +void NativeStructSchemaNode::add_child(std::string table_name, std::string file_name, + std::shared_ptr node) { + _children.emplace(std::move(table_name), Child {std::move(file_name), std::move(node)}); +} + +void NativeStructSchemaNode::add_missing_child(std::string table_name) { + _children.emplace(std::move(table_name), Child {}); +} + +std::shared_ptr NativeStructSchemaNode::child( + const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? nullptr : it->second.node; +} + +std::string NativeStructSchemaNode::file_child_name(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? std::string {} : it->second.file_name; +} + +bool NativeStructSchemaNode::has_child(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it != _children.end() && it->second.node != nullptr; +} + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result) { + if (projected_type == nullptr || result == nullptr) { + return Status::InvalidArgument("Native Parquet schema mapping input is null"); + } + const auto type = remove_nullable(projected_type); + switch (type->get_primitive_type()) { + case TYPE_STRUCT: { + if (file_schema.kind != ParquetColumnSchemaKind::STRUCT) { + return Status::Corruption("Parquet column {} is not a STRUCT", file_schema.name); + } + const auto* struct_type = assert_cast(type.get()); + std::map file_children; + for (const auto& child : file_schema.children) { + file_children.emplace(to_lower(child->name), child.get()); + } + auto node = std::make_shared(); + for (size_t i = 0; i < struct_type->get_elements().size(); ++i) { + const auto& table_name = struct_type->get_element_name(i); + // Native metadata keeps writer casing. Match normalized names while preserving the + // original file name used to address the physical child reader. + const auto child_it = file_children.find(to_lower(table_name)); + if (child_it == file_children.end()) { + node->add_missing_child(table_name); + continue; + } + std::shared_ptr child_node; + RETURN_IF_ERROR(build_native_schema_node(struct_type->get_element(i), *child_it->second, + &child_node)); + node->add_child(table_name, child_it->second->name, std::move(child_node)); + } + *result = std::move(node); + return Status::OK(); + } + case TYPE_ARRAY: { + if (file_schema.kind != ParquetColumnSchemaKind::LIST || file_schema.children.size() != 1) { + return Status::Corruption("Parquet column {} is not an ARRAY", file_schema.name); + } + const auto* array_type = assert_cast(type.get()); + std::shared_ptr element; + RETURN_IF_ERROR(build_native_schema_node(array_type->get_nested_type(), + *file_schema.children[0], &element)); + *result = std::make_shared(std::move(element)); + return Status::OK(); + } + case TYPE_MAP: { + if (file_schema.kind != ParquetColumnSchemaKind::MAP || file_schema.children.size() != 2) { + return Status::Corruption("Parquet column {} is not a MAP", file_schema.name); + } + const auto* map_type = assert_cast(type.get()); + std::shared_ptr key; + std::shared_ptr value; + RETURN_IF_ERROR( + build_native_schema_node(map_type->get_key_type(), *file_schema.children[0], &key)); + RETURN_IF_ERROR(build_native_schema_node(map_type->get_value_type(), + *file_schema.children[1], &value)); + *result = std::make_shared(std::move(key), std::move(value)); + return Status::OK(); + } + default: + if (file_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { + return Status::Corruption("Parquet column {} is not a scalar", file_schema.name); + } + *result = std::make_shared(); + return Status::OK(); + } +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.h b/be/src/format_v2/parquet/native_schema_node.h new file mode 100644 index 00000000000000..25c42ad4b2c6d4 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" + +namespace doris::format::parquet { + +struct ParquetColumnSchema; + +// V2-owned semantic mapping consumed by the native decoder. It deliberately contains no V1 +// reader/schema-helper state, so FileScannerV2 can build the mapping from its native metadata tree. +class NativeSchemaNode { +public: + virtual ~NativeSchemaNode() = default; + + virtual std::shared_ptr child(const std::string&) const { return nullptr; } + virtual std::string file_child_name(const std::string&) const { return {}; } + virtual bool has_child(const std::string&) const { return false; } + virtual std::shared_ptr element() const { return nullptr; } + virtual std::shared_ptr key() const { return nullptr; } + virtual std::shared_ptr value() const { return nullptr; } +}; + +class NativeScalarSchemaNode final : public NativeSchemaNode {}; + +class NativeStructSchemaNode final : public NativeSchemaNode { +public: + void add_child(std::string table_name, std::string file_name, + std::shared_ptr node); + void add_missing_child(std::string table_name); + + std::shared_ptr child(const std::string& table_name) const override; + std::string file_child_name(const std::string& table_name) const override; + bool has_child(const std::string& table_name) const override; + +private: + struct Child { + std::string file_name; + std::shared_ptr node; + }; + std::map _children; +}; + +class NativeArraySchemaNode final : public NativeSchemaNode { +public: + explicit NativeArraySchemaNode(std::shared_ptr element) + : _element(std::move(element)) {} + std::shared_ptr element() const override { return _element; } + +private: + std::shared_ptr _element; +}; + +class NativeMapSchemaNode final : public NativeSchemaNode { +public: + NativeMapSchemaNode(std::shared_ptr key, + std::shared_ptr value) + : _key(std::move(key)), _value(std::move(value)) {} + std::shared_ptr key() const override { return _key; } + std::shared_ptr value() const override { return _value; } + +private: + std::shared_ptr _key; + std::shared_ptr _value; +}; + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp index 1cdfed80bd273b..610f234a79741c 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.cpp +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -15,481 +15,214 @@ #include "format_v2/parquet/parquet_column_schema.h" -#include - #include #include #include #include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/parquet_type.h" namespace doris::format::parquet { namespace { -struct SchemaBuildContext { - int32_t local_id = -1; // child ordinal in the parent node - int16_t definition_level = 0; // accumulated optional/repeated level count - int16_t repetition_level = 0; // accumulated repeated level count - int16_t nullable_definition_level = 0; // definition level of the nearest optional node - int16_t repeated_repetition_level = 0; // repetition level of the nearest repeated node - int16_t repeated_ancestor_definition_level = 0; // definition level of the nearest repeated node -}; - -enum class SchemaBuildMode { - // Normal recursive schema build. Bare repeated fields are exposed as Doris ARRAY for - // protobuf/legacy Parquet compatibility, while repeated LIST/MAP annotated groups are rejected - // because Parquet LIST/MAP outer groups are not allowed to be repeated at a top-level or struct - // field boundary. - NORMAL, - // Build the current repeated node as the already-selected element of an enclosing LIST. This - // is the compatibility path for Arrow/parquet-format legacy two-level LIST encodings where the - // repeated node itself is the array element instead of a wrapper that should be stripped. - REPEATED_NODE_AS_LIST_ELEMENT, - // Build the current repeated group as a STRUCT element of an enclosing LIST, ignoring LIST/MAP - // annotations on the repeated group itself. This keeps compatibility with the old Doris - // Parquet schema parser for Hive/legacy wrappers named "array" or "_tuple". - REPEATED_NODE_AS_STRUCT_ELEMENT, -}; - -// Result of applying Parquet LIST backward compatibility rules to the single repeated child of a -// LIST-annotated group. The repeated child can either be a physical wrapper whose only child is the -// element, or the element node itself. -struct ListElementResolution { - // Parquet node that should be exposed as Doris ARRAY element. - const ::parquet::schema::Node* element_node = nullptr; - // Level state after consuming the LIST repeated child. The parent ARRAY schema keeps this state - // to materialize offsets, empty arrays and null arrays. - SchemaBuildContext repeated_context; - // Level state used to build element_node. This equals repeated_context when the repeated child - // itself is the element, and includes the wrapper's only child when standard 3-level LIST - // encoding is stripped. - SchemaBuildContext element_context; - // Build mode for element_node. Non-NORMAL modes mean element_node is the repeated child itself, - // and the repeated level must not be interpreted as a second unrelated array at the same - // boundary. - SchemaBuildMode element_build_mode = SchemaBuildMode::NORMAL; -}; - -// Resolved repeated entry group of a MAP-annotated group. The entry wrapper is a physical Parquet -// encoding detail; Doris folds it into the parent MAP schema and exposes only direct [key, value] -// children. -struct MapEntryResolution { - const ::parquet::schema::GroupNode* entry_group = nullptr; - // Level state after consuming the repeated entry group. The parent MAP schema keeps this state - // to materialize offsets, empty maps and null maps. - SchemaBuildContext entry_context; -}; - -bool is_list_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::LIST || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_list()); -} - -bool is_map_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::MAP || - node.converted_type() == ::parquet::ConvertedType::MAP_KEY_VALUE || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_map()); -} - -bool has_logical_annotation(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return (node.converted_type() != ::parquet::ConvertedType::NONE && - node.converted_type() != ::parquet::ConvertedType::UNDEFINED) || - (logical_type != nullptr && logical_type->is_valid() && !logical_type->is_none()); -} - -bool has_structural_list_name(const std::string& list_name, const std::string& repeated_name) { - return repeated_name == "array" || repeated_name == list_name + "_tuple"; -} - -bool should_build_repeated_field_as_list(const ::parquet::schema::Node& node) { - return node.is_repeated() && !is_list_node(node) && !is_map_node(node); -} - -DataTypePtr nullable_if_needed(DataTypePtr type, const ::parquet::schema::Node& node) { - return node.is_optional() ? make_nullable(type) : type; -} - -void inherit_common_schema_state(const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - column_schema->local_id = context.local_id; - column_schema->parquet_field_id = node.field_id(); - column_schema->name = node.name(); - column_schema->max_definition_level = context.definition_level; - column_schema->max_repetition_level = context.repetition_level; - column_schema->nullable_definition_level = context.nullable_definition_level; - column_schema->definition_level = context.definition_level; - column_schema->repetition_level = context.repetition_level; - column_schema->repeated_ancestor_definition_level = context.repeated_ancestor_definition_level; - column_schema->repeated_repetition_level = context.repeated_repetition_level; -} - -SchemaBuildContext child_context(const SchemaBuildContext& parent, - const ::parquet::schema::Node& child_node, int32_t child_idx) { - SchemaBuildContext result = parent; - result.local_id = child_idx; - if (child_node.repetition() == ::parquet::Repetition::OPTIONAL) { - result.definition_level++; - result.nullable_definition_level = result.definition_level; - } - if (child_node.is_repeated()) { - result.repetition_level++; - result.definition_level++; - result.repeated_repetition_level = result.repetition_level; - result.repeated_ancestor_definition_level = result.definition_level; - } - return result; -} - -void propagate_child_levels(ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - for (const auto& child : column_schema->children) { - column_schema->max_definition_level = - std::max(column_schema->max_definition_level, child->max_definition_level); - column_schema->max_repetition_level = - std::max(column_schema->max_repetition_level, child->max_repetition_level); - } -} - -// Mirrors Arrow's ResolveList() compatibility rules, but only decides which Parquet node is the -// logical LIST element. The caller still builds Doris' semantic LIST->[element] schema tree. -// Important cases: -// - repeated primitive: the primitive itself is the element (legacy two-level LIST). -// - repeated group with multiple children: the group itself is a STRUCT element. -// - repeated group named "array" or "_tuple": the group itself is a STRUCT element per -// Parquet backward compatibility rules, even when it has one child or its own logical annotation. -// This also keeps v2 file-local schema aligned with Doris' old schema parser used by HDFS TVF. -// - other repeated group with a logical annotation, or whose only child is repeated: the group -// itself is the element. This preserves nested LIST/MAP and repeated fields inside struct -// elements. -// - otherwise, strip the one-child repeated wrapper as standard three-level LIST encoding. -Status resolve_list_element_node(const ::parquet::schema::GroupNode& list_group, - const SchemaBuildContext& list_context, - ListElementResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (list_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - const auto& repeated_node = *list_group.field(0); - if (!repeated_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - result->repeated_context = child_context(list_context, repeated_node, 0); - if (repeated_node.is_primitive()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - const auto& repeated_group = static_cast(repeated_node); - if (repeated_group.field_count() == 0) { - return Status::NotSupported("Unsupported parquet LIST element layout for column {}", - list_group.name()); +ParquetTimeUnit native_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; } - const bool repeated_group_has_logical_annotation = has_logical_annotation(repeated_group); - if (repeated_group.field_count() > 1 || - has_structural_list_name(list_group.name(), repeated_group.name())) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT; - return Status::OK(); + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; } - if (repeated_group_has_logical_annotation) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; } - - const auto& only_child = *repeated_group.field(0); - if (only_child.is_repeated()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - result->element_node = &only_child; - result->element_context = child_context(result->repeated_context, only_child, 0); - return Status::OK(); -} - -// Resolves the repeated entry group of a MAP/MAP_KEY_VALUE node. Unlike LIST, MAP has no supported -// two-level form in this reader: Doris requires a repeated group with exactly key and value -// children, then folds that physical entry group out of ParquetColumnSchema. Some external writers -// emit optional MAP keys even though standard Parquet MAP keys are required; keep the key's -// definition levels and expose it as nullable for compatibility with the old reader. -Status resolve_map_entry_group(const ::parquet::schema::GroupNode& map_group, - const SchemaBuildContext& map_context, MapEntryResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (map_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - const auto& entry_node = *map_group.field(0); - if (!entry_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - if (entry_node.is_primitive()) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - const auto& entry_group = static_cast(entry_node); - if (entry_group.field_count() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - // The Parquet logical MAP spec requires key to be REQUIRED. Some legacy/Hive-written files - // still mark the key field OPTIONAL even when all actual keys are non-null, for example: - // optional group t_map_varchar (MAP) { - // repeated group key_value { - // optional binary key (STRING); - // optional binary value (STRING); - // } - // } - // Accept that schema here so compatible files can be read. MapColumnReader validates the - // materialized key column and rejects data that really contains null map keys. - result->entry_group = &entry_group; - result->entry_context = child_context(map_context, entry_node, 0); - return Status::OK(); + return ParquetTimeUnit::UNKNOWN; } -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode); - -// Builds a semantic ARRAY schema for a bare repeated field. Arrow handles this in -// NodeToSchemaField()/GroupToSchemaField(); Doris needs the same compatibility behavior because -// protobuf and old parquet writers often encode repeated fields without a LIST annotation. -// Example: -// optional group event { -// repeated group links { -// optional binary url (UTF8); -// optional int32 rank; -// } -// } -// Doris exposes event.links as ARRAY>, not STRUCT. This keeps v2's -// file-local schema aligned with the old schema parser used by HDFS TVF schema fetching. -// When the repeated field appears inside an already resolved LIST element, only the nested repeated -// child should be wrapped: -// optional group a (LIST) { -// repeated group element { -// repeated int32 items; -// } -// } -// The outer LIST element is the repeated "element" group, and its repeated "items" child should be -// represented as a field of type ARRAY inside the struct element. -Status build_repeated_field_as_list_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& repeated_node, - const SchemaBuildContext& repeated_context, - std::unique_ptr* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); +ParquetExtraTypeInfo native_time_extra(ParquetTimeUnit unit) { + switch (unit) { + case ParquetTimeUnit::MILLIS: + return ParquetExtraTypeInfo::UNIT_MS; + case ParquetTimeUnit::MICROS: + return ParquetExtraTypeInfo::UNIT_MICROS; + case ParquetTimeUnit::NANOS: + return ParquetExtraTypeInfo::UNIT_NS; + case ParquetTimeUnit::UNKNOWN: + default: + return ParquetExtraTypeInfo::NONE; } - auto list_schema = std::make_unique(); - inherit_common_schema_state(repeated_node, repeated_context, list_schema.get()); - list_schema->kind = ParquetColumnSchemaKind::LIST; - list_schema->definition_level = repeated_context.definition_level; - list_schema->repetition_level = repeated_context.repetition_level; - list_schema->repeated_repetition_level = repeated_context.repeated_repetition_level; - - std::unique_ptr element_child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, repeated_node, repeated_context, - &element_child, - SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT)); - element_child->name = "element"; - list_schema->type = std::make_shared(element_child->type); - list_schema->children.push_back(std::move(element_child)); - propagate_child_levels(list_schema.get()); - *result = std::move(list_schema); - return Status::OK(); } -// Recursively builds ParquetColumnSchema for the given schema node and its children in Parquet -// file's metadata. NORMAL mode exposes bare repeated fields as ARRAY for legacy compatibility. -// REPEATED_NODE_AS_LIST_ELEMENT mode means the current repeated node was already selected as an -// enclosing LIST element, so only its nested bare repeated children should be wrapped. -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (mode == SchemaBuildMode::NORMAL && should_build_repeated_field_as_list(node)) { - return build_repeated_field_as_list_schema(schema, node, context, result); - } - - auto column_schema = std::make_unique(); - inherit_common_schema_state(node, context, column_schema.get()); - - if (node.is_primitive()) { - const int leaf_column_id = schema.ColumnIndex(node); - if (leaf_column_id < 0) { - return Status::InvalidArgument("Cannot find leaf column id for parquet column {}", - node.name()); - } - column_schema->kind = ParquetColumnSchemaKind::PRIMITIVE; - column_schema->leaf_column_id = leaf_column_id; - column_schema->descriptor = schema.Column(leaf_column_id); - if (column_schema->descriptor != nullptr) { - column_schema->max_definition_level = column_schema->descriptor->max_definition_level(); - column_schema->max_repetition_level = column_schema->descriptor->max_repetition_level(); +void fill_native_type_descriptor(const NativeFieldSchema& field, ParquetTypeDescriptor* result) { + DORIS_CHECK(result != nullptr); + const auto& schema = field.parquet_schema; + result->doris_type = field.data_type; + result->physical_type = static_cast(field.physical_type); + result->fixed_length = schema.__isset.type_length ? schema.type_length : -1; + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.DECIMAL) { + result->is_decimal = true; + result->decimal_precision = logical.DECIMAL.precision; + result->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.INTEGER) { + result->integer_bit_width = logical.INTEGER.bitWidth; + result->is_unsigned_integer = !logical.INTEGER.isSigned; + } else if (logical.__isset.TIME) { + result->time_unit = native_time_unit(logical.TIME.unit); + result->extra_type_info = native_time_extra(result->time_unit); + if (logical.TIME.isAdjustedToUTC) { + result->unsupported_reason = + "Parquet TIME with isAdjustedToUTC=true is not supported"; + } + } else if (logical.__isset.TIMESTAMP) { + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + result->time_unit = native_time_unit(logical.TIMESTAMP.unit); + result->extra_type_info = native_time_extra(result->time_unit); + } else if (logical.__isset.FLOAT16) { + result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; } - column_schema->type_descriptor = resolve_parquet_type(column_schema->descriptor); - column_schema->type = column_schema->type_descriptor.doris_type; - if (column_schema->type == nullptr && - !column_schema->type_descriptor.unsupported_reason.empty()) { - // Keep unsupported logical leaves in the file schema using their physical storage - // type. For example, a file `{id: INT32, clock: TIME_MILLIS}` remains readable for - // `SELECT id`: schema mapping sees `clock` as its physical INT32 but never creates its - // reader. `SELECT clock` still fails explicitly in ParquetColumnReaderFactory before - // any physical value is decoded, preserving the unsupported-type contract. - column_schema->type = column_schema->type_descriptor.physical_doris_type; - } - if (column_schema->type == nullptr) { - return Status::NotSupported("Unsupported parquet column type for column {}", - node.name()); - } - column_schema->type = node.is_optional() - ? make_nullable(remove_nullable(column_schema->type)) - : remove_nullable(column_schema->type); - *result = std::move(column_schema); - return Status::OK(); - } - - const auto& group = static_cast(node); - if (is_list_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet LIST column {}", node.name()); + } else if (schema.__isset.converted_type) { + switch (schema.converted_type) { + case tparquet::ConvertedType::DECIMAL: + result->is_decimal = true; + result->decimal_precision = schema.__isset.precision ? schema.precision : -1; + result->decimal_scale = schema.__isset.scale ? schema.scale : -1; + break; + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::UINT_8: + result->integer_bit_width = 8; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_8; + break; + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::UINT_16: + result->integer_bit_width = 16; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_16; + break; + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::UINT_32: + result->integer_bit_width = 32; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_32; + break; + case tparquet::ConvertedType::INT_64: + case tparquet::ConvertedType::UINT_64: + result->integer_bit_width = 64; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_64; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = true; + result->time_unit = schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS + ? ParquetTimeUnit::MILLIS + : ParquetTimeUnit::MICROS; + result->extra_type_info = native_time_extra(result->time_unit); + break; + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::TIME_MICROS: + result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; + break; + default: + break; } - column_schema->kind = ParquetColumnSchemaKind::LIST; - ListElementResolution list_element; - RETURN_IF_ERROR(resolve_list_element_node(group, context, &list_element)); - column_schema->definition_level = list_element.repeated_context.definition_level; - column_schema->repetition_level = list_element.repeated_context.repetition_level; - column_schema->repeated_repetition_level = - list_element.repeated_context.repeated_repetition_level; - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, *list_element.element_node, - list_element.element_context, &child, - list_element.element_build_mode)); - child->name = "element"; - column_schema->type = - nullable_if_needed(std::make_shared(child->type), node); - column_schema->children.push_back(std::move(child)); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); } - if (is_map_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet MAP column {}", node.name()); + if (result->is_decimal) { + switch (result->physical_type) { + case tparquet::Type::INT32: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; + break; + case tparquet::Type::INT64: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; + break; + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; + break; + default: + break; } - column_schema->kind = ParquetColumnSchemaKind::MAP; - MapEntryResolution map_entry; - RETURN_IF_ERROR(resolve_map_entry_group(group, context, &map_entry)); - column_schema->definition_level = map_entry.entry_context.definition_level; - column_schema->repetition_level = map_entry.entry_context.repetition_level; - column_schema->repeated_repetition_level = - map_entry.entry_context.repeated_repetition_level; - for (int child_idx = 0; child_idx < map_entry.entry_group->field_count(); ++child_idx) { - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode( - schema, *map_entry.entry_group->field(child_idx), - child_context(map_entry.entry_context, *map_entry.entry_group->field(child_idx), - child_idx), - &child, SchemaBuildMode::NORMAL)); - child->name = child_idx == 0 ? "key" : "value"; - column_schema->children.push_back(std::move(child)); - } - if (column_schema->children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - node.name()); - } - auto key_type = make_nullable(column_schema->children[0]->type); - auto value_type = make_nullable(column_schema->children[1]->type); - column_schema->type = - nullable_if_needed(std::make_shared(key_type, value_type), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); - } + } else if (result->physical_type == tparquet::Type::INT96) { + result->is_timestamp = true; + result->extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; + } + result->is_string_like = !result->is_decimal && + result->extra_type_info != ParquetExtraTypeInfo::FLOAT16 && + (result->physical_type == tparquet::Type::BYTE_ARRAY || + result->physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY); +} - column_schema->kind = ParquetColumnSchemaKind::STRUCT; - DataTypes child_types; - Strings child_names; - child_types.reserve(group.field_count()); - child_names.reserve(group.field_count()); - for (int child_idx = 0; child_idx < group.field_count(); ++child_idx) { - const auto& child_node = *group.field(child_idx); - std::unique_ptr child; - const auto child_ctx = child_context(context, child_node, child_idx); - if (should_build_repeated_field_as_list(child_node)) { - RETURN_IF_ERROR( - build_repeated_field_as_list_schema(schema, child_node, child_ctx, &child)); - } else { - RETURN_IF_ERROR(build_node_schema_with_mode(schema, child_node, child_ctx, &child, - SchemaBuildMode::NORMAL)); - } - child_types.push_back(make_nullable(child->type)); - child_names.push_back(child->name); - column_schema->children.push_back(std::move(child)); +void propagate_native_max_levels(ParquetColumnSchema* schema) { + DORIS_CHECK(schema != nullptr); + for (const auto& child : schema->children) { + DORIS_CHECK(child != nullptr); + propagate_native_max_levels(child.get()); + schema->max_definition_level = + std::max(schema->max_definition_level, child->max_definition_level); + schema->max_repetition_level = + std::max(schema->max_repetition_level, child->max_repetition_level); } - column_schema->type = - nullable_if_needed(std::make_shared(child_types, child_names), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); } -Status build_node_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, const SchemaBuildContext& context, - std::unique_ptr* result) { - return build_node_schema_with_mode(schema, node, context, result, SchemaBuildMode::NORMAL); +std::unique_ptr build_native_node_schema(const NativeFieldSchema& field, + int32_t local_id) { + auto result = std::make_unique(); + result->local_id = local_id; + result->parquet_field_id = field.field_id; + result->name = field.name; + result->type = field.data_type; + result->definition_level = field.definition_level; + result->repetition_level = field.repetition_level; + result->max_definition_level = field.definition_level; + result->max_repetition_level = field.repetition_level; + result->nullable_definition_level = field.data_type != nullptr && field.data_type->is_nullable() + ? field.definition_level - field.repetition_level + : 0; + result->repeated_ancestor_definition_level = field.repeated_parent_def_level; + result->repeated_repetition_level = field.repetition_level; + + const auto primitive_type = remove_nullable(field.data_type)->get_primitive_type(); + if (field.children.empty()) { + result->kind = ParquetColumnSchemaKind::PRIMITIVE; + result->leaf_column_id = field.physical_column_index; + fill_native_type_descriptor(field, &result->type_descriptor); + return result; + } + if (primitive_type == TYPE_ARRAY) { + result->kind = ParquetColumnSchemaKind::LIST; + } else if (primitive_type == TYPE_MAP) { + result->kind = ParquetColumnSchemaKind::MAP; + } else { + result->kind = ParquetColumnSchemaKind::STRUCT; + } + result->children.reserve(field.children.size()); + for (size_t child_idx = 0; child_idx < field.children.size(); ++child_idx) { + result->children.push_back( + build_native_node_schema(field.children[child_idx], cast_set(child_idx))); + } + propagate_native_max_levels(result.get()); + return result; } } // namespace -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields) { if (fields == nullptr) { return Status::InvalidArgument("fields is null"); } fields->clear(); - const auto* root = schema.group_node(); - if (root == nullptr) { - return Status::InvalidArgument("Parquet schema root is null"); - } - fields->reserve(root->field_count()); - for (int field_idx = 0; field_idx < root->field_count(); ++field_idx) { - std::unique_ptr field; - SchemaBuildContext context; - RETURN_IF_ERROR(build_node_schema( - schema, *root->field(field_idx), - child_context(context, *root->field(field_idx), field_idx), &field)); - fields->push_back(std::move(field)); + const auto& native_fields = schema.get_fields_schema(); + fields->reserve(native_fields.size()); + for (size_t field_idx = 0; field_idx < native_fields.size(); ++field_idx) { + // Unsupported logical leaves stay in the file schema so request-level validation can + // ignore unprojected fields and COUNT(*) placeholders without weakening real projections. + // The scan projection and native readers must share one tree; rebuilding wrappers through + // Arrow changes legacy LIST/STRUCT boundaries and makes valid nested values look absent. + fields->push_back( + build_native_node_schema(native_fields[field_idx], cast_set(field_idx))); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_column_schema.h b/be/src/format_v2/parquet/parquet_column_schema.h index 1fb7262aabde6f..697bcd498382b1 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.h +++ b/be/src/format_v2/parquet/parquet_column_schema.h @@ -23,18 +23,15 @@ #include "core/data_type/data_type.h" #include "format_v2/parquet/parquet_type.h" -namespace parquet { -class ColumnDescriptor; -class SchemaDescriptor; -} // namespace parquet - namespace doris::format::parquet { +class NativeFieldDescriptor; + enum class ParquetColumnSchemaKind { - PRIMITIVE, // primitive leaf -> ScalarColumnReader - STRUCT, // struct -> StructColumnReader - LIST, // array -> ListColumnReader - MAP, // map -> MapColumnReader + PRIMITIVE, // physical primitive leaf + STRUCT, // Parquet group with STRUCT semantics + LIST, // Parquet group with LIST semantics + MAP, // Parquet group with MAP semantics }; // ============================================================================ @@ -55,8 +52,6 @@ struct ParquetColumnSchema { ParquetColumnSchemaKind kind = ParquetColumnSchemaKind::PRIMITIVE; - const ::parquet::ColumnDescriptor* descriptor = nullptr; - // ======== Dremel Levels ======== int16_t max_definition_level = 0; @@ -74,7 +69,7 @@ struct ParquetColumnSchema { std::vector> children {}; }; -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields); } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index aa8f2622ade43b..00956b3d09ce65 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -15,154 +15,125 @@ #include "format_v2/parquet/parquet_file_context.h" -#include -#include #include -#include -#include #include #include #include #include -#include #include #include +#include "common/cast_set.h" #include "common/check.h" #include "common/config.h" +#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" -#include "storage/cache/page_cache.h" +#include "runtime/exec_env.h" +#include "util/coding.h" #include "util/slice.h" +#include "util/thrift_util.h" +#include "util/time.h" namespace doris::format::parquet { -namespace detail { - -namespace { - -bool page_cache_range_less(const ParquetPageCacheRange& lhs, const ParquetPageCacheRange& rhs) { - return lhs.offset < rhs.offset || (lhs.offset == rhs.offset && lhs.size < rhs.size); -} +constexpr size_t V2_PARQUET_FOOTER_SIZE = 8; -} // namespace - -ParquetPageCacheRangeIndex::ParquetPageCacheRangeIndex(size_t max_ranges) - : _max_ranges(max_ranges) { - DORIS_CHECK(_max_ranges > 0); +NativeParquetMetadata::NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size) + : _metadata(std::move(metadata)), _parsed_size(parsed_size) { + ExecEnv::GetInstance()->parquet_meta_tracker()->consume(get_mem_size()); } -void ParquetPageCacheRangeIndex::insert(ParquetPageCacheRange range) { - std::lock_guard lock(_mutex); - auto it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); - if (it != _ranges.end() && it->offset == range.offset && it->size == range.size) { - return; - } - if (_ranges.size() >= _max_ranges) { - _ranges.erase(_ranges.begin()); - it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); - } - _ranges.insert(it, range); +NativeParquetMetadata::~NativeParquetMetadata() { + ExecEnv::GetInstance()->parquet_meta_tracker()->release(get_mem_size()); } -void ParquetPageCacheRangeIndex::erase(ParquetPageCacheRange range) { - std::lock_guard lock(_mutex); - const auto it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); - if (it != _ranges.end() && it->offset == range.offset && it->size == range.size) { - _ranges.erase(it); +Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + _schema.set_enable_mapping_varbinary(enable_mapping_varbinary); + _schema.set_enable_mapping_timestamp_tz(enable_mapping_timestamp_tz); + RETURN_IF_ERROR(_schema.parse_from_thrift(_metadata.schema)); + // Native readers address projected leaves by stable DFS IDs. Assign them only on the private + // v2 schema object so v1's cached schema lifecycle and numbering remain untouched. + _schema.assign_ids(); + for (size_t row_group_idx = 0; row_group_idx < _metadata.row_groups.size(); ++row_group_idx) { + const auto& row_group = _metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.columns.size() != _schema.physical_fields_size()) { + // All v2 planners index chunks by the native DFS leaf order, so validate cardinality + // once before any projection, prefetch, or decoder can perform indexed access. + return Status::Corruption( + "Parquet row group {} has {} column chunks but schema has {} physical fields", + row_group_idx, row_group.columns.size(), _schema.physical_fields_size()); + } + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + if (chunk.meta_data.type != _schema.get_physical_field(column_idx)->physical_type) { + return Status::Corruption( + "Parquet row group {} column {} physical type {} does not match schema {}", + row_group_idx, column_idx, tparquet::to_string(chunk.meta_data.type), + tparquet::to_string(_schema.get_physical_field(column_idx)->physical_type)); + } + } } + return Status::OK(); } -std::vector ParquetPageCacheRangeIndex::ranges() const { - std::lock_guard lock(_mutex); - return _ranges; -} - -size_t ParquetPageCacheRangeIndex::size() const { - std::lock_guard lock(_mutex); - return _ranges.size(); -} - -ParquetPageCacheRangeDirectory::ParquetPageCacheRangeDirectory(size_t max_files) - : _max_files(max_files) { - DORIS_CHECK(_max_files > 0); -} +namespace detail { -std::shared_ptr ParquetPageCacheRangeDirectory::get_or_create( - const std::string& file_key) { - DORIS_CHECK(!file_key.empty()); - std::lock_guard lock(_mutex); - if (const auto it = _indexes.find(file_key); it != _indexes.end()) { - return it->second; +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit) { + if (file_size < V2_PARQUET_FOOTER_SIZE || + serialized_size > file_size - V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 footer size {} exceeds file size {}", serialized_size, + file_size); } - if (_indexes.size() >= _max_files) { - _indexes.erase(_indexes.begin()); + if (serialized_size > metadata_size_limit) { + return Status::Corruption("Parquet v2 footer size {} exceeds metadata limit {}", + serialized_size, metadata_size_limit); } - auto index = std::make_shared(); - _indexes.emplace(file_key, index); - return index; -} - -size_t ParquetPageCacheRangeDirectory::size() const { - std::lock_guard lock(_mutex); - return _indexes.size(); + return Status::OK(); } -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges) { - if (position < 0 || nbytes <= 0) { +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable) { + const int64_t mtime = description_mtime != 0 ? description_mtime : reader_mtime; + if (mtime == 0 && !is_immutable) { + // Unknown version is not a stable identity: an overwrite can preserve path and size while + // changing both footer semantics and page bytes. return {}; } + const int64_t file_size = description_file_size >= 0 ? description_file_size : reader_file_size; + return fmt::format("{}::{}::mtime={}::size={}", fs_name, path, mtime, file_size); +} - std::vector ranges; - ranges.reserve(cached_ranges.size()); - const int64_t request_end = position + nbytes; - for (const auto& range : cached_ranges) { - if (range.size > 0 && range.offset < request_end && position < range.end_offset()) { - ranges.push_back(range); - } - } - std::sort(ranges.begin(), ranges.end(), [](const auto& lhs, const auto& rhs) { - if (lhs.offset != rhs.offset) { - return lhs.offset < rhs.offset; - } - return lhs.size > rhs.size; - }); - - std::vector plan; - int64_t cursor = position; - while (cursor < request_end) { - // At each cursor position, choose the cached range that already covers the cursor and - // extends farthest to the right. This handles both adjacent ranges and overlapping - // ranges. If no range covers the current cursor, there is a gap and the request must - // miss as a whole. - auto best = ranges.end(); - int64_t best_end = cursor; - for (auto it = ranges.begin(); it != ranges.end(); ++it) { - const int64_t cached_end = it->end_offset(); - if (it->offset <= cursor && cursor < cached_end && cached_end > best_end) { - best = it; - best_end = cached_end; - } - } - if (best == ranges.end()) { - return {}; - } - const int64_t copy_size = std::min(best_end, request_end) - cursor; - ParquetPageCacheReadPlanEntry entry; - entry.cached_range = *best; - entry.copy_offset_in_cache = cursor - best->offset; - entry.output_offset = cursor - position; - entry.copy_size = copy_size; - plan.push_back(entry); - cursor += copy_size; +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length) { + if (offset < 0 || length <= 0 || length > MAX_SERIALIZED_PARQUET_INDEX_BYTES || + static_cast(offset) > file_size) { + return false; } - return plan; + return static_cast(length) <= file_size - static_cast(offset); +} + +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end) { + return span_offset >= 0 && span_end >= span_offset && + span_end - span_offset <= MAX_SERIALIZED_PARQUET_INDEX_BYTES; } std::vector valid_prefetch_ranges( @@ -197,455 +168,426 @@ bool should_use_merge_range_reader(const std::vector& ran avg_io_size < io::MergeRangeFileReader::SMALL_IO; } +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size) { + return file_size <= in_memory_file_size && + (path.starts_with("http://") || path.starts_with("https://")); +} + } // namespace detail namespace { -detail::ParquetPageCacheRangeDirectory& cached_page_range_directory() { - // Directory lookup is paid once when a reader opens. ReadAt() then synchronizes only on the - // shared index for this file, so unrelated Parquet files no longer serialize on a process-wide - // hot-path mutex. Strong references deliberately keep range discovery alive after reader A - // closes: reader B can reuse cached [100, 200) for a request [120, 150). The directory and each - // per-file index are independently capped, bounding stale metadata left by page-cache eviction. - static detail::ParquetPageCacheRangeDirectory directory; - return directory; +constexpr uint8_t V2_PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'}; +constexpr size_t V2_INITIAL_FOOTER_READ_SIZE = 48 * 1024; + +Status parse_native_parquet_footer(io::FileReaderSPtr file, + std::unique_ptr* metadata, + size_t* footer_size, io::IOContext* io_ctx, + bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(footer_size != nullptr); + const size_t file_size = file->size(); + if (file_size < V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 file is too small for a footer: {}", file_size); + } + + const size_t tail_size = std::min(file_size, V2_INITIAL_FOOTER_READ_SIZE); + std::vector tail(tail_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - tail_size, Slice(tail.data(), tail.size()), + &bytes_read, io_ctx)); + if (bytes_read != tail.size()) { + return Status::Corruption("Short Parquet v2 footer read: expected {}, got {}", tail.size(), + bytes_read); + } + const auto* magic = tail.data() + tail.size() - sizeof(V2_PARQUET_MAGIC); + if (memcmp(magic, V2_PARQUET_MAGIC, sizeof(V2_PARQUET_MAGIC)) != 0) { + return Status::Corruption("Invalid Parquet v2 footer magic in {}", file->path().native()); + } + + const uint32_t serialized_size = + decode_fixed32_le(tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE); + // The configured Thrift message ceiling also bounds this file-controlled allocation. Keep the + // check before both allocation and the optional second read so a sparse file cannot force a + // process-sized metadata buffer merely by advertising a large footer. + const size_t metadata_size_limit = + static_cast(std::max(config::thrift_max_message_size, 0)); + RETURN_IF_ERROR( + detail::validate_native_footer_size(serialized_size, file_size, metadata_size_limit)); + std::vector serialized_metadata(serialized_size); + if (serialized_size <= tail.size() - V2_PARQUET_FOOTER_SIZE) { + const auto* metadata_start = + tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE - serialized_size; + memcpy(serialized_metadata.data(), metadata_start, serialized_size); + } else { + bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - V2_PARQUET_FOOTER_SIZE - serialized_size, + Slice(serialized_metadata.data(), serialized_metadata.size()), + &bytes_read, io_ctx)); + if (bytes_read != serialized_metadata.size()) { + return Status::Corruption("Short Parquet v2 metadata read: expected {}, got {}", + serialized_metadata.size(), bytes_read); + } + } + + uint32_t thrift_size = serialized_size; + tparquet::FileMetaData thrift_metadata; + RETURN_IF_ERROR(deserialize_thrift_msg(serialized_metadata.data(), &thrift_size, true, + &thrift_metadata)); + auto parsed = + std::make_unique(std::move(thrift_metadata), serialized_size); + RETURN_IF_ERROR(parsed->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz)); + *footer_size = V2_PARQUET_FOOTER_SIZE + serialized_size; + *metadata = std::move(parsed); + return Status::OK(); } std::string build_page_cache_file_key(const io::FileReader& file_reader, const io::FileDescription& file_description) { - const int64_t mtime = - file_description.mtime != 0 ? file_description.mtime : file_reader.mtime(); - if (mtime == 0 && !file_description.is_immutable) { - // mtime == 0 means "unknown version", not the Unix epoch. V1 historically caches such a - // file under path::0, but copying that behavior for every V2 file is unsafe: a mutable file - // can be overwritten with different bytes while retaining both its path and size, causing - // process-global page cache entries to return stale data. Only callers that explicitly - // guarantee path immutability may use the mtime=0 cache key below. - return {}; - } - const int64_t file_size = file_description.file_size >= 0 - ? file_description.file_size - : static_cast(file_reader.size()); - return fmt::format("{}::{}::mtime={}::size={}", file_description.fs_name, - file_reader.path().native(), mtime, file_size); + return detail::build_native_file_cache_key( + file_description.fs_name, file_reader.path().native(), file_description.mtime, + file_reader.mtime(), file_description.file_size, + static_cast(file_reader.size()), file_description.is_immutable); } -class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { -public: - DorisRandomAccessFile(io::FileReaderSPtr file_reader, io::IOContext* io_ctx, - bool enable_page_cache, std::string page_cache_file_key) - : _file_reader(std::move(file_reader)), - _base_file_reader(_file_reader), - _io_ctx(io_ctx), - _enable_page_cache(enable_page_cache), - _page_cache_file_key(std::move(page_cache_file_key)), - _cached_page_range_index( - _enable_page_cache && !_page_cache_file_key.empty() - ? cached_page_range_directory().get_or_create(_page_cache_file_key) - : nullptr) { - DORIS_CHECK(_file_reader != nullptr); - if (auto tracing_reader = std::dynamic_pointer_cast(_file_reader)) { - _file_reader_stats = tracing_reader->stats(); - _base_file_reader = tracing_reader->inner_reader(); - } - DORIS_CHECK(_base_file_reader != nullptr); - set_mode(arrow::io::FileMode::READ); - } - - arrow::Status Close() override { - if (!_closed) { - collect_active_merge_range_profile(); - std::lock_guard lock(_page_cache_mutex); - // Page payloads and their bounded per-file range index intentionally outlive this - // reader for warm scans. Only reader-specific projected ranges are released here. - std::vector().swap(_page_cache_ranges); - _closed = true; - } - return arrow::Status::OK(); - } - - bool closed() const override { return _closed; } - - arrow::Result Tell() const override { return _pos; } - - arrow::Status Seek(int64_t position) override { - if (position < 0) { - return arrow::Status::Invalid("negative seek position"); - } - _pos = position; - return arrow::Status::OK(); - } - - arrow::Result GetSize() override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - return static_cast(_file_reader->size()); - } - - arrow::Result Read(int64_t nbytes, void* out) override { - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(_pos, nbytes, out)); - _pos += bytes_read; - return bytes_read; - } - - arrow::Result> Read(int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, Read(nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; - } +} // namespace - arrow::Result ReadAt(int64_t position, int64_t nbytes, void* out) override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - if (position < 0 || nbytes < 0) { - return arrow::Status::Invalid("negative read position or length"); - } - if (try_read_from_page_cache(position, nbytes, out)) { - return nbytes; - } - size_t bytes_read = 0; - Status st = _file_reader->read_at( - static_cast(position), - Slice(static_cast(out), static_cast(nbytes)), &bytes_read, - _io_ctx); - if (!st.ok()) { - return arrow::Status::IOError(st.to_string_no_stack()); - } - insert_page_cache(position, nbytes, out, bytes_read); - return static_cast(bytes_read); - } +Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, + bool enable_page_cache, const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) { + DORIS_CHECK(input_file_reader != nullptr); + if (detail::should_stage_small_http_file(input_file_reader->path().native(), + input_file_reader->size(), + config::in_memory_file_size)) { + // A metadata-cache hit can make the first physical read start inside a tiny HTTP file. + // Read it from byte zero once so EOF-range quirks cannot make warm scans less reliable + // than cold scans, while keeping this compatibility policy entirely inside v2. + native_file = std::make_shared(std::move(input_file_reader)); + } else { + native_file = std::move(input_file_reader); + } + native_io_ctx = io_ctx; + + // Footer and page bytes must use the same stable identity. In particular, fs_name separates + // identical HDFS paths from different nameservices, while an unknown mutable version bypasses + // both caches rather than reusing an overwritten file of the same size. + auto* meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + auto meta_cache_key = build_page_cache_file_key(*native_file, file_description); + const bool has_stable_meta_cache_identity = !meta_cache_key.empty(); + if (has_stable_meta_cache_identity) { + // The discriminator prevents a v1 metadata value from being cast as the v2-owned type; + // schema mapping flags participate because they change the cached native schema tree. + meta_cache_key.append("\0v2", 3); + meta_cache_key.push_back(static_cast(enable_mapping_varbinary)); + meta_cache_key.push_back(static_cast(enable_mapping_timestamp_tz)); + } + size_t native_footer_size = 0; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled() && + meta_cache->lookup(meta_cache_key, &native_meta_cache_handle)) { + native_metadata = native_meta_cache_handle.data(); + ++native_footer_cache_hits; + } else { + RETURN_IF_ERROR(parse_native_parquet_footer( + native_file, &native_metadata_owner, &native_footer_size, io_ctx, + enable_mapping_varbinary, enable_mapping_timestamp_tz)); + ++native_footer_read_calls; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled()) { + meta_cache->insert(meta_cache_key, native_metadata_owner.release(), + &native_meta_cache_handle); + native_metadata = native_meta_cache_handle.data(); + } else { + native_metadata = native_metadata_owner.get(); + } + } + DORIS_CHECK(native_metadata != nullptr); + + auto page_cache_file_key = build_page_cache_file_key(*native_file, file_description); + native_page_cache_enabled = enable_page_cache && !page_cache_file_key.empty(); + // Native page readers use the FileDescription-derived immutable identity directly. + native_page_cache_file_key = page_cache_file_key; + return Status::OK(); +} - arrow::Result> ReadAt(int64_t position, - int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(position, nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; +Status ParquetFileContext::load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const { + DORIS_CHECK(offset_indexes != nullptr); + offset_indexes->clear(); + if (leaf_column_ids.empty()) { + return Status::OK(); } - - void register_page_cache_ranges(std::vector ranges) { - std::lock_guard lock(_page_cache_mutex); - _page_cache_ranges = std::move(ranges); + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for OffsetIndex", row_group_id); } - - void prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { - auto cached_reader = cached_remote_file_reader(); - if (cached_reader == nullptr) { - return; - } - const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : _io_ctx; - for (const auto& range : ranges) { - if (range.offset < 0 || range.size <= 0) { + const auto& native_row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + try { + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(native_row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for OffsetIndex", + leaf_column_id); + } + const auto& column_chunk = native_row_group.columns[leaf_column_id]; + if (!column_chunk.__isset.offset_index_offset || + !column_chunk.__isset.offset_index_length || + column_chunk.offset_index_length <= 0) { continue; } - cached_reader->prefetch_range(static_cast(range.offset), - static_cast(range.size), prefetch_io_ctx); - } - } - - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - reset_active_file_reader(); - const auto valid_ranges = detail::valid_prefetch_ranges(ranges); - if (!detail::should_use_merge_range_reader( - valid_ranges, avg_io_size, - typeid_cast(_base_file_reader.get()) != nullptr)) { - return false; - } - - std::vector random_access_ranges; - random_access_ranges.reserve(valid_ranges.size()); - for (const auto& range : valid_ranges) { - random_access_ranges.emplace_back(static_cast(range.offset), - static_cast(range.end_offset())); - } - - // This mirrors the v1 parquet reader: when projected column chunks in a row group are - // small random IOs, make the actual ReadAt path range-aware. Arrow still drives decoding, - // but every page read below this point sees MergeRangeFileReader instead of the raw remote - // reader, so adjacent small requests can be coalesced and served from merge buffers. - // Example: a row group projects leaf chunks [1MB, 1.5MB) and [1.6MB, 2MB). Arrow later - // issues page reads inside those chunks; MergeRangeFileReader can fetch a wider slice once - // and satisfy the following ReadAt calls from its boxes, reducing remote request count. - _merge_range_active = true; - set_active_file_reader(std::make_shared( - profile, _base_file_reader, random_access_ranges, merge_read_slice_size)); - return true; - } - - void reset_random_access_ranges() { reset_active_file_reader(); } - - ParquetPageCacheStats page_cache_stats() const { - std::lock_guard lock(_page_cache_mutex); - return _page_cache_stats; - } - -private: - bool page_cache_enabled() const { - return _enable_page_cache && !config::disable_storage_page_cache && - StoragePageCache::instance() != nullptr && !_page_cache_file_key.empty() && - _cached_page_range_index != nullptr; - } - - bool range_in_page_cache_scope(int64_t position, int64_t nbytes) const { - if (nbytes <= 0) { - return false; - } - const int64_t end = position + nbytes; - for (const auto& range : _page_cache_ranges) { - const int64_t range_end = range.offset + range.size; - if (position >= range.offset && end <= range_end) { - return true; + const int64_t index_offset = column_chunk.offset_index_offset; + const int64_t index_length = column_chunk.offset_index_length; + if (!detail::is_serialized_index_range_safe(native_file->size(), index_offset, + index_length)) { + // OffsetIndex is optional. A malformed range must not allocate from untrusted + // footer values or redirect the native reader outside the file. + continue; + } + std::vector serialized_index(static_cast(index_length)); + Slice index_slice(serialized_index.data(), serialized_index.size()); + size_t bytes_read = 0; + if (!native_file->read_at(index_offset, index_slice, &bytes_read, native_io_ctx).ok() || + bytes_read != serialized_index.size()) { + continue; + } + uint32_t thrift_length = static_cast(serialized_index.size()); + tparquet::OffsetIndex native_index; + if (!deserialize_thrift_msg(serialized_index.data(), &thrift_length, true, + &native_index) + .ok() || + native_index.page_locations.empty()) { + continue; } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + native_row_group.columns[leaf_column_id].meta_data, native_file->size(), + compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index( + native_index, chunk_range, + native_row_group.columns[leaf_column_id].meta_data.data_page_offset, + native_row_group.num_rows)) { + // OffsetIndex is optional. Reject the complete index instead of letting one bad + // location redirect an indexed reader outside its owning column chunk. + continue; + } + offset_indexes->emplace(leaf_column_id, std::move(native_index)); } - return false; + } catch (const std::exception&) { + // OffsetIndex is optional. Selected logical ranges still enforce correctness, while the + // native reader conservatively falls back to sequential page traversal. + offset_indexes->clear(); } + return Status::OK(); +} - StoragePageCache::CacheKey page_cache_key(int64_t position, int64_t nbytes) const { - return StoragePageCache::CacheKey(_page_cache_file_key, - static_cast(position + nbytes), position); +Status ParquetFileContext::load_native_page_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, int64_t* read_time, + int64_t* parse_time) const { + DORIS_CHECK(page_indexes != nullptr); + page_indexes->clear(); + if (leaf_column_ids.empty()) { + return Status::OK(); } - - bool copy_cached_range(const ParquetPageCacheRange& cached_range, int64_t copy_position, - int64_t copy_size, void* out, int64_t output_offset) { - PageCacheHandle handle; - if (!StoragePageCache::instance()->lookup( - page_cache_key(cached_range.offset, cached_range.size), &handle, - segment_v2::DATA_PAGE)) { - _cached_page_range_index->erase(cached_range); - return false; + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for PageIndex", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + + struct SerializedIndexRange { + int leaf_column_id; + int64_t offset; + int64_t length; + }; + struct PendingPageIndex { + NativeParquetPageIndex indexes; + bool has_column_index = false; + bool has_offset_index = false; + }; + std::vector column_index_ranges; + std::vector offset_index_ranges; + std::unordered_map pending_indexes; + + auto valid_index_range = [&](int64_t offset, int64_t length) { + return detail::is_serialized_index_range_safe(native_file->size(), offset, length); + }; + + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for PageIndex", leaf_column_id); + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (!chunk.__isset.column_index_offset || !chunk.__isset.column_index_length || + !chunk.__isset.offset_index_offset || !chunk.__isset.offset_index_length) { + continue; } - Slice cached = handle.data(); - const int64_t cache_offset = copy_position - cached_range.offset; - DORIS_CHECK(cache_offset >= 0); - DORIS_CHECK(cached.size >= static_cast(cache_offset + copy_size)); - memcpy(static_cast(out) + output_offset, cached.data + cache_offset, - static_cast(copy_size)); - return true; - } - - bool try_read_from_cached_ranges(int64_t position, int64_t nbytes, void* out) { - auto plan = detail::plan_page_cache_range_read(position, nbytes, - _cached_page_range_index->ranges()); - if (plan.empty()) { - return false; + if (!valid_index_range(chunk.column_index_offset, chunk.column_index_length) || + !valid_index_range(chunk.offset_index_offset, chunk.offset_index_length)) { + continue; } - for (const auto& entry : plan) { - if (!copy_cached_range(entry.cached_range, - entry.cached_range.offset + entry.copy_offset_in_cache, - entry.copy_size, out, entry.output_offset)) { - return false; + column_index_ranges.push_back( + {leaf_column_id, chunk.column_index_offset, chunk.column_index_length}); + offset_index_ranges.push_back( + {leaf_column_id, chunk.offset_index_offset, chunk.offset_index_length}); + pending_indexes.try_emplace(leaf_column_id); + } + + auto read_coalesced_indexes = [&](std::vector* ranges, + bool column_index) { + std::sort(ranges->begin(), ranges->end(), + [](const auto& lhs, const auto& rhs) { return lhs.offset < rhs.offset; }); + size_t range_begin = 0; + while (range_begin < ranges->size()) { + size_t range_end = range_begin + 1; + int64_t span_end = (*ranges)[range_begin].offset + (*ranges)[range_begin].length; + while (range_end < ranges->size() && (*ranges)[range_end].offset <= span_end) { + span_end = std::max(span_end, + (*ranges)[range_end].offset + (*ranges)[range_end].length); + ++range_end; } - } - return true; - } - bool try_read_from_page_cache(int64_t position, int64_t nbytes, void* out) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes)) { - return false; - } - ++_page_cache_stats.read_count; - // Fast path: Arrow issues the same ReadAt(offset, size) again, so the exact - // StoragePageCache key matches. - // Fallback path: Arrow may read a different but related byte range on another scan. - // Examples: - // - Current request [120, 150) can be served from cached [100, 200) by copying the - // 30-byte subset starting at cached offset 20. - // - Current request [100, 260) can be served by stitching cached [100, 180) and - // [180, 260). If any middle span is missing, it is a miss and the file reader fills - // the whole request from storage. - if (!copy_cached_range(ParquetPageCacheRange {position, nbytes}, position, nbytes, out, - 0) && - !try_read_from_cached_ranges(position, nbytes, out)) { - ++_page_cache_stats.miss_count; - return false; - } - ++_page_cache_stats.hit_count; - ++_page_cache_stats.compressed_hit_count; - return true; - } - - void insert_page_cache(int64_t position, int64_t nbytes, const void* data, size_t bytes_read) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes) || - bytes_read != static_cast(nbytes)) { - return; + const int64_t span_offset = (*ranges)[range_begin].offset; + if (!detail::is_serialized_index_span_safe(span_offset, span_end)) { + // Optional indexes share one allocation per contiguous footer block. Skipping the + // whole block prevents many individually small ranges from bypassing the budget. + range_begin = range_end; + continue; + } + const int64_t span_length = span_end - span_offset; + std::vector serialized(static_cast(span_length)); + Slice slice(serialized.data(), serialized.size()); + size_t bytes_read = 0; + Status read_status; + int64_t read_time_sink = 0; + { + SCOPED_RAW_TIMER(read_time == nullptr ? &read_time_sink : read_time); + read_status = native_file->read_at(span_offset, slice, &bytes_read, native_io_ctx); + } + if (read_status.ok() && bytes_read == serialized.size()) { + for (size_t i = range_begin; i < range_end; ++i) { + const auto& range = (*ranges)[i]; + auto pending = pending_indexes.find(range.leaf_column_id); + if (pending == pending_indexes.end()) { + continue; + } + uint32_t thrift_length = static_cast(range.length); + const auto* thrift_data = + serialized.data() + static_cast(range.offset - span_offset); + int64_t parse_time_sink = 0; + SCOPED_RAW_TIMER(parse_time == nullptr ? &parse_time_sink : parse_time); + if (column_index) { + pending->second.has_column_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.column_index) + .ok(); + } else { + pending->second.has_offset_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.offset_index) + .ok(); + } + } + } + range_begin = range_end; } - auto* page = new DataPage(bytes_read, true, segment_v2::DATA_PAGE); - memcpy(page->data(), data, bytes_read); - PageCacheHandle handle; - StoragePageCache::instance()->insert(page_cache_key(position, nbytes), page, &handle, - segment_v2::DATA_PAGE); - _cached_page_range_index->insert( - ParquetPageCacheRange {.offset = position, .size = nbytes}); - ++_page_cache_stats.write_count; - ++_page_cache_stats.compressed_write_count; - } + }; - void set_active_file_reader(io::FileReaderSPtr reader) { - DORIS_CHECK(reader != nullptr); - _file_reader = _file_reader_stats != nullptr - ? std::make_shared(std::move(reader), - _file_reader_stats) - : std::move(reader); - } + // Parquet writers place each index kind in a contiguous block. Reading overlapping/adjacent + // ranges as one span keeps cold small-file planning from paying two remote round trips per + // projected leaf, while refusing gaps avoids amplifying reads from untrusted footer offsets. + read_coalesced_indexes(&column_index_ranges, true); + read_coalesced_indexes(&offset_index_ranges, false); - void reset_active_file_reader() { - collect_active_merge_range_profile(); - _merge_range_active = false; - set_active_file_reader(_base_file_reader); - } - - void collect_active_merge_range_profile() { - if (_merge_range_active && _file_reader != nullptr) { - // MergeRangeFileReader writes its MergedSmallIO counters only from - // collect_profile_before_close(). v2 replaces the active reader for every row group, - // so collect before overwriting it; Close() handles the final row group. Example: - // RG0 installs a merge reader, RG1 calls set_random_access_ranges() and resets the - // active reader first, so RG0's RequestIO/MergedIO counters must be flushed here. - _file_reader->collect_profile_before_close(); - } - } - - std::shared_ptr cached_remote_file_reader() { - if (_merge_range_active) { - return nullptr; - } - auto reader = _file_reader; - if (reader == nullptr) { - return nullptr; + for (auto& [leaf_column_id, pending] : pending_indexes) { + const auto& chunk = row_group.columns[leaf_column_id]; + auto& indexes = pending.indexes; + if (!pending.has_column_index || !pending.has_offset_index || + indexes.column_index.null_pages.size() != indexes.offset_index.page_locations.size()) { + continue; } - // FileReader::init wraps the physical reader with TracingFileReader when scan IO stats are - // enabled. Prefetch should target the physical cached reader below that tracing wrapper, - // otherwise v2 scans with profiling would silently lose prefetch. - if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { - reader = tracing_reader->inner_reader(); + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, native_file->size(), compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index(indexes.offset_index, chunk_range, + chunk.meta_data.data_page_offset, row_group.num_rows)) { + continue; } - return std::dynamic_pointer_cast(reader); + page_indexes->emplace(leaf_column_id, std::move(indexes)); } + return Status::OK(); +} - io::FileReaderSPtr _file_reader; - io::FileReaderSPtr _base_file_reader; - io::FileReaderStats* _file_reader_stats = nullptr; - io::IOContext* _io_ctx = nullptr; - int64_t _pos = 0; - bool _closed = false; - bool _enable_page_cache = false; - bool _merge_range_active = false; - std::string _page_cache_file_key; - mutable std::mutex _page_cache_mutex; - std::vector _page_cache_ranges; - std::shared_ptr _cached_page_range_index; - ParquetPageCacheStats _page_cache_stats; -}; - -} // namespace - -Status arrow_status_to_doris_status(const arrow::Status& status) { - if (status.ok()) { - return Status::OK(); +void ParquetFileContext::prefetch_ranges(const std::vector& ranges, + const io::IOContext* io_ctx) { + io::FileReaderSPtr reader = native_file; + if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { + reader = tracing_reader->inner_reader(); } - if (status.IsIOError()) { - return Status::IOError(status.ToString()); + auto cached_reader = std::dynamic_pointer_cast(reader); + if (cached_reader == nullptr) { + return; } - if (status.IsInvalid()) { - return Status::InvalidArgument(status.ToString()); + const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : native_io_ctx; + for (const auto& range : detail::valid_prefetch_ranges(ranges)) { + cached_reader->prefetch_range(cast_set(range.offset), cast_set(range.size), + prefetch_io_ctx); } - return Status::InternalError(status.ToString()); } -Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, - bool enable_page_cache, - const io::FileDescription& file_description) { - DORIS_CHECK(input_file_reader != nullptr); - auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); - arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, - enable_page_cache, - std::move(page_cache_file_key)); - try { - // TODO: Cache parquet metadata in file system layer to avoid repeated metadata read for same file. - this->file_reader = ::parquet::ParquetFileReader::Open( - arrow_file, ::parquet::default_reader_properties()); - metadata = this->file_reader->metadata(); - schema = metadata != nullptr ? metadata->schema() : nullptr; - } catch (const ::parquet::ParquetException& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet file: {}", e.what()); - } catch (const std::exception& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet file: {}", e.what()); +bool ParquetFileContext::set_native_random_access_ranges( + const std::vector& ranges, size_t avg_io_size, + RuntimeProfile* profile, int64_t merge_read_slice_size) { + DORIS_CHECK(native_file != nullptr); + if (!detail::should_use_merge_range_reader( + ranges, avg_io_size, + typeid_cast(native_file.get()) != nullptr)) { + native_row_group_file = native_file; + return false; } - if (metadata == nullptr || schema == nullptr) { - return Status::Corruption("Failed to read parquet metadata"); + const auto valid_ranges = detail::valid_prefetch_ranges(ranges); + std::vector native_ranges; + native_ranges.reserve(valid_ranges.size()); + for (const auto& range : valid_ranges) { + native_ranges.emplace_back(cast_set(range.offset), + cast_set(range.end_offset())); } - return Status::OK(); -} - -void ParquetFileContext::register_page_cache_ranges(std::vector ranges) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get()) - ->register_page_cache_ranges(std::move(ranges)); -} - -void ParquetFileContext::prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->prefetch_ranges(ranges, io_ctx); -} - -bool ParquetFileContext::set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - DORIS_CHECK(arrow_file != nullptr); - return static_cast(arrow_file.get()) - ->set_random_access_ranges(ranges, avg_io_size, profile, merge_read_slice_size); + std::ranges::sort(native_ranges, {}, &io::PrefetchRange::start_offset); + native_row_group_file = std::make_shared( + profile, native_file, native_ranges, merge_read_slice_size); + return true; } void ParquetFileContext::reset_random_access_ranges() { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->reset_random_access_ranges(); + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); } ParquetPageCacheStats ParquetFileContext::page_cache_stats() const { - if (arrow_file == nullptr) { - return {}; - } - return static_cast(arrow_file.get())->page_cache_stats(); + return {}; } Status ParquetFileContext::close() { - if (file_reader != nullptr) { - try { - file_reader->Close(); - } catch (const std::exception&) { - } - } - if (arrow_file != nullptr) { - static_cast(arrow_status_to_doris_status(arrow_file->Close())); - } - file_reader.reset(); - arrow_file.reset(); + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); + native_metadata = nullptr; + native_metadata_owner.reset(); + native_meta_cache_handle = {}; + native_file.reset(); + native_io_ctx = nullptr; + native_page_cache_enabled = false; + native_page_cache_file_key.clear(); return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 67e9102139dbf2..0cd413e10557ac 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -15,19 +15,21 @@ #pragma once -#include -#include +#include #include #include #include -#include #include +#include #include +#include #include #include "common/status.h" +#include "format_v2/parquet/native_schema_desc.h" #include "io/fs/file_reader.h" +#include "util/obj_lru_cache.h" namespace doris::io { struct FileDescription; @@ -40,6 +42,26 @@ class RuntimeProfile; namespace doris::format::parquet { +struct NativeParquetPageIndex; + +// V2-owned footer/schema tree. Production planning and decoding consume this object directly; +// Arrow metadata is intentionally not materialized from the serialized footer. +class NativeParquetMetadata { +public: + NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size); + ~NativeParquetMetadata(); + + Status init_schema(bool enable_mapping_varbinary, bool enable_mapping_timestamp_tz); + const tparquet::FileMetaData& to_thrift() const { return _metadata; } + const NativeFieldDescriptor& schema() const { return _schema; } + size_t get_mem_size() const { return _parsed_size; } + +private: + tparquet::FileMetaData _metadata; + NativeFieldDescriptor _schema; + size_t _parsed_size = 0; +}; + struct ParquetPageCacheRange { int64_t offset = 0; int64_t size = 0; @@ -47,17 +69,6 @@ struct ParquetPageCacheRange { int64_t end_offset() const { return offset + size; } }; -struct ParquetPageCacheReadPlanEntry { - // The exact cached StoragePageCache entry. The final cache key is still exact-range based: - // file key + cached_range.end_offset() + cached_range.offset. - ParquetPageCacheRange cached_range; - // Byte offset inside cached_range to start copying from. - int64_t copy_offset_in_cache = 0; - // Byte offset inside the current ReadAt output buffer to start writing to. - int64_t output_offset = 0; - int64_t copy_size = 0; -}; - struct ParquetPageCacheStats { int64_t read_count = 0; int64_t write_count = 0; @@ -69,54 +80,19 @@ struct ParquetPageCacheStats { namespace detail { -class ParquetPageCacheRangeIndex { -public: - static constexpr size_t DEFAULT_MAX_RANGES = 65536; - - explicit ParquetPageCacheRangeIndex(size_t max_ranges = DEFAULT_MAX_RANGES); +inline constexpr int64_t MAX_SERIALIZED_PARQUET_INDEX_BYTES = 64LL << 20; - void insert(ParquetPageCacheRange range); - void erase(ParquetPageCacheRange range); +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit); - std::vector ranges() const; - size_t size() const; +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable); -private: - const size_t _max_ranges; - mutable std::mutex _mutex; - std::vector _ranges; -}; - -class ParquetPageCacheRangeDirectory { -public: - static constexpr size_t DEFAULT_MAX_FILES = 4096; - - explicit ParquetPageCacheRangeDirectory(size_t max_files = DEFAULT_MAX_FILES); +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length); - std::shared_ptr get_or_create(const std::string& file_key); - size_t size() const; - -private: - const size_t _max_files; - mutable std::mutex _mutex; - std::unordered_map> _indexes; -}; - -// Build the copy plan for a ReadAt(position, nbytes) request from the range metadata of -// previously cached entries. -// StoragePageCache cannot do range lookup by itself; it can only lookup an exact key. The -// caller therefore keeps lightweight cached range metadata and uses this function to decide -// which exact cache entries to fetch and which byte spans to copy. -// Examples: -// 1. Subset hit: -// request [120, 150), cached [100, 200) -> copy 30 bytes from cached offset 20. -// 2. Superset hit covered by multiple cached entries: -// request [100, 260), cached [100, 180) and [180, 260) -// -> two copies: [100, 180) to output offset 0, [180, 260) to output offset 80. -// 3. Partial overlap is a miss: -// request [100, 260), cached [100, 180) only -> empty plan, caller reads from file. -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges); +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end); // Keep only byte ranges that are safe to hand to FileReader implementations. Parquet metadata is // expected to contain non-negative offsets and positive compressed sizes, but tests and corrupted @@ -125,54 +101,74 @@ std::vector plan_page_cache_range_read( std::vector valid_prefetch_ranges( const std::vector& ranges); -// Average projected column-chunk size for one row group. The v1 parquet path uses this value to -// decide whether a row group is dominated by small random IOs; v2 uses the same signal before -// installing MergeRangeFileReader. Example: chunks of 512KB and 1MB average below SMALL_IO and are -// good merge-reader candidates, while two 8MB chunks should stay on the raw random-access reader. +// Average projected column-chunk size for one row group. V2 uses this signal to decide whether the +// row group is dominated by small random IOs before installing MergeRangeFileReader. Example: +// chunks of 512KB and 1MB average below SMALL_IO and are good merge-reader candidates, while two +// 8MB chunks should stay on the raw random-access reader. size_t average_prefetch_range_size(const std::vector& ranges); -// Decide whether Arrow ReadAt() should be routed through MergeRangeFileReader for the current row -// group. This is intentionally stricter than the background warm-up path: +// Decide whether native data-page ReadAt() should be routed through MergeRangeFileReader for the +// current row group. This is intentionally stricter than the background warm-up path: // - no valid projected chunks -> nothing to merge; // - in-memory file readers already avoid remote random IO; // - average chunk size >= MergeRangeFileReader::SMALL_IO would make merged reading wasteful. bool should_use_merge_range_reader(const std::vector& ranges, size_t avg_io_size, bool is_in_memory_reader); +// HTTP range servers may reject an overlong range near EOF even after accepting the capability +// probe. Staging a bounded small HTTP object also turns all native page reads into memory copies. +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size); + } // namespace detail struct ParquetFileContext { - std::shared_ptr arrow_file; // Arrow wrapper for Doris FileReader - std::unique_ptr<::parquet::ParquetFileReader> file_reader; // Arrow Parquet file parser - std::shared_ptr<::parquet::FileMetaData> metadata; // footer metadata (RowGroup information) - const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema + // Native metadata, index, and data-page paths share Doris' FileReader without transferring + // ownership to an external metadata tree. + io::FileReaderSPtr native_file; + // Row-group-scoped view of native_file. Small projected chunks use MergeRangeFileReader; + // large chunks and in-memory files keep native_file. + io::FileReaderSPtr native_row_group_file; + io::IOContext* native_io_ctx = nullptr; + // V2-owned Thrift footer/schema used to construct native page/encoding readers. A cache hit is + // owned by native_meta_cache_handle; a miss without cache is owned by native_metadata_owner. + const NativeParquetMetadata* native_metadata = nullptr; + std::unique_ptr native_metadata_owner; + ObjLRUCache::CacheHandle native_meta_cache_handle; + int64_t native_footer_read_calls = 0; + int64_t native_footer_cache_hits = 0; + bool native_page_cache_enabled = false; + std::string native_page_cache_file_key; Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, - const io::FileDescription& file_description); - // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page - // decoding, so v2 caches the serialized bytes read inside these ranges and excludes - // footer/metadata reads that happen before registration. - void register_page_cache_ranges(std::vector ranges); + const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); + Status load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const; + Status load_native_page_indexes(int row_group_id, + const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, + int64_t* read_time = nullptr, + int64_t* parse_time = nullptr) const; // Best-effort asynchronous warm-up for Parquet column chunks. This only has an effect when // the underlying Doris file reader is a CachedRemoteFileReader; other readers keep the same // random-access behavior and simply skip prefetch. void prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx); - // Switch the active reader used by Arrow ReadAt() to v1's MergeRangeFileReader when the current - // row group's projected column chunks are small random IOs. This is the real v1-compatible - // prefetch path: subsequent Arrow page reads go through the merged reader instead of merely - // warming file cache in the background. Returns true when merge-range reading is active. - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size); - // Restore Arrow ReadAt() to the base Doris file reader and flush any active merge-reader - // counters. Row-group setup uses this before dictionary-page probes, because those probes are - // a separate pass over the column chunk from the later Arrow RecordReader data-page stream. + // Install the row-group-scoped MergeRangeFileReader on the native data-page path. Dictionary + // probes must run before this method because their native ReadAt order is independent of the + // sequential projected chunk ranges consumed by MergeRangeFileReader. + bool set_native_random_access_ranges(const std::vector& ranges, + size_t avg_io_size, RuntimeProfile* profile, + int64_t merge_read_slice_size); + const io::FileReaderSPtr& native_data_file() const { + return native_row_group_file != nullptr ? native_row_group_file : native_file; + } + // Restore native ReadAt() to the base Doris file reader and flush merge-reader counters. void reset_random_access_ranges(); ParquetPageCacheStats page_cache_stats() const; Status close(); }; -Status arrow_status_to_doris_status(const arrow::Status& status); - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 29783e14a05071..9079b5f6060eb0 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -18,6 +18,7 @@ #include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/parquet_statistics.h" +#include "runtime/file_scan_profile.h" namespace doris::format::parquet { @@ -26,9 +27,13 @@ void ParquetProfile::init(RuntimeProfile* profile) { return; } + file_scan_profile::ensure_hierarchy(profile); static const char* parquet_profile = "ParquetReader"; - ADD_TIMER_WITH_LEVEL(profile, parquet_profile, 1); + total_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, parquet_profile, file_scan_profile::FILE_READER, 1); + // Row-group counters are part of the long-standing ParquetReader profile contract. Keep them + // below the format node so profile parsers and operators can attribute pruning to Parquet. filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsFiltered", TUnit::UNIT, parquet_profile, 1); filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -55,10 +60,16 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::BYTES, parquet_profile, 1); selected_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRows", TUnit::UNIT, parquet_profile, 1); + // Keep every Parquet scan metric below the format node: profile consumers extract that + // subtree and otherwise silently lose this counter even though filtering happened. rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByConjunct", TUnit::UNIT, parquet_profile, 1); total_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); + dense_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DenseBatches", TUnit::UNIT, parquet_profile, 1); + selected_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedBatches", TUnit::UNIT, + parquet_profile, 1); empty_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptySelectionBatches", TUnit::UNIT, parquet_profile, 1); range_gap_skipped_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RangeGapSkippedRows", @@ -69,14 +80,30 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); reader_select_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSelectRows", TUnit::UNIT, parquet_profile, 1); - arrow_read_records_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowReadRecordsTime", parquet_profile, 1); - arrow_skip_records_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowSkipRecordsTime", parquet_profile, 1); + level_only_read_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlyReadTime", parquet_profile, 1); + level_only_skip_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlySkipTime", parquet_profile, 1); materialization_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); + hybrid_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionBatches", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionRanges", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_null_fallback_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "HybridSelectionNullFallbackBatches", TUnit::UNIT, parquet_profile, 1); + native_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativeReadCalls", TUnit::UNIT, + parquet_profile, 1); + native_page_fragments = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativePageFragments", + TUnit::UNIT, parquet_profile, 1); + page_crossing_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCrossingBatches", + TUnit::UNIT, parquet_profile, 1); + nested_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NestedBatches", TUnit::UNIT, parquet_profile, 1); lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", TUnit::UNIT, parquet_profile, 1); + // Format-specific counters stay below ParquetReader so subtree consumers cannot silently + // attribute them to a different file format initialized on the same RuntimeProfile. filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, parquet_profile, 1); raw_rows_read = @@ -88,7 +115,8 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, parquet_profile, 1); - page_index_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, 1); + page_index_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, + parquet_profile, 1); page_index_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexFilterTime", parquet_profile, 1); read_page_index_time = @@ -103,8 +131,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); row_group_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); - file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); - file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); + file_footer_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", + TUnit::UNIT, parquet_profile, 1); + file_footer_hit_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, + parquet_profile, 1); decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1); @@ -139,6 +169,16 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); predicate_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateFilterTime", parquet_profile, 1); + predicate_compaction_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateCompactionTime", parquet_profile, 1); + predicate_compaction_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionBytes", + TUnit::BYTES, parquet_profile, 1); + predicate_compaction_count = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionCount", + TUnit::UNIT, parquet_profile, 1); + plain_predicate_direct_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "PlainPredicateDirectBatches", TUnit::UNIT, parquet_profile, 1); + plain_predicate_direct_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PlainPredicateDirectRows", + TUnit::UNIT, parquet_profile, 1); dict_filter_rewrite_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterRewriteTime", parquet_profile, 1); dict_filter_expr_rewrite_time = @@ -157,7 +197,6 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); rows_filtered_by_dict_filter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByDictFilter", TUnit::UNIT, parquet_profile, 1); - convert_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ConvertTime", parquet_profile, 1); bloom_filter_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "BloomFilterReadTime", parquet_profile, 1); } @@ -176,12 +215,42 @@ void ParquetProfile::update_pruning_stats(const ParquetPruningStats& pruning_sta COUNTER_UPDATE(total_row_groups, pruning_stats.total_row_groups); COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); + COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); + COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); + COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); + COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); + COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); + COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); + COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); + COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); + COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); +} + +void ParquetProfile::update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const { + const int64_t filtered = selected ? 0 : 1; + COUNTER_UPDATE(filtered_row_groups, filtered); + COUNTER_UPDATE(filtered_row_groups_by_dictionary, + pruning_stats.filtered_row_groups_by_dictionary); + COUNTER_UPDATE(filtered_row_groups_by_bloom_filter, + pruning_stats.filtered_row_groups_by_bloom_filter); + COUNTER_UPDATE(filtered_row_groups_by_page_index, + pruning_stats.filtered_row_groups_by_page_index); + // Initial footer planning counts every surviving candidate as readable. Lazy probes correct + // that estimate only when a later dictionary, Bloom, or page-index check removes the group. + COUNTER_UPDATE(to_read_row_groups, -filtered); + COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); + COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); @@ -199,9 +268,35 @@ ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { .reader_read_rows = reader_read_rows, .reader_skip_rows = reader_skip_rows, .reader_select_rows = reader_select_rows, - .arrow_read_records_time = arrow_read_records_time, - .arrow_skip_records_time = arrow_skip_records_time, + .level_only_read_time = level_only_read_time, + .level_only_skip_time = level_only_skip_time, .materialization_time = materialization_time, + .hybrid_selection_batches = hybrid_selection_batches, + .hybrid_selection_ranges = hybrid_selection_ranges, + .hybrid_selection_null_fallback_batches = hybrid_selection_null_fallback_batches, + .decompress_time = decompress_time, + .decompress_count = decompress_cnt, + .decode_header_time = decode_header_time, + .decode_value_time = decode_value_time, + .decode_dictionary_time = decode_dict_time, + .decode_level_time = decode_level_time, + .decode_null_map_time = decode_null_map_time, + .page_index_read_calls = page_index_read_calls, + .skip_page_header_count = skip_page_header_num, + .parse_page_header_count = parse_page_header_num, + .read_page_header_time = read_page_header_time, + .page_read_count = page_read_counter, + .page_cache_write_count = page_cache_write_counter, + .page_cache_compressed_write_count = page_cache_compressed_write_counter, + .page_cache_decompressed_write_count = page_cache_decompressed_write_counter, + .page_cache_hit_count = page_cache_hit_counter, + .page_cache_miss_count = page_cache_missing_counter, + .page_cache_compressed_hit_count = page_cache_compressed_hit_counter, + .page_cache_decompressed_hit_count = page_cache_decompressed_hit_counter, + .native_read_calls = native_read_calls, + .native_page_fragments = native_page_fragments, + .page_crossing_batches = page_crossing_batches, + .nested_batches = nested_batches, }; } @@ -212,10 +307,17 @@ ParquetScanProfile ParquetProfile::scan_profile() const { .rows_filtered_by_conjunct = rows_filtered_by_conjunct, .lazy_read_filtered_rows = lazy_read_filtered_rows, .total_batches = total_batches, + .dense_batches = dense_batches, + .selected_batches = selected_batches, .empty_selection_batches = empty_selection_batches, .range_gap_skipped_rows = range_gap_skipped_rows, .column_read_time = column_read_time, .predicate_filter_time = predicate_filter_time, + .predicate_compaction_time = predicate_compaction_time, + .predicate_compaction_bytes = predicate_compaction_bytes, + .predicate_compaction_count = predicate_compaction_count, + .plain_predicate_direct_batches = plain_predicate_direct_batches, + .plain_predicate_direct_rows = plain_predicate_direct_rows, .dict_filter_rewrite_time = dict_filter_rewrite_time, .dict_filter_expr_rewrite_time = dict_filter_expr_rewrite_time, .dict_filter_read_dict_time = dict_filter_read_dict_time, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 827cac45d94df0..f1b06be19d60c4 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -31,28 +31,65 @@ struct ParquetPageSkipProfile { // ============================================================================ // ============================================================================ struct ParquetColumnReaderProfile { - RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() - RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() - RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() - RuntimeProfile::Counter* arrow_read_records_time = nullptr; // Arrow RecordReader time (ns) - RuntimeProfile::Counter* arrow_skip_records_time = nullptr; // Arrow SkipRecords time (ns) - RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() + RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() + RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() + // COUNT(nullable_col) shape-only path; ordinary scans keep both counters zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; + RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; + // Native page/encoding reader internals. These counters keep page IO, decompression, levels, + // value decode and conversion attributable to separate stages. + RuntimeProfile::Counter* decompress_time = nullptr; + RuntimeProfile::Counter* decompress_count = nullptr; + RuntimeProfile::Counter* decode_header_time = nullptr; + RuntimeProfile::Counter* decode_value_time = nullptr; + RuntimeProfile::Counter* decode_dictionary_time = nullptr; + RuntimeProfile::Counter* decode_level_time = nullptr; + RuntimeProfile::Counter* decode_null_map_time = nullptr; + RuntimeProfile::Counter* page_index_read_calls = nullptr; + RuntimeProfile::Counter* skip_page_header_count = nullptr; + RuntimeProfile::Counter* parse_page_header_count = nullptr; + RuntimeProfile::Counter* read_page_header_time = nullptr; + RuntimeProfile::Counter* page_read_count = nullptr; + RuntimeProfile::Counter* page_cache_write_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_miss_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_hit_count = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; // native column-reader calls + RuntimeProfile::Counter* native_page_fragments = nullptr; // page-bounded read fragments + RuntimeProfile::Counter* page_crossing_batches = nullptr; // batches spanning multiple pages + RuntimeProfile::Counter* nested_batches = nullptr; // complex-column read batches }; // ============================================================================ // ============================================================================ struct ParquetScanProfile { - RuntimeProfile::Counter* raw_rows_read = nullptr; // raw rows read from RecordReader + RuntimeProfile::Counter* raw_rows_read = nullptr; // logical rows consumed before filtering RuntimeProfile::Counter* selected_rows = nullptr; // rows selected after conjunct filtering RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; // rows filtered by conjuncts RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; // rows avoided by late materialization RuntimeProfile::Counter* total_batches = nullptr; // total batch count + RuntimeProfile::Counter* dense_batches = nullptr; // batches retaining every physical row + RuntimeProfile::Counter* selected_batches = + nullptr; // non-empty batches compacted by predicates RuntimeProfile::Counter* empty_selection_batches = nullptr; // empty batches after full filtering RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; // rows skipped by range gaps RuntimeProfile::Counter* column_read_time = nullptr; // column read time (ns) RuntimeProfile::Counter* predicate_filter_time = nullptr; // predicate filter time (ns) + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* plain_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* plain_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; // dictionary rewrite time (ns) RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; // expression/residual rewrite time (ns) @@ -73,11 +110,15 @@ struct ParquetScanProfile { struct ParquetProfile { void init(RuntimeProfile* profile); void update_pruning_stats(const ParquetPruningStats& pruning_stats) const; + void update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const; ParquetPageSkipProfile page_skip_profile() const; ParquetColumnReaderProfile column_reader_profile() const; ParquetScanProfile scan_profile() const; + RuntimeProfile::Counter* total_time = nullptr; + RuntimeProfile::Counter* filtered_row_groups = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_min_max = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_dictionary = nullptr; @@ -96,6 +137,8 @@ struct ParquetProfile { RuntimeProfile::Counter* selected_rows = nullptr; RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; RuntimeProfile::Counter* total_batches = nullptr; + RuntimeProfile::Counter* dense_batches = nullptr; + RuntimeProfile::Counter* selected_batches = nullptr; RuntimeProfile::Counter* empty_selection_batches = nullptr; RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; @@ -103,9 +146,17 @@ struct ParquetProfile { RuntimeProfile::Counter* reader_read_rows = nullptr; RuntimeProfile::Counter* reader_skip_rows = nullptr; RuntimeProfile::Counter* reader_select_rows = nullptr; - RuntimeProfile::Counter* arrow_read_records_time = nullptr; - RuntimeProfile::Counter* arrow_skip_records_time = nullptr; + // COUNT(nullable_col) shape-only path; ordinary scans keep these zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; + RuntimeProfile::Counter* native_page_fragments = nullptr; + RuntimeProfile::Counter* page_crossing_batches = nullptr; + RuntimeProfile::Counter* nested_batches = nullptr; RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; RuntimeProfile::Counter* filtered_bytes = nullptr; @@ -149,6 +200,11 @@ struct ParquetProfile { RuntimeProfile::Counter* parse_page_header_num = nullptr; RuntimeProfile::Counter* predicate_filter_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* plain_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* plain_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_read_dict_time = nullptr; @@ -158,7 +214,6 @@ struct ParquetProfile { RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; RuntimeProfile::Counter* dict_filter_read_failures = nullptr; RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; - RuntimeProfile::Counter* convert_time = nullptr; RuntimeProfile::Counter* bloom_filter_read_time = nullptr; }; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index cd84ab74ef953e..758ff8c5a1490b 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -39,7 +39,7 @@ #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/parquet_statistics.h" -#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/count_column_reader.h" #include "io/io_common.h" #include "runtime/runtime_state.h" @@ -57,69 +57,6 @@ struct ParquetReaderScanState { bool enable_strict_mode = false; }; -int64_t column_chunk_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); -} - -void collect_all_leaf_column_ids(const ParquetColumnSchema& column_schema, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { - if (column_schema.leaf_column_id >= 0) { - leaf_column_ids->insert(column_schema.leaf_column_id); - } - return; - } - for (const auto& child : column_schema.children) { - DORIS_CHECK(child != nullptr); - collect_all_leaf_column_ids(*child, leaf_column_ids); - } -} - -void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex& projection, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - if (projection.project_all_children || projection.children.empty()) { - collect_all_leaf_column_ids(column_schema, leaf_column_ids); - return; - } - for (const auto& child_projection : projection.children) { - const auto child_it = - std::ranges::find_if(column_schema.children, [&](const auto& child_schema) { - return child_schema->local_id == child_projection.local_id(); - }); - DORIS_CHECK(child_it != column_schema.children.end()); - collect_projected_leaf_column_ids(**child_it, child_projection, leaf_column_ids); - } -} - -void collect_request_leaf_column_ids( - const std::vector>& file_schema, - const format::FileScanRequest& request, std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - auto collect_scan_column = [&](const format::LocalColumnIndex& projection) { - const auto local_id = projection.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID || - local_id == format::GLOBAL_ROWID_COLUMN_ID) { - return; - } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); - collect_projected_leaf_column_ids(*file_schema[local_id], projection, leaf_column_ids); - }; - for (const auto& column : request.predicate_columns) { - collect_scan_column(column); - } - for (const auto& column : request.non_predicate_columns) { - if (!request.is_count_star_placeholder(column.column_id())) { - collect_scan_column(column); - } - } -} - Status validate_all_projected_leaves_supported(const ParquetColumnSchema& column_schema) { if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { if (!column_schema.type_descriptor.unsupported_reason.empty()) { @@ -155,6 +92,8 @@ Status validate_projected_leaves_supported(const ParquetColumnSchema& column_sch Status validate_requested_columns_supported( const std::vector>& file_schema, const format::FileScanRequest& request) { + // Validate the projected native-schema leaves before pruning: checking a physical carrier at + // a later read site can let an unsupported logical type silently pass when every row is pruned. auto validate_scan_column = [&](const format::LocalColumnIndex& projection) -> Status { const auto local_id = projection.local_id(); if (local_id == format::ROW_POSITION_COLUMN_ID || @@ -176,33 +115,6 @@ Status validate_requested_columns_supported( return Status::OK(); } -std::vector build_page_cache_ranges( - const ::parquet::FileMetaData& metadata, - const std::vector>& file_schema, - const format::FileScanRequest& request, const RowGroupScanPlan& row_group_plan) { - std::unordered_set leaf_column_ids; - collect_request_leaf_column_ids(file_schema, request, &leaf_column_ids); - std::vector ranges; - ranges.reserve(row_group_plan.row_groups.size() * leaf_column_ids.size()); - for (const auto& row_group_plan_item : row_group_plan.row_groups) { - auto row_group_metadata = metadata.RowGroup(row_group_plan_item.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - for (const auto leaf_column_id : leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_chunk_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); - DORIS_CHECK(offset >= 0); - DORIS_CHECK(size >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); - } - } - } - return ranges; -} - const ParquetColumnSchema& projected_root_schema( const std::vector>& file_schema, const format::LocalColumnIndex& projection) { @@ -213,11 +125,10 @@ const ParquetColumnSchema& projected_root_schema( } int64_t count_loaded_non_null_values(const ParquetColumnSchema& root_schema, - const ParquetColumnReader& shape_reader, - int64_t expected_rows) { - const auto& def_levels = shape_reader.nested_definition_levels(); - const auto& rep_levels = shape_reader.nested_repetition_levels(); - const int64_t levels_written = shape_reader.nested_levels_written(); + const CountColumnReader& shape_reader, int64_t expected_rows) { + const auto& def_levels = shape_reader.definition_levels(); + const auto& rep_levels = shape_reader.repetition_levels(); + const int64_t levels_written = shape_reader.levels_written(); DORIS_CHECK(levels_written >= expected_rows); if (root_schema.max_repetition_level == 0) { DORIS_CHECK(levels_written == expected_rows); @@ -265,7 +176,7 @@ int timestamp_tz_scale(const ParquetTypeDescriptor& type_descriptor) { bool should_map_to_timestamp_tz(const ParquetColumnSchema& column_schema) { const auto& type_descriptor = column_schema.type_descriptor; - return type_descriptor.physical_type == ::parquet::Type::INT96 || + return type_descriptor.physical_type == tparquet::Type::INT96 || (type_descriptor.is_timestamp && type_descriptor.timestamp_is_adjusted_to_utc); } @@ -346,10 +257,9 @@ static Status find_projected_minmax_leaf(const ParquetColumnSchema& column_schem } static Status validate_minmax_aggregate_statistics(const ParquetColumnSchema& column_schema) { - DORIS_CHECK(column_schema.descriptor != nullptr); - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + switch (column_schema.type_descriptor.physical_type) { + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: // Arrow 17 does not expose Parquet's min/max exactness flags. Binary statistics may be // truncated bounds rather than values present in the file, so they are safe for pruning // but cannot be returned as exact aggregate results. @@ -386,14 +296,17 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _enable_mapping_varbinary(enable_mapping_varbinary) {} ParquetReader::~ParquetReader() = default; Status ParquetReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_parquet_profile.total_time); if (_io_ctx != nullptr && _io_ctx->should_stop) { return Status::EndOfFile("stop"); } @@ -414,6 +327,7 @@ Status ParquetReader::init(RuntimeState* state) { _state->enable_strict_mode = state->enable_strict_mode(); _state->scheduler.set_timezone(&state->timezone_obj()); _state->scheduler.set_enable_strict_mode(_state->enable_strict_mode); + _state->scheduler.set_runtime_state(state); } int64_t merge_read_slice_size = -1; if (state != nullptr && state->query_options().__isset.merge_read_slice_size) { @@ -421,16 +335,30 @@ Status ParquetReader::init(RuntimeState* state) { } _state->scheduler.set_merge_read_options(_profile, merge_read_slice_size); _state->scheduler.set_batch_size(_batch_size); - // Open parquet file and parse metadata to get file schema. - RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description)); + // Opening the file parses the footer before any row group can be scheduled. Keep this timer + // around the whole operation so footer/cache latency cannot disappear from a slow profile. + { + SCOPED_TIMER(_parquet_profile.parse_footer_time); + RETURN_IF_ERROR(_state->file_context.open( + _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, + _enable_mapping_timestamp_tz, _enable_mapping_varbinary)); + } + if (_profile != nullptr) { + COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, + _state->file_context.native_footer_read_calls); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, + _state->file_context.native_footer_cache_hits); + } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier - RETURN_IF_ERROR( - build_parquet_column_schema(*_state->file_context.schema, &_state->file_schema)); - if (_enable_mapping_timestamp_tz) { - for (auto& column_schema : _state->file_schema) { - apply_timestamp_tz_mapping(column_schema.get()); + { + SCOPED_TIMER(_parquet_profile.parse_meta_time); + RETURN_IF_ERROR(build_parquet_column_schema(_state->file_context.native_metadata->schema(), + &_state->file_schema)); + if (_enable_mapping_timestamp_tz) { + for (auto& column_schema : _state->file_schema) { + apply_timestamp_tz_mapping(column_schema.get()); + } } } return Status::OK(); @@ -444,11 +372,12 @@ void ParquetReader::set_batch_size(size_t batch_size) { } Status ParquetReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_parquet_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } file_schema->clear(); - if (_state == nullptr || _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } @@ -471,8 +400,8 @@ std::unique_ptr ParquetReader::create_column_mapper( } Status ParquetReader::open(std::shared_ptr request) { - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + SCOPED_TIMER(_parquet_profile.total_time); + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } auto request_snapshot = request; @@ -525,19 +454,20 @@ Status ParquetReader::open(std::shared_ptr request) { scan_range.file_size = _file_description->file_size; // Get selected ranges in row groups according to metadata (Row-Group level index and Page Index including Zonemap, Dictionary, Bloom Filter). RETURN_IF_ERROR(plan_parquet_row_groups( - *_state->file_context.metadata, _state->file_context.file_reader.get(), - _state->file_schema, *request_snapshot, scan_range, _state->enable_bloom_filter, - &row_group_plan, _state->timezone, _state->runtime_state)); + *_state->file_context.native_metadata, _state->file_schema, *request_snapshot, + scan_range, _state->enable_bloom_filter, &row_group_plan, _state->timezone, + _state->runtime_state, &_state->file_context, + _parquet_profile.column_reader_profile())); if (_profile != nullptr) { _parquet_profile.update_pruning_stats(row_group_plan.pruning_stats); } - if (_state->enable_page_cache) { - _state->file_context.register_page_cache_ranges( - build_page_cache_ranges(*_state->file_context.metadata, _state->file_schema, - *request_snapshot, row_group_plan)); - } + // Native page readers admit exact validated page payloads to cache. Do not pre-register whole + // column chunks here: footer offsets are untrusted and this obsolete range map is not consumed. _state->scan_plan = row_group_plan; _state->scheduler.set_page_skip_profile(_parquet_profile.page_skip_profile()); + if (_profile != nullptr) { + _state->scheduler.set_pruning_profile(&_parquet_profile); + } _state->scheduler.set_global_rowid_context(_global_rowid_context); _state->scheduler.set_scan_profile(_parquet_profile.scan_profile()); _state->scheduler.set_plan(std::move(row_group_plan)); @@ -546,8 +476,8 @@ Status ParquetReader::open(std::shared_ptr request) { } Status ParquetReader::get_block(Block* file_block, size_t* rows, bool* eof) { - if (_state == nullptr || _state->file_context.file_reader == nullptr || - _state->file_context.schema == nullptr) { + SCOPED_TIMER(_parquet_profile.total_time); + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } *rows = 0; @@ -646,9 +576,9 @@ int64_t ParquetReader::get_total_rows() const { Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_parquet_profile.total_time); DORIS_CHECK(result != nullptr); - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } if (_should_stop()) { @@ -662,6 +592,14 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r request.agg_type); } + // Aggregate pushdown bypasses the scheduler but still requires the exact pruned row-group set. + // Finish lazy remote probes here; normal scans keep them at current-row-group granularity. + RETURN_IF_ERROR(finalize_parquet_row_group_plans( + *_state->file_context.native_metadata, _state->file_schema, *_request, + _state->enable_bloom_filter, &_state->scan_plan, _state->timezone, + _state->runtime_state, &_state->file_context, _parquet_profile.column_reader_profile(), + _profile == nullptr ? nullptr : &_parquet_profile)); + for (const auto& aggregate_column : request.columns) { const auto local_id = aggregate_column.projection.local_id(); if (local_id < 0 || local_id >= static_cast(_state->file_schema.size())) { @@ -677,10 +615,9 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r // Aggregate row count in all selected row groups. For MIN/MAX aggregate, this is used to determine whether there is no row group selected. for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - result->count += row_group_metadata->num_rows(); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + result->count += row_group_metadata.num_rows; } if (request.agg_type == TPushAggOp::type::COUNT) { if (request.columns.empty()) { @@ -703,31 +640,14 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } result->count = 0; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - std::shared_ptr<::parquet::RowGroupReader> row_group; - try { - row_group = _state->file_context.file_reader->RowGroup(row_group_plan.row_group_id); - } catch (const ::parquet::ParquetException& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } catch (const std::exception& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } - - ParquetColumnReaderFactory column_reader_factory( - row_group, _state->file_context.schema->num_columns(), - &row_group_plan.page_skip_plans, _parquet_profile.page_skip_profile(), - _state->timezone, _state->enable_strict_mode, - _parquet_profile.scan_profile().column_reader_profile); - std::unique_ptr shape_reader; - RETURN_IF_ERROR(column_reader_factory.create_count_shape_reader( - root_schema, &count_projection, &shape_reader)); + std::unique_ptr shape_reader; + RETURN_IF_ERROR(CountColumnReader::create( + _state->file_context.native_data_file(), _state->file_context.native_metadata, + row_group_plan.row_group_id, root_schema, &count_projection, + _state->file_context.native_io_ctx, + _state->file_context.native_page_cache_enabled, + _state->file_context.native_page_cache_file_key, + _parquet_profile.scan_profile().column_reader_profile, &shape_reader)); DORIS_CHECK(shape_reader != nullptr); int64_t row_group_cursor = 0; @@ -741,18 +661,19 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r while (range_rows_read < selected_range.length) { const int64_t batch_rows = std::min(_batch_size, selected_range.length - range_rows_read); - // COUNT(col) only needs the top-level NULL state. The shape reader loads - // def/rep levels from one representative leaf and does not build value_indices - // or values_column. MAP chooses the key leaf; ARRAY/STRUCT may choose a string - // leaf, but the levels-only protocol still avoids Doris-side string - // materialization for that leaf. + int64_t rows_read = 0; RETURN_IF_ERROR(_stop_status_if_requested( - shape_reader->load_nested_levels_batch(batch_rows))); - _record_scan_rows(batch_rows); + shape_reader->read_levels(batch_rows, &rows_read))); + if (rows_read != batch_rows) { + return Status::Corruption( + "Parquet COUNT reader returned {} rows, expected {}", rows_read, + batch_rows); + } + _record_scan_rows(rows_read); result->count += - count_loaded_non_null_values(root_schema, *shape_reader, batch_rows); - range_rows_read += batch_rows; - row_group_cursor += batch_rows; + count_loaded_non_null_values(root_schema, *shape_reader, rows_read); + range_rows_read += rows_read; + row_group_cursor += rows_read; } } } @@ -779,13 +700,25 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r auto& aggregate_column = result->columns[request_column_idx]; aggregate_column.projection = request.columns[request_column_idx].projection; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - auto column_chunk = row_group_metadata->ColumnChunk(leaf_schema->leaf_column_id); - DORIS_CHECK(column_chunk != nullptr); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + DORIS_CHECK(leaf_schema->leaf_column_id >= 0 && + leaf_schema->leaf_column_id < + static_cast(row_group_metadata.columns.size())); + const auto& column_chunk = row_group_metadata.columns[leaf_schema->leaf_column_id]; + DORIS_CHECK(column_chunk.__isset.meta_data); + const auto& column_metadata = column_chunk.meta_data; + std::optional safe_statistics; + if (column_metadata.__isset.statistics) { + safe_statistics = detail::sanitize_native_footer_statistics( + leaf_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order( + _state->file_context.native_metadata->to_thrift(), + leaf_schema->leaf_column_id)); + } const auto statistics = ParquetStatisticsUtils::TransformColumnStatistics( - *leaf_schema, column_chunk->statistics(), _state->timezone); + *leaf_schema, safe_statistics.has_value() ? &*safe_statistics : nullptr, + column_metadata.num_values, _state->timezone); if (!statistics.has_min_max) { return Status::NotSupported("Missing parquet min/max statistics for column {}", leaf_schema->name); @@ -807,7 +740,9 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } Status ParquetReader::close() { + SCOPED_TIMER(_parquet_profile.total_time); if (_state != nullptr) { + _state->scheduler.close(); _sync_page_cache_profile(); RETURN_IF_ERROR(_state->file_context.close()); } diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 6c8e88cc27a9b6..da6135b81ae838 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -46,7 +46,7 @@ class ParquetReader : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); ~ParquetReader() override; Status init(RuntimeState* state) override; @@ -89,6 +89,7 @@ class ParquetReader : public format::FileReader { std::optional _global_rowid_context; // global RowId context size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ + bool _enable_mapping_varbinary = false; // whether raw BYTE_ARRAY is mapped to VARBINARY }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index eafb741a53161d..4a12153cd30845 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -15,11 +15,11 @@ #include "format_v2/parquet/parquet_scan.h" -#include - #include +#include #include #include +#include #include #include #include @@ -35,37 +35,93 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/global_rowid_column_reader.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native_column_reader.h" +#include "format_v2/parquet/reader/row_position_column_reader.h" +#include "util/defer_op.h" +#include "util/time.h" namespace doris::format::parquet { -namespace { +namespace detail { + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats) { + if (std::ranges::any_of(positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return positions; + } + auto ordered = positions; + std::stable_sort(ordered.begin(), ordered.end(), [&](size_t left, size_t right) { + const auto score = [&](size_t position) { + const auto& sample = stats.at(position); + return sample.cost_per_input_row_ns / std::max(1.0 - sample.survival_ratio, 0.01); + }; + return score(left) < score(right); + }); + return ordered; +} -int64_t column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability) { + if (std::ranges::any_of(ordered_positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return ordered_positions; + } + std::vector result; + double reach_probability = 1; + for (const size_t position : ordered_positions) { + if (!result.empty() && reach_probability < minimum_reach_probability) { + break; + } + result.push_back(position); + reach_probability *= stats.at(position).survival_ratio; + } + return result; } -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence) { + constexpr size_t WARMUP_SAMPLES = 8; + constexpr size_t STEADY_STATE_INTERVAL = 16; + return samples < WARMUP_SAMPLES || batch_sequence % STEADY_STATE_INTERVAL == 0; +} + +} // namespace detail + +namespace { + +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( + const format::FileScanRequest& request); + +bool is_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; } -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; +bool is_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; } -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; +bool is_data_page_type(tparquet::PageType::type page_type) { + return page_type == tparquet::PageType::DATA_PAGE || + page_type == tparquet::PageType::DATA_PAGE_V2; } -bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { +bool is_fully_dictionary_encoded_chunk(const tparquet::ColumnMetaData& column_metadata) { + if (!column_metadata.__isset.dictionary_page_offset || + column_metadata.dictionary_page_offset < 0) { return false; } - const auto& encoding_stats = column_metadata.encoding_stats(); + const auto& encoding_stats = column_metadata.encoding_stats; if (!encoding_stats.empty()) { bool has_dictionary_data_page = false; for (const auto& encoding_stat : encoding_stats) { @@ -81,7 +137,7 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { + for (const auto encoding : column_metadata.encodings) { if (is_dictionary_data_encoding(encoding)) { has_dictionary_encoding = true; continue; @@ -94,14 +150,18 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool supports_row_level_dictionary_filter(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr || + const tparquet::ColumnMetaData& column_metadata) { + if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || column_schema.type == nullptr || column_schema.max_repetition_level > 0) { return false; } if (!column_schema.type_descriptor.is_string_like || - column_metadata.type() != ::parquet::Type::BYTE_ARRAY) { + column_metadata.type != tparquet::Type::BYTE_ARRAY) { + return false; + } + if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_VARBINARY) { + // A table STRING predicate can be rewritten through a raw VARBINARY file slot. Evaluating + // it on dictionary Fields before the mapping expression is neither type-safe nor exact. return false; } // Row-level dictionary filtering consumes dictionary ids from DATA_PAGE payloads. It is exact @@ -143,35 +203,6 @@ void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, } } -bool is_row_group_outside_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, int row_group_idx) { - if (scan_range.size < 0) { - return false; - } - const int64_t range_start_offset = scan_range.start_offset; - const int64_t range_end_offset = range_start_offset + scan_range.size; - DORIS_CHECK(range_start_offset >= 0); - DORIS_CHECK(range_end_offset >= range_start_offset); - if (range_start_offset == 0 && - (scan_range.file_size < 0 || range_end_offset >= scan_range.file_size)) { - return false; - } - - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - DORIS_CHECK(row_group_metadata->num_columns() > 0); - const auto first_column = row_group_metadata->ColumnChunk(0); - const auto last_column = row_group_metadata->ColumnChunk(row_group_metadata->num_columns() - 1); - DORIS_CHECK(first_column != nullptr); - DORIS_CHECK(last_column != nullptr); - const int64_t row_group_start_offset = column_start_offset(*first_column); - const int64_t row_group_end_offset = - column_start_offset(*last_column) + last_column->total_compressed_size(); - const int64_t row_group_mid_offset = - row_group_start_offset + (row_group_end_offset - row_group_start_offset) / 2; - return row_group_mid_offset < range_start_offset || row_group_mid_offset >= range_end_offset; -} - std::vector request_scan_columns(const format::FileScanRequest& request) { std::vector scan_columns; scan_columns.reserve(request.predicate_columns.size() + request.non_predicate_columns.size()); @@ -212,10 +243,17 @@ void materialize_count_star_placeholders(const format::FileScanRequest& request, } } -std::vector build_row_group_prefetch_ranges( - const ::parquet::FileMetaData& metadata, +} // namespace + +namespace detail { + +Status build_native_prefetch_ranges( + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, - const std::vector& scan_columns, int row_group_idx) { + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); std::unordered_set leaf_column_ids; for (const auto& projection : scan_columns) { const auto local_id = projection.local_id(); @@ -223,92 +261,233 @@ std::vector build_row_group_prefetch_ranges( local_id == format::GLOBAL_ROWID_COLUMN_ID) { continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return Status::Corruption("Invalid Parquet projected column id {}", local_id); + } // Prefetch and merge-reader ranges must be physical leaf chunks, not Doris logical slots. // Example: for a struct column s, projecting only s.a should include only // the Parquet leaf chunk of a. Projecting the whole struct includes both a and b. collect_projected_leaf_column_ids(*file_schema[local_id], projection, &leaf_column_ids); } - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group index {}", row_group_idx); + } + const auto& row_group_metadata = metadata.row_groups[row_group_idx]; std::vector ordered_leaf_column_ids(leaf_column_ids.begin(), leaf_column_ids.end()); std::ranges::sort(ordered_leaf_column_ids); - std::vector ranges; - ranges.reserve(ordered_leaf_column_ids.size()); + ranges->reserve(ordered_leaf_column_ids.size()); for (const auto leaf_column_id : ordered_leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); - DORIS_CHECK(offset >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(row_group_metadata.columns.size())) { + return Status::Corruption("Invalid Parquet leaf column id {}", leaf_column_id); + } + const auto& chunk = row_group_metadata.columns[leaf_column_id]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet leaf column {} has no chunk metadata", + leaf_column_id); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range(chunk.meta_data, file_size, + parquet_816_padding, &chunk_range)); + if (chunk_range.length > 0) { + if (chunk_range.offset > static_cast(std::numeric_limits::max()) || + chunk_range.length > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet column chunk range exceeds int64 coordinates"); + } + // Prefetch must use the same checked chunk extent as the decoder, including the + // PARQUET-816 compatibility padding, so warm-up cannot target different bytes. + ranges->push_back( + ParquetPageCacheRange {.offset = static_cast(chunk_range.offset), + .size = static_cast(chunk_range.length)}); } } - return ranges; + return Status::OK(); } -Status select_row_groups_by_scan_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, - std::vector* row_group_first_rows, - std::vector* selected_row_groups) { - DORIS_CHECK(row_group_first_rows != nullptr); - DORIS_CHECK(selected_row_groups != nullptr); - row_group_first_rows->assign(metadata.num_row_groups(), 0); +} // namespace detail + +namespace detail { + +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups) { + DORIS_CHECK(row_group_first_rows != nullptr && selected_row_groups != nullptr); + if (scan_range.start_offset < 0 || scan_range.size < -1 || + (scan_range.size >= 0 && + scan_range.start_offset > std::numeric_limits::max() - scan_range.size)) { + return Status::Corruption("Invalid Parquet scan range [{}, {})", scan_range.start_offset, + scan_range.size); + } + const uint64_t range_start = static_cast(scan_range.start_offset); + const uint64_t range_end = scan_range.size < 0 + ? std::numeric_limits::max() + : range_start + static_cast(scan_range.size); + const size_t file_size = scan_range.file_size < 0 ? std::numeric_limits::max() + : static_cast(scan_range.file_size); + const bool full_file_range = + scan_range.size < 0 || (range_start == 0 && scan_range.file_size >= 0 && + range_end >= static_cast(scan_range.file_size)); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + row_group_first_rows->assign(metadata.row_groups.size(), 0); selected_row_groups->clear(); - selected_row_groups->reserve(metadata.num_row_groups()); - int64_t next_row_group_first_row = 0; - for (int row_group_idx = 0; row_group_idx < metadata.num_row_groups(); ++row_group_idx) { - (*row_group_first_rows)[row_group_idx] = next_row_group_first_row; - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows < 0) { + selected_row_groups->reserve(metadata.row_groups.size()); + int64_t next_first_row = 0; + for (size_t row_group_idx = 0; row_group_idx < metadata.row_groups.size(); ++row_group_idx) { + (*row_group_first_rows)[row_group_idx] = next_first_row; + const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { return Status::Corruption("Invalid negative row count in parquet row group {}", row_group_idx); } - next_row_group_first_row += row_group_rows; - if (!is_row_group_outside_range(metadata, scan_range, row_group_idx)) { - selected_row_groups->push_back(row_group_idx); + if (row_group.num_rows > std::numeric_limits::max() - next_first_row) { + return Status::Corruption("Parquet row counts overflow at row group {}", row_group_idx); + } + next_first_row += row_group.num_rows; + bool selected = full_file_range; + if (!full_file_range) { + if (row_group.columns.empty()) { + return Status::Corruption("Parquet row group {} has no column chunks", + row_group_idx); + } + size_t group_start = std::numeric_limits::max(); + size_t group_end = 0; + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, file_size, compat.parquet_816_padding, &chunk_range)); + group_start = std::min(group_start, chunk_range.offset); + group_end = std::max(group_end, chunk_range.offset + chunk_range.length); + } + // Checked chunk ranges make end >= start; this midpoint form cannot overflow even + // when footer offsets are close to the host coordinate limit. + const uint64_t group_mid = + static_cast(group_start) + (group_end - group_start) / 2; + selected = group_mid >= range_start && group_mid < range_end; + } + if (selected) { + selected_row_groups->push_back(cast_set(row_group_idx)); } } return Status::OK(); } -Status build_row_group_read_plans( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, +} // namespace detail + +namespace { + +std::vector intersect_row_ranges(const std::vector& left, + const std::vector& right) { + std::vector result; + size_t left_idx = 0; + size_t right_idx = 0; + while (left_idx < left.size() && right_idx < right.size()) { + const int64_t left_end = left[left_idx].start + left[left_idx].length; + const int64_t right_end = right[right_idx].start + right[right_idx].length; + const int64_t start = std::max(left[left_idx].start, right[right_idx].start); + const int64_t end = std::min(left_end, right_end); + if (start < end) { + result.push_back({.start = start, .length = end - start}); + } + if (left_end < right_end) { + ++left_idx; + } else { + ++right_idx; + } + } + return result; +} + +Status finalize_native_row_group_read_plan( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, + RowGroupReadPlan* row_group_plan, ParquetPruningStats* pruning_stats, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + bool* selected) { + DORIS_CHECK(row_group_plan != nullptr && pruning_stats != nullptr && file_context != nullptr && + selected != nullptr); + *selected = true; + if (!row_group_plan->expensive_pruning_pending) { + return Status::OK(); + } + row_group_plan->expensive_pruning_pending = false; + const auto& thrift = metadata.to_thrift(); + const std::vector candidate {row_group_plan->row_group_id}; + std::vector metadata_selected; + RETURN_IF_ERROR(select_row_groups_by_metadata( + thrift, file_schema, request, &candidate, &metadata_selected, enable_bloom_filter, + pruning_stats, timezone, runtime_state, file_context, column_reader_profile, + ParquetMetadataProbeMode::EXPENSIVE_ONLY)); + if (metadata_selected.empty()) { + *selected = false; + return Status::OK(); + } + + std::unordered_set requested_leaf_ids; + for (const auto& projection : request_scan_columns(request)) { + const auto local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + continue; + } + collect_projected_leaf_column_ids(*file_schema[local_id], projection, &requested_leaf_ids); + } + std::unordered_map page_indexes; + if (can_use_parquet_page_index(request, runtime_state)) { + RETURN_IF_ERROR(file_context->load_native_page_indexes( + row_group_plan->row_group_id, requested_leaf_ids, &page_indexes, + &pruning_stats->read_page_index_time, &pruning_stats->parse_page_index_time)); + } + std::vector page_selected_ranges; + std::map page_skip_plans; + RETURN_IF_ERROR(select_row_group_ranges_by_native_page_index( + thrift, page_indexes, file_schema, request, row_group_plan->row_group_rows, + &page_selected_ranges, &page_skip_plans, pruning_stats, timezone, runtime_state)); + row_group_plan->selected_ranges = + intersect_row_ranges(row_group_plan->selected_ranges, page_selected_ranges); + row_group_plan->page_skip_plans = std::move(page_skip_plans); + for (auto& [leaf_column_id, indexes] : page_indexes) { + row_group_plan->offset_indexes.emplace(leaf_column_id, std::move(indexes.offset_index)); + } + if (row_group_plan->selected_ranges.empty()) { + *selected = false; + return Status::OK(); + } + pruning_stats->selected_row_ranges += row_group_plan->selected_ranges.size(); + return Status::OK(); +} + +Status build_native_row_group_read_plans( + const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector& selected_row_groups, const std::vector& row_group_first_rows, RowGroupScanPlan* plan, - const cctz::time_zone* timezone, const RuntimeState* runtime_state) { - DORIS_CHECK(plan != nullptr); + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + const auto& thrift = metadata.to_thrift(); plan->row_groups.reserve(selected_row_groups.size()); - for (const auto row_group_idx : selected_row_groups) { - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(static_cast(row_group_idx) < row_group_first_rows.size()); - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows == 0) { + for (const int row_group_idx : selected_row_groups) { + const auto& row_group = thrift.row_groups[row_group_idx]; + if (row_group.num_rows == 0) { continue; } - RowGroupReadPlan row_group_plan; row_group_plan.row_group_id = row_group_idx; row_group_plan.first_file_row = row_group_first_rows[row_group_idx]; - row_group_plan.row_group_rows = row_group_rows; - RETURN_IF_ERROR(select_row_group_ranges_by_page_index( - file_reader, file_schema, request, row_group_idx, row_group_rows, - &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, - &plan->pruning_stats, timezone, runtime_state)); - if (row_group_plan.selected_ranges.empty()) { - continue; - } - plan->pruning_stats.selected_row_ranges += row_group_plan.selected_ranges.size(); + row_group_plan.row_group_rows = row_group.num_rows; + row_group_plan.selected_ranges = {{.start = 0, .length = row_group.num_rows}}; + row_group_plan.expensive_pruning_pending = true; plan->row_groups.push_back(std::move(row_group_plan)); } return Status::OK(); @@ -316,52 +495,58 @@ Status build_row_group_read_plans( } // namespace -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - DORIS_CHECK(plan != nullptr); + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); plan->row_groups.clear(); - plan->pruning_stats = ParquetPruningStats {}; - - // Row-group planning flow: - // - // parquet footer row groups - // | - // v - // split byte-range candidates - // | - // v - // row-group metadata pruning - // statistics/ZoneMap -> dictionary -> bloom filter - // | - // v - // page-index pruning per selected row group - // | - // v - // RowGroupReadPlan with selected row ranges - // - // Metadata pruning removes whole row groups before readers are opened. Page index pruning runs - // only for remaining row groups and produces selected row ranges; the scan scheduler later skips - // gaps between those ranges, while row-level VExpr conjuncts still run on loaded batches for - // correctness. + plan->pruning_stats = {}; + plan->enable_bloom_filter = enable_bloom_filter; std::vector row_group_first_rows; - std::vector scan_range_selected_row_groups; - RETURN_IF_ERROR(select_row_groups_by_scan_range(metadata, scan_range, &row_group_first_rows, - &scan_range_selected_row_groups)); - - std::vector metadata_selected_row_groups; + std::vector scan_range_selected; + RETURN_IF_ERROR(detail::select_native_row_groups_by_scan_range( + metadata.to_thrift(), scan_range, &row_group_first_rows, &scan_range_selected)); + std::vector metadata_selected; RETURN_IF_ERROR(select_row_groups_by_metadata( - metadata, file_reader, file_schema, request, &scan_range_selected_row_groups, - &metadata_selected_row_groups, enable_bloom_filter, &plan->pruning_stats, timezone, - runtime_state)); + metadata.to_thrift(), file_schema, request, &scan_range_selected, &metadata_selected, + enable_bloom_filter, &plan->pruning_stats, timezone, runtime_state, file_context, + column_reader_profile, ParquetMetadataProbeMode::FOOTER_ONLY)); + RETURN_IF_ERROR(build_native_row_group_read_plans(metadata, file_schema, request, + metadata_selected, row_group_first_rows, plan, + timezone, runtime_state, file_context)); + plan->pruning_stats.selected_row_groups = plan->row_groups.size(); + return Status::OK(); +} - RETURN_IF_ERROR(build_row_group_read_plans(metadata, file_reader, file_schema, request, - metadata_selected_row_groups, row_group_first_rows, - plan, timezone, runtime_state)); +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + std::vector selected_plans; + selected_plans.reserve(plan->row_groups.size()); + for (auto& row_group_plan : plan->row_groups) { + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + metadata, file_schema, request, enable_bloom_filter, &row_group_plan, + &deferred_stats, timezone, runtime_state, file_context, column_reader_profile, + &selected)); + if (parquet_profile != nullptr) { + parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (selected) { + selected_plans.push_back(std::move(row_group_plan)); + } + } + plan->row_groups = std::move(selected_plans); plan->pruning_stats.selected_row_groups = plan->row_groups.size(); return Status::OK(); } @@ -623,6 +808,7 @@ std::vector filter_ranges_by_condition_cache(const std::vectorflush_profile(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + reader->flush_profile(); + } +} + +bool ParquetScanScheduler::finish_current_reader_batch_profiles() { + bool crossed_page = false; + // A scheduler batch is counted once even when several projected leaves cross page boundaries. + for (const auto& reader : _current_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + return crossed_page; +} + +const detail::PredicateConjunctSchedule& ParquetScanScheduler::predicate_conjunct_schedule( + const format::FileScanRequest& request) { + if (_predicate_schedule_request == &request) { + return _predicate_schedule; + } + + // FileScanRequest is frozen by ParquetReader::open(). Its address therefore identifies both + // the conjunct set and local-position mapping for the scheduler lifetime. + _predicate_schedule = build_predicate_conjunct_schedule(request); + _predicate_schedule_request = &request; + _predicate_positions_scratch.clear(); + _predicate_indices_by_position_scratch.clear(); + _predicate_positions_scratch.reserve(request.predicate_columns.size()); + _predicate_indices_by_position_scratch.reserve(request.predicate_columns.size()); + for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + const auto position_it = + request.local_positions.find(request.predicate_columns[idx].column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + _predicate_positions_scratch.push_back(position); + _predicate_indices_by_position_scratch.emplace(position, idx); + } + return _predicate_schedule; +} + +std::vector ParquetScanScheduler::adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const { + std::vector positions; + std::unordered_map columns_by_position; + positions.reserve(request.predicate_columns.size()); + columns_by_position.reserve(request.predicate_columns.size()); + for (const auto& column : request.predicate_columns) { + const auto position_it = request.local_positions.find(column.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + positions.push_back(position); + columns_by_position.emplace(position, &column); + } + auto ordered = detail::order_adaptive_predicates(positions, _predicate_runtime_stats); + ordered = detail::adaptive_prefetch_prefix(ordered, _predicate_runtime_stats, 0.25); + std::vector result; + result.reserve(ordered.size()); + for (const size_t position : ordered) { + result.push_back(*columns_by_position.at(position)); + } + return result; +} + Status ParquetScanScheduler::open_next_row_group( ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, bool* has_row_group) { *has_row_group = false; - if (_next_row_group_plan_idx >= _row_group_plans.size()) { + RowGroupReadPlan* selected_plan = nullptr; + while (_next_row_group_plan_idx < _row_group_plans.size()) { + RowGroupReadPlan& candidate_plan = _row_group_plans[_next_row_group_plan_idx++]; + // Probe only the row group about to execute. This keeps LIMIT/cancellation latency + // independent of the number of later remote row groups while preserving eager footer + // statistics pruning during open. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + *file_context.native_metadata, file_schema, request, _enable_bloom_filter, + &candidate_plan, &deferred_stats, _timezone, _runtime_state, &file_context, + _scan_profile.column_reader_profile, &selected)); + if (_parquet_profile != nullptr) { + _parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (!selected) { + continue; + } + selected_plan = &candidate_plan; + break; + } + if (selected_plan == nullptr) { + // The last row group's native readers have already been released by + // reset_current_row_group(). Flush the shared merge reader now so its counters are visible + // when EOF is returned and its bounded scratch does not survive until file close. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; return Status::OK(); } - const RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; + RowGroupReadPlan& row_group_plan = *selected_plan; const int row_group_idx = row_group_plan.row_group_id; - // Row-level dictionary filters read dictionary pages before Arrow RecordReaders are created. - // Keep that probe on the base reader: MergeRangeFileReader expects each registered range to be - // consumed as one forward pass, while the later RecordReader opens the same column chunk again - // for the data-page stream. + // Dictionary probes and data-page readers share the native metadata tree. Reset the previous + // row-group merge reader before probing because dictionary-page offsets are not scan ordered. file_context.reset_random_access_ranges(); _current_merge_range_active = false; - try { - _current_row_group = file_context.file_reader->RowGroup(row_group_idx); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } - - auto row_group_metadata = file_context.metadata->RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - _current_row_group_rows = row_group_metadata->num_rows(); + + const auto& row_group_metadata = + file_context.native_metadata->to_thrift().row_groups[row_group_idx]; + _current_row_group_rows = row_group_metadata.num_rows; DORIS_CHECK(_current_row_group_rows == row_group_plan.row_group_rows); DORIS_CHECK(_current_row_group_rows > 0); _current_row_group_id = row_group_idx; + _has_current_row_group = true; DORIS_CHECK(!row_group_plan.selected_ranges.empty()); _current_row_group_first_row = row_group_plan.first_file_row; _current_row_group_rows_read = 0; _current_selected_ranges = row_group_plan.selected_ranges; + _current_offset_indexes = std::move(row_group_plan.offset_indexes); + // Condition Cache and split planning can narrow logical ranges without a physical OffsetIndex. + // Native readers must keep the sequential level/value cursor path valid in that case; only a + // PageIndex-derived skip plan requires the transferred indexes below. + for (const auto& [leaf_column_id, skip_plan] : row_group_plan.page_skip_plans) { + if (!_current_offset_indexes.contains(leaf_column_id)) { + continue; + } + for (size_t page = 0; page < skip_plan.skipped_pages.size(); ++page) { + if (!skip_plan.should_skip_page(page)) { + continue; + } + if (_page_skip_profile.skipped_pages != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); + } + if (_page_skip_profile.skipped_bytes != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_bytes, + skip_plan.skipped_page_compressed_size(page)); + } + } + } _current_range_idx = 0; _current_range_rows_read = 0; _current_predicate_columns.clear(); _current_non_predicate_columns.clear(); _current_dictionary_filters.clear(); RETURN_IF_ERROR(prepare_current_dictionary_filters(file_context, file_schema, request, - row_group_idx, *row_group_metadata)); - _current_merge_range_active = - prepare_current_row_group_reader(file_context, file_schema, request, row_group_idx); - - ParquetColumnReaderFactory column_reader_factory( - _current_row_group, file_context.schema->num_columns(), &row_group_plan.page_skip_plans, - _page_skip_profile, _timezone, _enable_strict_mode, - _scan_profile.column_reader_profile); + row_group_idx, row_group_metadata)); + // Dictionary probing is complete, so the native data-page readers can now share the same + // row-group-scoped MergeRangeFileReader policy as v1. Sharing one wrapper is important: a + // separate merge reader per leaf would duplicate its 128MB scratch capacity and defeat lazy + // materialization for wide schemas. + const auto& thrift_metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : std::string {}); + std::vector native_ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + thrift_metadata, file_schema, request_scan_columns(request), row_group_idx, + file_context.native_file->size(), compat.parquet_816_padding, &native_ranges)); + _current_merge_range_active = file_context.set_native_random_access_ranges( + native_ranges, detail::average_prefetch_range_size(native_ranges), _profile, + _merge_read_slice_size); + for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + const auto local_id = col.column_id(); + if (_current_predicate_columns.contains(local_id)) { + continue; + } + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { + _current_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + _current_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR( - column_reader_factory.create(*column_schema, &col, &column_reader, - _current_dictionary_filters.contains(local_id))); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, + _current_dictionary_filters.contains(local_id), _scan_profile.column_reader_profile, + &column_reader)); _current_predicate_columns[local_id] = std::move(column_reader); } - // Start warming filter-column chunks as soon as their row group is selected. Parquet v2 still - // reads through Arrow's random-access reader; this prefetch only warms Doris file cache blocks - // in the background and never changes the row/column materialization order. + // Start warming filter-column chunks as soon as their row group is selected. The native + // BufferedFileStreamReader later consumes the same Doris file-cache blocks; prefetch never + // changes row/column materialization order. if (!_current_merge_range_active) { - prefetch_current_row_group_columns(file_context, file_schema, request.predicate_columns, - &_current_predicate_prefetched); + const auto prefetch_columns = adaptive_predicate_prefetch_columns(request); + RETURN_IF_ERROR(prefetch_current_row_group_columns( + file_context, file_schema, prefetch_columns, &_current_predicate_prefetched)); } for (const auto& col : request.non_predicate_columns) { - const auto local_id = col.local_id(); + const auto local_id = col.column_id(); if (request.is_count_star_placeholder(col.column_id())) { continue; } - if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_non_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { + _current_non_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_non_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + _current_non_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR(column_reader_factory.create(*column_schema, &col, &column_reader)); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, false, _scan_profile.column_reader_profile, + &column_reader)); _current_non_predicate_columns[local_id] = std::move(column_reader); } - if (!_current_merge_range_active && request.conjuncts.empty() && - request.delete_conjuncts.empty()) { + if (!_current_merge_range_active && + ((request.conjuncts.empty() && request.delete_conjuncts.empty()) || + _predicate_survival_ratio >= 0.8)) { // With no row-level filters there is no lazy-read decision to wait for, so start warming // output chunks immediately after their readers are created. Filtered scans still defer // this until at least one row survives the predicate phase. - prefetch_current_row_group_columns(file_context, file_schema, - physical_non_predicate_columns(request), - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); } *has_row_group = true; return Status::OK(); @@ -849,12 +1177,7 @@ Status ParquetScanScheduler::flush_pending_non_predicate_skip_rows() { namespace { -struct PredicateConjunctSchedule { - std::map single_column_conjuncts; - VExprContextSPtrs remaining_conjuncts; -}; - -PredicateConjunctSchedule build_predicate_conjunct_schedule( +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( const format::FileScanRequest& request) { std::unordered_set predicate_block_positions; predicate_block_positions.reserve(request.predicate_columns.size()); @@ -864,7 +1187,7 @@ PredicateConjunctSchedule build_predicate_conjunct_schedule( predicate_block_positions.insert(position_it->second.value()); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; for (const auto& conjunct : request.conjuncts) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); @@ -962,21 +1285,11 @@ uint16_t count_selected_rows(const IColumn::Filter& filter) { return selected_rows; } -Status filter_read_predicate_columns(Block* file_block, const std::vector& positions, - const IColumn::Filter& compact_filter) { - if (positions.empty()) { - return Status::OK(); - } - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(file_block, positions, compact_filter)); - return Status::OK(); -} - IColumn::Filter build_dictionary_entry_filter(size_t block_position, const ParquetColumnSchema& column_schema, const VExprContextSPtrs& conjuncts, - const ParquetDictionaryWords& dict_words) { - auto fields = dictionary_fields_from_words(dict_words); - IColumn::Filter dictionary_filter(fields.size(), 1); + const IColumn& dictionary) { + IColumn::Filter dictionary_filter(dictionary.size(), 1); DictionaryEvalContext ctx; auto& slot = ctx.slots .emplace(static_cast(block_position), @@ -984,14 +1297,15 @@ IColumn::Filter build_dictionary_entry_filter(size_t block_position, .data_type = column_schema.type, .values = {}}) .first->second; slot.values.reserve(1); - - for (size_t dict_idx = 0; dict_idx < fields.size(); ++dict_idx) { + for (size_t dictionary_id = 0; dictionary_id < dictionary.size(); ++dictionary_id) { + Field value; + dictionary.get(dictionary_id, value); slot.values.clear(); - slot.values.push_back(fields[dict_idx]); - dictionary_filter[dict_idx] = VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch - ? 0 - : 1; + slot.values.push_back(std::move(value)); + dictionary_filter[dictionary_id] = VExprContext::evaluate_dictionary_filter( + conjuncts, ctx) == ZoneMapFilterResult::kNoMatch + ? 0 + : 1; } return dictionary_filter; } @@ -1002,16 +1316,16 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata) { + const tparquet::RowGroup& row_group_metadata) { _current_dictionary_filters.clear(); _current_dictionary_residual_conjuncts.clear(); if (request.conjuncts.empty()) { return Status::OK(); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; { SCOPED_TIMER(_scan_profile.dict_filter_expr_rewrite_time); - schedule = build_predicate_conjunct_schedule(request); + schedule = predicate_conjunct_schedule(request); } if (schedule.single_column_conjuncts.empty()) { return Status::OK(); @@ -1019,8 +1333,8 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( SCOPED_TIMER(_scan_profile.dict_filter_rewrite_time); for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); - if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + const auto local_id = col.column_id(); + if (!local_id.is_valid() || local_id.value() >= static_cast(file_schema.size())) { continue; } const auto position_it = request.local_positions.find(col.column_id()); @@ -1036,29 +1350,38 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( // This optimization is deliberately limited to single-column predicates with a dictionary // evaluable part. Mixed AND predicates are split so dictionary-covered children run as a // dict-id prefilter and residual children keep the normal row-level expression path. - const auto& column_schema = file_schema[local_id]; + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); if (column_schema->leaf_column_id < 0 || - column_schema->leaf_column_id >= row_group_metadata.num_columns()) { + column_schema->leaf_column_id >= static_cast(row_group_metadata.columns.size())) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } - auto column_chunk = row_group_metadata.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_row_level_dictionary_filter(*column_schema, *column_chunk)) { + const auto& column_chunk = row_group_metadata.columns[column_schema->leaf_column_id]; + if (!column_chunk.__isset.meta_data || + !supports_row_level_dictionary_filter(*column_schema, column_chunk.meta_data)) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } - ParquetDictionaryWords dict_words; + std::unique_ptr column_reader; + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_file, file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, true, _scan_profile.column_reader_profile, + &column_reader)); + MutableColumnPtr dictionary_values; { SCOPED_TIMER(_scan_profile.dict_filter_read_dict_time); - if (!read_dictionary_words(file_context.file_reader.get(), row_group_idx, - column_schema->leaf_column_id, *column_schema, - &dict_words)) { + auto dictionary_result = column_reader->dictionary_values(); + if (!dictionary_result.has_value()) { update_counter_if_not_null(_scan_profile.dict_filter_read_failures, 1); + // Dictionary filtering is optional: a probe failure must not reject a file that + // the normal native read path can still decode. continue; } + dictionary_values = std::move(dictionary_result).value(); } // Build a safe dictionary prefilter from the dictionary-filter interface instead of @@ -1069,8 +1392,8 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( DictionaryResidualConjuncts residual_conjuncts; { SCOPED_TIMER(_scan_profile.dict_filter_build_time); - dictionary_filter = build_dictionary_entry_filter(block_position, *column_schema, - conjunct_it->second, dict_words); + dictionary_filter = build_dictionary_entry_filter( + block_position, *column_schema, conjunct_it->second, *dictionary_values); residual_conjuncts = build_dictionary_residual_conjuncts(conjunct_it->second); } @@ -1078,6 +1401,7 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( // predicate with an integer lookup and only materialize STRING values for surviving rows. _current_dictionary_filters.emplace(local_id, std::move(dictionary_filter)); _current_dictionary_residual_conjuncts.emplace(local_id, std::move(residual_conjuncts)); + _current_predicate_columns.emplace(local_id, std::move(column_reader)); update_counter_if_not_null(_scan_profile.dict_filter_columns, 1); } return Status::OK(); @@ -1094,16 +1418,122 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, if (!request.conjuncts.empty() || !request.delete_conjuncts.empty()) { selection->resize(static_cast(batch_rows)); } - const auto schedule = build_predicate_conjunct_schedule(request); + const auto& schedule = predicate_conjunct_schedule(request); + std::unordered_set residual_predicate_positions; + auto remember_residual_positions = [&](const VExprContextSPtrs& conjuncts) { + for (const auto& conjunct : conjuncts) { + std::set positions; + conjunct->root()->collect_slot_column_ids(positions); + for (const int position : positions) { + if (position >= 0) { + residual_predicate_positions.insert(cast_set(position)); + } + } + } + }; + remember_residual_positions(schedule.remaining_conjuncts); + remember_residual_positions(request.delete_conjuncts); + const size_t predicate_batch_sequence = _predicate_batch_sequence++; const bool can_read_predicate_columns_round_by_round = !schedule.single_column_conjuncts.empty(); - std::vector read_column_positions; + auto& read_column_positions = _read_column_positions_scratch; + read_column_positions.clear(); read_column_positions.reserve(request.predicate_columns.size()); + for (auto& rows : _predicate_column_selection_scratch | std::views::values) { + rows.clear(); + } + + auto remember_column_selection = [&](uint32_t position) { + auto& rows = _predicate_column_selection_scratch[position]; + rows.resize(*selected_rows); + for (uint16_t row = 0; row < *selected_rows; ++row) { + // SelectionVector and the scanner batch contract both bound row ordinals to uint16_t; + // keep the checked conversion explicit when persisting the coordinate mapping. + rows[row] = cast_set(selection->get_index(row)); + } + }; + + auto compact_predicate_columns = [&](bool discard_predicate_only_payload) -> Status { + bool compacted = false; + int64_t compacted_bytes = 0; + for (const uint32_t position : read_column_positions) { + auto& source_rows = _predicate_column_selection_scratch[position]; + const auto& old_column = file_block->get_by_position(position).column; + if (old_column->size() != source_rows.size()) { + return Status::Corruption( + "Predicate column {} has {} values but {} remembered source rows", position, + old_column->size(), source_rows.size()); + } + bool predicate_only = false; + if (discard_predicate_only_payload) { + predicate_only = std::ranges::any_of( + request.predicate_only_columns, [&](format::LocalColumnId local_id) { + const auto position_it = request.local_positions.find(local_id); + return position_it != request.local_positions.end() && + position_it->second.value() == position; + }); + } + if (predicate_only) { + auto placeholder = old_column->clone_empty(); + // Hidden predicate values are dead after the last filter, but every file-block + // column must retain the selected row count until TableReader drops hidden slots. + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(position, std::move(placeholder)); + remember_column_selection(position); + continue; + } + bool already_compact = source_rows.size() == *selected_rows && + old_column->size() == static_cast(*selected_rows); + for (uint16_t row = 0; already_compact && row < *selected_rows; ++row) { + already_compact = source_rows[row] == selection->get_index(row); + } + if (already_compact) { + continue; + } + auto& filter = _predicate_compaction_filter_scratch; + // resize_fill() preserves bytes when the next predicate column is smaller. Clear the + // whole reusable mask so survivors from an earlier coordinate space cannot reappear. + filter.resize(source_rows.size()); + std::ranges::fill(filter, 0); + size_t source_idx = 0; + uint16_t selected_idx = 0; + while (source_idx < source_rows.size() && selected_idx < *selected_rows) { + const auto source_row = source_rows[source_idx]; + const auto selected_row = selection->get_index(selected_idx); + if (source_row < selected_row) { + ++source_idx; + continue; + } + DORIS_CHECK_EQ(source_row, selected_row); + filter[source_idx++] = 1; + ++selected_idx; + } + DORIS_CHECK_EQ(selected_idx, *selected_rows); + compacted_bytes += static_cast(old_column->byte_size()); + RETURN_IF_CATCH_EXCEPTION(file_block->replace_by_position( + position, old_column->filter(filter, *selected_rows))); + remember_column_selection(position); + compacted = true; + } + if (compacted) { + update_counter_if_not_null(_scan_profile.predicate_compaction_bytes, compacted_bytes); + update_counter_if_not_null(_scan_profile.predicate_compaction_count, 1); + } + // The output path must not apply a batch-coordinate filter to columns that now use compact + // coordinates. The loop above establishes this invariant even when no bytes moved because + // every column was already aligned. + *predicate_columns_filtered = !read_column_positions.empty(); + return Status::OK(); + }; - auto read_predicate_column = [&](ParquetColumnReader* column_reader, size_t block_position, - ColumnId local_id, bool* used_dictionary_filter) -> Status { + auto read_predicate_column = + [&](ParquetColumnReader* column_reader, size_t block_position, + format::LocalColumnId local_id, const VExprContextSPtrs* single_column_conjuncts, + bool* used_dictionary_filter, bool* used_plain_filter) -> Status { DORIS_CHECK(used_dictionary_filter != nullptr); + DORIS_CHECK(used_plain_filter != nullptr); *used_dictionary_filter = false; + *used_plain_filter = false; DCHECK(remove_nullable(column_reader->type()) ->equals(*remove_nullable(file_block->get_by_position(block_position).type))) << column_reader->type()->get_name() << " " @@ -1130,22 +1560,63 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, update_counter_if_not_null(_scan_profile.rows_filtered_by_dict_filter, filtered_rows); if (new_selected_rows != selected_rows_before) { - // The dictionary reader has already appended only surviving values for the - // current column. Apply the compact row filter only to columns read before this - // one, then update the shared selection for later predicate/lazy columns. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); + // The dictionary reader already appended only survivors for this column. Keep + // older predicate columns in their original coordinate spaces and compact all + // of them once at the expression/output boundary below. *selected_rows = apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); *used_dictionary_filter = true; return Status::OK(); } } + if (single_column_conjuncts != nullptr && + !residual_predicate_positions.contains(block_position) && + request.is_predicate_only(local_id)) { + VExprSPtrs direct_conjuncts; + direct_conjuncts.reserve(single_column_conjuncts->size()); + std::ranges::transform(*single_column_conjuncts, std::back_inserter(direct_conjuncts), + [](const auto& context) { return context->root(); }); + if (!direct_conjuncts.empty()) { + const uint16_t selected_rows_before = *selected_rows; + IColumn::Filter compact_filter; + bool used_filter = false; + RETURN_IF_ERROR(column_reader->select_with_plain_filter( + *selection, *selected_rows, batch_rows, direct_conjuncts, + cast_set(block_position), &compact_filter, &used_filter)); + if (used_filter) { + DORIS_CHECK_EQ(compact_filter.size(), selected_rows_before); + update_counter_if_not_null(_scan_profile.plain_predicate_direct_batches, 1); + update_counter_if_not_null(_scan_profile.plain_predicate_direct_rows, + selected_rows_before); + const uint16_t new_selected_rows = count_selected_rows(compact_filter); + const auto filtered_rows = static_cast(selected_rows_before) - + static_cast(new_selected_rows); + if (conjunct_filtered_rows != nullptr) { + *conjunct_filtered_rows += filtered_rows; + } + if (new_selected_rows != selected_rows_before) { + *selected_rows = apply_compact_filter_to_selection( + compact_filter, selection, selected_rows_before); + } + // This slot is absent from every residual/delete conjunct, so no later + // expression can observe its payload. Keep only the block row-shape contract. + auto placeholder = column->clone_empty(); + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(block_position, std::move(placeholder)); + read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); + *predicate_columns_filtered = true; + *used_plain_filter = true; + return Status::OK(); + } + } + } + if (*selected_rows == batch_rows) { int64_t column_rows = 0; RETURN_IF_ERROR(column_reader->read(batch_rows, column, &column_rows)); @@ -1166,6 +1637,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); return Status::OK(); }; @@ -1187,16 +1659,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // All columns read so far are already compacted to the current selection. Apply the - // compact filter to those columns and the selection vector together, so later predicate - // columns can read only rows that survived previous predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1220,16 +1686,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // Dictionary-covered children have already reduced the compact block. Apply only the - // residual child filters here, then keep the same compacted-column invariant as the - // normal conjunct path for later predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1266,24 +1726,23 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, compact_filter.resize_fill(selected_rows_before, 0); } if (can_filter_all || count_selected_rows(compact_filter) != selected_rows_before) { - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; auto read_all_predicate_columns = [&]() -> Status { for (const auto& [fid, column_reader] : _current_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); bool used_dictionary_filter = false; + bool used_plain_filter = false; RETURN_IF_ERROR(read_predicate_column(column_reader.get(), position_it->second.value(), - fid, &used_dictionary_filter)); + fid, nullptr, &used_dictionary_filter, + &used_plain_filter)); } return Status::OK(); }; @@ -1303,45 +1762,69 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, // Single-column conjuncts can be evaluated immediately after their column is read. Once // selection shrinks, later predicate columns use ParquetColumnReader::select() so the // reader skips rows already rejected by earlier predicates instead of materializing them. - for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + _ordered_predicate_positions_scratch = detail::order_adaptive_predicates( + _predicate_positions_scratch, _predicate_runtime_stats); + const auto& ordered_positions = _ordered_predicate_positions_scratch; + for (size_t order_idx = 0; order_idx < ordered_positions.size(); ++order_idx) { + const size_t position = ordered_positions[order_idx]; + const size_t idx = _predicate_indices_by_position_scratch.at(position); const auto& col = request.predicate_columns[idx]; - const auto fid = col.local_id(); + const auto fid = col.column_id(); auto reader_it = _current_predicate_columns.find(fid); DORIS_CHECK(reader_it != _current_predicate_columns.end()); auto position_it = request.local_positions.find(col.column_id()); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); + const uint16_t rows_before = *selected_rows; + auto& stats = _predicate_runtime_stats[position]; + const bool sample = detail::should_sample_adaptive_predicate(stats.samples, + predicate_batch_sequence); + const int64_t start_ns = sample ? MonotonicNanos() : 0; bool used_dictionary_filter = false; + bool used_plain_filter = false; + const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); + const VExprContextSPtrs* column_conjuncts = + conjunct_it == schedule.single_column_conjuncts.end() ? nullptr + : &conjunct_it->second; RETURN_IF_ERROR(read_predicate_column(reader_it->second.get(), block_position, fid, - &used_dictionary_filter)); - if (*selected_rows == 0) { - for (size_t remaining_idx = idx + 1; - remaining_idx < request.predicate_columns.size(); ++remaining_idx) { - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); - auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); - DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); - RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); + column_conjuncts, &used_dictionary_filter, + &used_plain_filter)); + if (*selected_rows != 0 && conjunct_it != schedule.single_column_conjuncts.end()) { + if (used_dictionary_filter) { + const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); + DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); + RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( + residual_it->second)); + } else if (!used_plain_filter) { + RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); } - return Status::OK(); - } - const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); - if (conjunct_it == schedule.single_column_conjuncts.end()) { - continue; } - if (used_dictionary_filter) { - const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); - DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); - RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( - residual_it->second)); - } else { - RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); + if (sample) { + const double cost_per_row = static_cast(MonotonicNanos() - start_ns) / + std::max(rows_before, 1); + const double survival = + static_cast(*selected_rows) / std::max(rows_before, 1); + constexpr double ADAPTIVE_ALPHA = 0.25; + if (stats.samples == 0) { + stats.cost_per_input_row_ns = cost_per_row; + stats.survival_ratio = survival; + } else { + stats.cost_per_input_row_ns = + ADAPTIVE_ALPHA * cost_per_row + + (1 - ADAPTIVE_ALPHA) * stats.cost_per_input_row_ns; + stats.survival_ratio = + ADAPTIVE_ALPHA * survival + (1 - ADAPTIVE_ALPHA) * stats.survival_ratio; + } + ++stats.samples; } if (*selected_rows != 0) { continue; } - for (size_t remaining_idx = idx + 1; remaining_idx < request.predicate_columns.size(); - ++remaining_idx) { - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); + for (size_t remaining_order_idx = order_idx + 1; + remaining_order_idx < ordered_positions.size(); ++remaining_order_idx) { + const size_t remaining_idx = _predicate_indices_by_position_scratch.at( + ordered_positions[remaining_order_idx]); + const auto remaining_fid = request.predicate_columns[remaining_idx].column_id(); auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); @@ -1351,47 +1834,59 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, return Status::OK(); }; + auto compact_predicate_columns_with_profile = + [&](bool discard_predicate_only_payload) -> Status { + const int64_t start_ns = MonotonicNanos(); + auto status = compact_predicate_columns(discard_predicate_only_payload); + update_counter_if_not_null(_scan_profile.predicate_compaction_time, + MonotonicNanos() - start_ns); + return status; + }; + RETURN_IF_ERROR(read_round_by_round()); + // Single-column expressions only touch the just-read column, so earlier columns can retain + // their own row mappings. Compact only when a later expression needs a shared coordinate + // space; otherwise the final boundary can discard hidden predicate payloads without scanning + // them again. + if (!schedule.remaining_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); + } RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(schedule.remaining_conjuncts)); - if (_scan_profile.predicate_filter_time == nullptr) { - return execute_scheduled_delete_conjuncts(); + if (!request.delete_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); } - SCOPED_TIMER(_scan_profile.predicate_filter_time); - return execute_scheduled_delete_conjuncts(); -} - -bool ParquetScanScheduler::prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx) { - if (file_context.metadata == nullptr) { - return false; + if (_scan_profile.predicate_filter_time == nullptr) { + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); + } else { + SCOPED_TIMER(_scan_profile.predicate_filter_time); + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); } - const auto ranges = build_row_group_prefetch_ranges( - *file_context.metadata, file_schema, request_scan_columns(request), row_group_idx); - const size_t avg_io_size = detail::average_prefetch_range_size(ranges); - return file_context.set_random_access_ranges(ranges, avg_io_size, _profile, - _merge_read_slice_size); + return compact_predicate_columns_with_profile(true); } -void ParquetScanScheduler::prefetch_current_row_group_columns( +Status ParquetScanScheduler::prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched) { DORIS_CHECK(prefetched != nullptr); if (_current_merge_range_active || *prefetched || scan_columns.empty() || - _current_row_group_id < 0 || file_context.metadata == nullptr) { - return; + _current_row_group_id < 0 || file_context.native_metadata == nullptr) { + return Status::OK(); } *prefetched = true; // The scanner request separates predicate and non-predicate columns so Parquet can read // predicate columns first and lazily materialize the rest. Keep the same contract for // prefetch: callers decide which side to warm, and this helper only translates that selected // projection into physical column-chunk byte ranges for the current row group. - file_context.prefetch_ranges( - build_row_group_prefetch_ranges(*file_context.metadata, file_schema, scan_columns, - _current_row_group_id), - nullptr); + const auto& metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + std::vector ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + metadata, file_schema, scan_columns, _current_row_group_id, + file_context.native_file->size(), compat.parquet_816_padding, &ranges)); + file_context.prefetch_ranges(ranges, nullptr); + return Status::OK(); } Status ParquetScanScheduler::read_current_row_group_batch( @@ -1399,6 +1894,22 @@ Status ParquetScanScheduler::read_current_row_group_batch( const std::vector>& file_schema, int64_t batch_rows, const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, size_t* rows) { + // Reader statistics are cumulative plain integers. Publishing their delta recursively for + // every tiny batch is measurable on wide/nested scans, so flush periodically and force the + // tail at row-group reset/close. + Defer profile_flush {[this, batch_rows]() { + if (finish_current_reader_batch_profiles() && + _scan_profile.column_reader_profile.page_crossing_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.column_reader_profile.page_crossing_batches, 1); + } + const bool finishes_row_group = _current_range_idx + 1 == _current_selected_ranges.size() && + _current_range_rows_read + batch_rows == + _current_selected_ranges[_current_range_idx].length; + if (++_batches_since_profile_flush >= PROFILE_FLUSH_BATCH_INTERVAL || finishes_row_group) { + flush_current_reader_profiles(); + _batches_since_profile_flush = 0; + } + }}; if (_scan_profile.total_batches != nullptr) { COUNTER_UPDATE(_scan_profile.total_batches, 1); } @@ -1414,7 +1925,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( } return Status::OK(); } - SelectionVector selection; + auto& selection = _selection; DORIS_CHECK(batch_rows <= std::numeric_limits::max()); uint16_t selected_rows = static_cast(batch_rows); int64_t conjunct_filtered_rows = 0; @@ -1425,6 +1936,10 @@ Status ParquetScanScheduler::read_current_row_group_batch( mark_condition_cache_granules(selection, selected_rows, batch_first_file_row); const bool need_filter_output = selected_rows != batch_rows; + const double batch_survival = static_cast(selected_rows) / batch_rows; + _predicate_survival_ratio = _predicate_survival_ratio < 0 + ? batch_survival + : 0.25 * batch_survival + 0.75 * _predicate_survival_ratio; if (_scan_profile.selected_rows != nullptr) { COUNTER_UPDATE(_scan_profile.selected_rows, selected_rows); } @@ -1437,6 +1952,11 @@ Status ParquetScanScheduler::read_current_row_group_batch( } if (selected_rows == 0 && _scan_profile.empty_selection_batches != nullptr) { COUNTER_UPDATE(_scan_profile.empty_selection_batches, 1); + } else if (static_cast(selected_rows) == batch_rows && + _scan_profile.dense_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.dense_batches, 1); + } else if (_scan_profile.selected_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.selected_batches, 1); } if (need_filter_output && !predicate_columns_filtered) { IColumn::Filter output_filter = selection_to_filter(selection, selected_rows, batch_rows); @@ -1451,7 +1971,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( } if (selected_rows == 0) { // Predicate readers have consumed this physical batch, but touching every lazy column here - // turns a long rejected prefix into `empty_batches * lazy_columns` Arrow calls. Record only + // turns a long rejected prefix into `empty_batches * lazy_columns` native calls. Record only // the positional lag. If [0, 32), [32, 64), and [64, 96) are empty, the first surviving // batch performs one skip(96) per lazy column. If the row group ends instead, reset drops the // lazy readers without flushing because no value from them can be observed. @@ -1466,9 +1986,9 @@ Status ParquetScanScheduler::read_current_row_group_batch( // Do not prefetch lazy output columns until at least one row survives filtering. This is // the same decision point where the v2 reader switches from predicate-only reads to // materializing non-predicate columns, so fully filtered batches avoid unnecessary IO. - prefetch_current_row_group_columns(file_context, file_schema, - physical_non_predicate_columns(request), - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); } { @@ -1477,7 +1997,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( // selection vector. This also merges pending range gaps with fully filtered batches. RETURN_IF_ERROR(flush_pending_non_predicate_skip_rows()); for (const auto& [fid, column_reader] : _current_non_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); auto column = file_block->get_by_position(block_position).column->assert_mutable(); @@ -1536,8 +2056,23 @@ Status ParquetScanScheduler::read_next_batch( const std::vector>& file_schema, const format::FileScanRequest& request, Block* file_block, size_t* rows, bool* eof) { *rows = 0; + int64_t predicate_batch_rows = _batch_size; + const int64_t max_predicate_batch_rows = std::min( + std::numeric_limits::max(), + std::max(DEFAULT_READ_BATCH_SIZE, _runtime_state == nullptr + ? DEFAULT_READ_BATCH_SIZE + : _runtime_state->batch_size())); + auto grow_empty_predicate_batch = [max_predicate_batch_rows](int64_t current) { + for (const int64_t target : + {int64_t {256}, int64_t {1024}, int64_t {4096}, max_predicate_batch_rows}) { + if (current < target) { + return std::min(target, max_predicate_batch_rows); + } + } + return max_predicate_batch_rows; + }; while (true) { - if (_current_row_group == nullptr) { + if (!_has_current_row_group) { bool has_row_group = false; RETURN_IF_ERROR( open_next_row_group(file_context, file_schema, request, &has_row_group)); @@ -1572,7 +2107,7 @@ Status ParquetScanScheduler::read_next_batch( continue; } - const int64_t batch_rows = std::min(_batch_size, remaining_rows); + const int64_t batch_rows = std::min(predicate_batch_rows, remaining_rows); const int64_t physical_rows_read = batch_rows; const int64_t batch_first_file_row = _current_row_group_first_row + _current_row_group_rows_read; @@ -1585,6 +2120,10 @@ Status ParquetScanScheduler::read_next_batch( _current_range_rows_read = 0; } if (*rows == 0) { + // Output-width feedback has no sample for a fully rejected batch. Grow only the + // predicate-side physical read so a 32-row probe cannot pin a long filtered prefix; + // any eventual output still stays below RuntimeState's row cap. + predicate_batch_rows = grow_empty_predicate_batch(predicate_batch_rows); continue; } *eof = false; diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index c656b146cefeaa..ff3930bf7e4ddd 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include @@ -33,13 +36,6 @@ #include "runtime/runtime_profile.h" #include "storage/segment/condition_cache.h" -namespace parquet { -class FileMetaData; -class ParquetFileReader; -class RowGroupMetaData; -class RowGroupReader; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -57,6 +53,40 @@ namespace doris::format::parquet { struct ParquetFileContext; struct ParquetColumnSchema; +struct ParquetPageCacheRange; +struct ParquetScanRange; +class NativeParquetMetadata; + +namespace detail { +struct PredicateConjunctSchedule { + std::map single_column_conjuncts; + VExprContextSPtrs remaining_conjuncts; +}; + +struct AdaptivePredicateStats { + double cost_per_input_row_ns = 0; + double survival_ratio = 1; + size_t samples = 0; +}; + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats); +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability); +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence); +Status build_native_prefetch_ranges( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges); +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups); +} // namespace detail // ============================================================================ // ============================================================================ @@ -74,23 +104,39 @@ struct RowGroupReadPlan { std::vector selected_ranges; // row ranges to read after page-index pruning std::map page_skip_plans; // leaf_column_id -> data pages that can be skipped completely + // Deferred planning transfers parsed indexes to execution so narrowed scans never issue the + // same remote index reads a second time while opening the row group. + std::unordered_map offset_indexes; + // Footer statistics are cheap and eager. Remote dictionary/Bloom/page-index probes fill the + // remaining fields only when this row group reaches the scheduler. + bool expensive_pruning_pending = false; }; struct RowGroupScanPlan { std::vector row_groups; // row groups selected after pruning ParquetPruningStats pruning_stats; // pruning statistics + bool enable_bloom_filter = false; }; // ============================================================================ // ============================================================================ -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); + const RuntimeState* runtime_state = nullptr, + ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}); + +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile = nullptr); IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows); @@ -116,6 +162,9 @@ class ParquetScanScheduler { _page_skip_profile = page_skip_profile; } void set_scan_profile(ParquetScanProfile scan_profile) { _scan_profile = scan_profile; } + void set_pruning_profile(const ParquetProfile* parquet_profile) { + _parquet_profile = parquet_profile; + } void set_merge_read_options(RuntimeProfile* profile, int64_t merge_read_slice_size) { _profile = profile; _merge_read_slice_size = merge_read_slice_size; @@ -128,6 +177,10 @@ class ParquetScanScheduler { void set_enable_strict_mode(bool enable_strict_mode) { _enable_strict_mode = enable_strict_mode; } + void set_runtime_state(RuntimeState* runtime_state) { _runtime_state = runtime_state; } + // Release row-group readers before the owning RuntimeProfile is reported. Native readers + // publish their accumulated page/decode statistics from their destructor. + void close() { reset_current_row_group(); } // Upper scanner owns adaptive memory feedback; scheduler only applies the current row cap when // splitting selected row ranges into physical read batches. void set_batch_size(size_t batch_size) { @@ -145,7 +198,15 @@ class ParquetScanScheduler { bool* eof); private: + static constexpr size_t PROFILE_FLUSH_BATCH_INTERVAL = 16; + void reset_current_row_group(); + void flush_current_reader_profiles(); + bool finish_current_reader_batch_profiles(); + const detail::PredicateConjunctSchedule& predicate_conjunct_schedule( + const format::FileScanRequest& request); + std::vector adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const; Status open_next_row_group(ParquetFileContext& file_context, const std::vector>& file_schema, @@ -163,18 +224,13 @@ class ParquetScanScheduler { ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata); + const tparquet::RowGroup& row_group_metadata); - void prefetch_current_row_group_columns( + Status prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched); - bool prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx); - Status read_current_row_group_batch( ParquetFileContext& file_context, const std::vector>& file_schema, @@ -187,14 +243,18 @@ class ParquetScanScheduler { std::vector _row_group_plans; // row group queue to scan size_t _next_row_group_plan_idx = 0; // index of the next row group to process - std::shared_ptr<::parquet::RowGroupReader> _current_row_group; // Arrow RowGroup reader - std::map> + bool _has_current_row_group = false; + // Readers retain pointers into this immutable row-group map, so it must outlive both maps below. + std::unordered_map _current_offset_indexes; + // File-local ids are signed because virtual columns use reserved negative values. Keeping the + // typed id as the map key prevents GLOBAL_ROWID_COLUMN_ID from wrapping to a storage ColumnId. + std::map> _current_predicate_columns; // predicate ColumnReaders - std::map> + std::map> _current_non_predicate_columns; // non-predicate ColumnReaders - std::map + std::map _current_dictionary_filters; // local id -> dict entry bitmap - std::map>> + std::map>> _current_dictionary_residual_conjuncts; // local id -> row-level residual conjuncts int64_t _current_row_group_rows = 0; // current row group row count int _current_row_group_id = -1; // current row group id in parquet metadata @@ -214,12 +274,31 @@ class ParquetScanScheduler { bool _current_merge_range_active = false; ParquetPageSkipProfile _page_skip_profile; ParquetScanProfile _scan_profile; + const ParquetProfile* _parquet_profile = nullptr; RuntimeProfile* _profile = nullptr; int64_t _merge_read_slice_size = -1; std::optional _global_rowid_context; const cctz::time_zone* _timezone = nullptr; bool _enable_strict_mode = false; + bool _enable_bloom_filter = false; + RuntimeState* _runtime_state = nullptr; int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; + // Batch control scratch is scheduler-owned so adaptive row caps change logical sizes without + // reallocating selection indices, dense filter bytes, or compacted-column positions. + SelectionVector _selection; + std::vector _read_column_positions_scratch; + const format::FileScanRequest* _predicate_schedule_request = nullptr; + detail::PredicateConjunctSchedule _predicate_schedule; + std::vector _predicate_positions_scratch; + std::unordered_map _predicate_indices_by_position_scratch; + std::vector _ordered_predicate_positions_scratch; + std::unordered_map> + _predicate_column_selection_scratch; + IColumn::Filter _predicate_compaction_filter_scratch; + size_t _predicate_batch_sequence = 0; + size_t _batches_since_profile_flush = 0; + std::unordered_map _predicate_runtime_stats; + double _predicate_survival_ratio = -1; std::shared_ptr _condition_cache_ctx; int64_t _condition_cache_filtered_rows = 0; int64_t _predicate_filtered_rows = 0; diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index ec404f40e15ffb..eac5bed00ed531 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -15,15 +15,6 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include @@ -48,15 +39,93 @@ #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr_context.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/timestamp_statistics.h" #include "runtime/runtime_profile.h" +#include "storage/index/bloom_filter/bloom_filter.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/index/zone_map/zonemap_eval_context.h" +#include "util/thrift_util.h" +#include "util/unaligned.h" namespace doris::format::parquet { +namespace detail { + +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size) { + if (offset < 0 || header_size == 0 || payload_size < segment_v2::BloomFilter::MINIMUM_BYTES || + payload_size > segment_v2::BloomFilter::MAXIMUM_BYTES || payload_size % 32 != 0) { + return Status::Corruption( + "Invalid Parquet Bloom filter layout: offset {}, header {}, payload {}", offset, + header_size, payload_size); + } + const uint64_t unsigned_offset = static_cast(offset); + const uint64_t total_size = static_cast(header_size) + payload_size; + if (unsigned_offset > file_size || total_size > file_size - unsigned_offset) { + return Status::Corruption("Parquet Bloom filter range exceeds file size {}", file_size); + } + if (declared_length >= 0) { + const uint64_t unsigned_declared_length = static_cast(declared_length); + if (unsigned_declared_length < total_size || + unsigned_declared_length > file_size - unsigned_offset) { + return Status::Corruption( + "Parquet Bloom filter requires {} bytes, metadata declares {}, file has {}", + total_size, declared_length, file_size - unsigned_offset); + } + } + return Status::OK(); +} + +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id) { + return leaf_column_id >= 0 && metadata.__isset.column_orders && + leaf_column_id < static_cast(metadata.column_orders.size()) && + metadata.column_orders[leaf_column_id].__isset.TYPE_ORDER; +} + +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + auto sanitized = statistics; + if (!has_type_defined_order || !sanitized.__isset.min_value || !sanitized.__isset.max_value) { + sanitized.__isset.min_value = false; + sanitized.__isset.max_value = false; + sanitized.min_value.clear(); + sanitized.max_value.clear(); + } + const bool binary = type_descriptor.physical_type == tparquet::Type::BYTE_ARRAY || + type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; + if (!sanitized.__isset.min || !sanitized.__isset.max || + (binary && sanitized.min != sanitized.max)) { + sanitized.__isset.min = false; + sanitized.__isset.max = false; + sanitized.min.clear(); + sanitized.max.clear(); + } + return sanitized; +} + +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + const auto sanitized = + sanitize_native_footer_statistics(type_descriptor, statistics, has_type_defined_order); + return (sanitized.__isset.min_value && sanitized.__isset.max_value) || + (sanitized.__isset.min && sanitized.__isset.max); +} + +} // namespace detail + namespace { +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone); + enum class ParquetRowGroupPruneReason { NONE, // cannot prune; must read STATISTICS, // excluded by ZoneMap statistics @@ -64,6 +133,60 @@ enum class ParquetRowGroupPruneReason { BLOOM_FILTER, // excluded by bloom filter }; +Status read_native_bloom_filter(const tparquet::ColumnMetaData& metadata, + const io::FileReaderSPtr& file, io::IOContext* io_ctx, + std::unique_ptr* result) { + if (result == nullptr || file == nullptr || !metadata.__isset.bloom_filter_offset) { + return Status::NotSupported("Parquet Bloom filter is unavailable"); + } + constexpr size_t MAX_BLOOM_HEADER_BYTES = 64; + if (metadata.bloom_filter_offset < 0 || + (metadata.__isset.bloom_filter_length && metadata.bloom_filter_length <= 0)) { + return Status::Corruption("Invalid Parquet Bloom filter offset or declared length"); + } + const uint64_t bloom_offset = static_cast(metadata.bloom_filter_offset); + if (bloom_offset >= file->size()) { + return Status::Corruption("Parquet Bloom filter offset exceeds file size {}", file->size()); + } + const size_t available = file->size() - bloom_offset; + const size_t declared_available = + metadata.__isset.bloom_filter_length + ? std::min(metadata.bloom_filter_length, available) + : available; + const size_t header_read_size = std::min(declared_available, MAX_BLOOM_HEADER_BYTES); + std::vector header_buffer(header_read_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(metadata.bloom_filter_offset, + Slice(header_buffer.data(), header_buffer.size()), &bytes_read, + io_ctx)); + tparquet::BloomFilterHeader header; + uint32_t header_size = cast_set(bytes_read); + RETURN_IF_ERROR(deserialize_thrift_msg(header_buffer.data(), &header_size, true, &header)); + if (!header.algorithm.__isset.BLOCK || !header.compression.__isset.UNCOMPRESSED || + !header.hash.__isset.XXHASH || header.numBytes <= 0) { + return Status::NotSupported("Unsupported Parquet Bloom filter encoding"); + } + + // Validate the complete split-block layout before allocating or adding footer-controlled + // offsets; BloomFilter::init() otherwise receives a truncated or oversized backing buffer. + RETURN_IF_ERROR(detail::validate_native_bloom_filter_layout( + metadata.bloom_filter_offset, header_size, header.numBytes, + metadata.__isset.bloom_filter_length ? metadata.bloom_filter_length : -1, + file->size())); + + std::vector data(cast_set(header.numBytes)); + RETURN_IF_ERROR(file->read_at(static_cast(metadata.bloom_filter_offset) + header_size, + Slice(data.data(), data.size()), &bytes_read, io_ctx)); + if (bytes_read != data.size()) { + return Status::Corruption("Truncated Parquet Bloom filter payload"); + } + auto bloom_filter = std::make_unique(); + RETURN_IF_ERROR(bloom_filter->init(reinterpret_cast(data.data()), data.size(), + segment_v2::HashStrategyPB::XX_HASH_64)); + *result = std::move(bloom_filter); + return Status::OK(); +} + bool bloom_logical_type_supported(const ParquetColumnSchema& column_schema) { if (column_schema.type == nullptr) { return false; @@ -172,30 +295,6 @@ bool decoded_min_max_is_ordered(const ParquetColumnStatistics& column_statistics return !(column_statistics.max_value < column_statistics.min_value); } -template -bool set_decoded_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics>(statistics); - const auto& min_value = typed_statistics->min(); - const auto& max_value = typed_statistics->max(); - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; - } - } - if (!valid_min_max(min_value, max_value) || - !set_decoded_field(column_schema, value_kind, min_value, &column_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &column_statistics->max_value, - timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); -} - bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, const StringRef& value, Field* field, const cctz::time_zone* timezone) { @@ -206,54 +305,6 @@ bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedV return read_decoded_field(column_schema, view, field, timezone).ok(); } -bool set_string_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - switch (statistics->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::ByteArrayType>>( - statistics); - const auto min = ::parquet::ByteArrayToString(typed_statistics->min()); - const auto max = ::parquet::ByteArrayToString(typed_statistics->max()); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - if (column_schema.descriptor == nullptr || column_schema.descriptor->type_length() <= 0) { - return false; - } - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::FLBAType>>( - statistics); - const int type_length = column_schema.descriptor->type_length(); - const std::string min(reinterpret_cast(typed_statistics->min().ptr), - type_length); - const std::string max(reinterpret_cast(typed_statistics->max().ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - default: - return false; - } -} - template T load_predicate_value(const char* data) { T value; @@ -306,61 +357,19 @@ std::optional convert_logical_integer_to_physical_int32( return physical_value; } -class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { +class NativeParquetBloomFilterAdapter final : public segment_v2::BloomFilter { public: - ArrowParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, - const ::parquet::BloomFilter& bloom_filter) + NativeParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, + const segment_v2::BloomFilter& bloom_filter) : _column_schema(column_schema), _bloom_filter(bloom_filter) {} - void add_bytes(const char* buf, size_t size) override { DORIS_CHECK(false); } + void add_bytes(const char*, size_t) override { DORIS_CHECK(false); } bool test_bytes(const char* buf, size_t size) const override { - if (buf == nullptr) { - return true; - } - // Parquet bloom filters are populated from the physical column carrier, while VExpr - // literals are materialized as Doris logical values. Keep the logical type in - // BloomFilterEvalContext for expression compatibility, and normalize to the Parquet - // physical representation only at this adapter boundary. - switch (_column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - return test_boolean(buf, size); - case ::parquet::Type::INT32: - return test_physical_int32(buf, size); - case ::parquet::Type::INT64: - return test_int64(buf, size); - case ::parquet::Type::FLOAT: - return test_float(buf, size); - case ::parquet::Type::DOUBLE: - return test_double(buf, size); - case ::parquet::Type::BYTE_ARRAY: - return test_byte_array(buf, size); - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return test_fixed_len_byte_array(buf, size); - default: - return true; - } - } - - void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } - bool has_null() const override { return false; } - void add_hash(uint64_t hash) override { DORIS_CHECK(false); } - bool test_hash(uint64_t hash) const override { return _bloom_filter.FindHash(hash); } - -private: - bool test_boolean(const char* buf, size_t size) const { - if (size == sizeof(bool)) { - const int32_t value = load_predicate_value(buf) ? 1 : 0; - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - if (size == sizeof(int32_t)) { - const int32_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value != 0 ? 1 : 0)); + if (buf == nullptr || + _column_schema.type_descriptor.physical_type != tparquet::Type::INT32) { + return _bloom_filter.test_bytes(buf, size); } - return true; - } - - bool test_physical_int32(const char* buf, size_t size) const { const auto logical_value = load_predicate_integral_value(buf, size); if (!logical_value.has_value()) { return true; @@ -370,57 +379,20 @@ class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { if (!physical_value.has_value()) { return false; } - return find_int32(*physical_value); - } - - bool test_int64(const char* buf, size_t size) const { - if (size != sizeof(int64_t)) { - return true; - } - const int64_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_float(const char* buf, size_t size) const { - if (size != sizeof(float)) { - return true; - } - const float value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_double(const char* buf, size_t size) const { - if (size != sizeof(double)) { - return true; - } - const double value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_byte_array(const char* buf, size_t size) const { - ::parquet::ByteArray value(static_cast(size), - reinterpret_cast(buf)); - return _bloom_filter.FindHash(_bloom_filter.Hash(&value)); - } - - bool test_fixed_len_byte_array(const char* buf, size_t size) const { - if (_column_schema.type_descriptor.fixed_length <= 0) { - return true; - } - if (size != static_cast(_column_schema.type_descriptor.fixed_length)) { - return false; - } - ::parquet::FLBA value(reinterpret_cast(buf)); - return _bloom_filter.FindHash( - _bloom_filter.Hash(&value, _column_schema.type_descriptor.fixed_length)); + // Native file Bloom bytes are hashed from the Parquet physical carrier, not the wider + // Doris logical literal used by VExpr (for example UINT32 is exposed as BIGINT). + return _bloom_filter.test_bytes(reinterpret_cast(&*physical_value), + sizeof(*physical_value)); } - bool find_int32(int32_t value) const { - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } + void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } + bool has_null() const override { return false; } + void add_hash(uint64_t) override { DORIS_CHECK(false); } + bool test_hash(uint64_t hash) const override { return _bloom_filter.test_hash(hash); } +private: const ParquetColumnSchema& _column_schema; - const ::parquet::BloomFilter& _bloom_filter; + const segment_v2::BloomFilter& _bloom_filter; }; bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { @@ -428,14 +400,14 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { return false; } switch (column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BOOLEAN: + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::BYTE_ARRAY: return true; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return column_schema.type_descriptor.is_string_like && column_schema.type_descriptor.fixed_length > 0; default: @@ -443,220 +415,6 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { } } -bool bloom_filter_excludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - if (!bloom_filter_supported(column_schema)) { - return false; - } - ArrowParquetBloomFilterAdapter adapter(column_schema, bloom_filter); - BloomFilterEvalContext ctx; - ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { - .data_type = column_schema.type, - .bloom_filter = &adapter, - }); - return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; -} - -struct RowGroupBloomFilterCache { - using CacheKey = std::pair; - - ::parquet::BloomFilterReader* bloom_filter_reader = nullptr; - std::map> column_bloom_filters; - std::set loaded_columns; - - ::parquet::BloomFilter* get(int row_group_idx, int leaf_column_id, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_reader == nullptr || leaf_column_id < 0) { - return nullptr; - } - const CacheKey cache_key {row_group_idx, leaf_column_id}; - if (loaded_columns.find(cache_key) == loaded_columns.end()) { - loaded_columns.insert(cache_key); - try { - std::shared_ptr<::parquet::RowGroupBloomFilterReader> row_group_reader; - if (pruning_stats != nullptr) { - SCOPED_RAW_TIMER(&pruning_stats->bloom_filter_read_time); - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } else { - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } - } catch (const ::parquet::ParquetException&) { - return nullptr; - } catch (const std::exception&) { - return nullptr; - } - } - auto it = column_bloom_filters.find(cache_key); - return it == column_bloom_filters.end() ? nullptr : it->second.get(); - } -}; - -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; -} - -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; -} - -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; -} - -bool is_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { - return false; - } - - const auto& encoding_stats = column_metadata.encoding_stats(); - if (!encoding_stats.empty()) { - bool has_dictionary_data_page = false; - for (const auto& encoding_stat : encoding_stats) { - if (!is_data_page_type(encoding_stat.page_type) || encoding_stat.count <= 0) { - continue; - } - if (!is_dictionary_data_encoding(encoding_stat.encoding)) { - return false; - } - has_dictionary_data_page = true; - } - return has_dictionary_data_page; - } - - bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { - if (is_dictionary_data_encoding(encoding)) { - has_dictionary_encoding = true; - continue; - } - if (!is_level_encoding(encoding)) { - return false; - } - } - return has_dictionary_encoding; -} - -bool supports_dictionary_pruning(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr) { - return false; - } - if (!column_schema.type_descriptor.is_string_like) { - return false; - } - if (column_metadata.type() != ::parquet::Type::BYTE_ARRAY && - column_metadata.type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - return false; - } - return true; -} - -} // namespace - -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words) { - DORIS_CHECK(dict_words != nullptr); - dict_words->clear(); - if (file_reader == nullptr || leaf_column_id < 0) { - return false; - } - - auto row_group_reader = file_reader->RowGroup(row_group_idx); - if (row_group_reader == nullptr) { - return false; - } - auto page_reader = row_group_reader->GetColumnPageReader(leaf_column_id); - if (page_reader == nullptr) { - return false; - } - - std::shared_ptr<::parquet::Page> page; - try { - page = page_reader->NextPage(); - } catch (const ::parquet::ParquetException&) { - return false; - } catch (const std::exception&) { - return false; - } - if (page == nullptr || page->type() != ::parquet::PageType::DICTIONARY_PAGE) { - return false; - } - const auto* dictionary_page = static_cast(page.get()); - if (dictionary_page->encoding() != ::parquet::Encoding::PLAIN && - dictionary_page->encoding() != ::parquet::Encoding::PLAIN_DICTIONARY) { - return false; - } - const int32_t dictionary_length = dictionary_page->num_values(); - if (dictionary_length <= 0) { - return false; - } - const auto* dictionary_data = dictionary_page->data(); - const int dictionary_size = dictionary_page->size(); - - dict_words->values.reserve(static_cast(dictionary_length)); - if (column_schema.descriptor->physical_type() == ::parquet::Type::BYTE_ARRAY) { - auto decoder = ::parquet::MakeTypedDecoder<::parquet::ByteArrayType>( - ::parquet::Encoding::PLAIN, column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::ByteArray> byte_array_values(static_cast(dictionary_length)); - if (decoder->Decode(byte_array_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(byte_array_values[dict_idx].ptr), - byte_array_values[dict_idx].len); - } - dict_words->build_refs(); - return true; - } - if (column_schema.descriptor->physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; - } - auto decoder = ::parquet::MakeTypedDecoder<::parquet::FLBAType>(::parquet::Encoding::PLAIN, - column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::FixedLenByteArray> flba_values( - static_cast(dictionary_length)); - if (decoder->Decode(flba_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(flba_values[dict_idx].ptr), type_length); - } - dict_words->build_refs(); - return true; - } - return false; -} - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words) { - std::vector fields; - fields.reserve(dict_words.refs.size()); - for (const auto& ref : dict_words.refs) { - fields.push_back(Field::create_field(String(ref.data, ref.size))); - } - return fields; -} - -namespace { - const ParquetColumnSchema* resolve_local_leaf_schema( const std::vector>& schema, const format::LocalColumnId file_column_id) { @@ -762,164 +520,111 @@ void accumulate_zonemap_stats(const ZoneMapEvalContext& ctx, ParquetPruningStats } // namespace +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state) { + return config::enable_parquet_page_index && has_expr_zonemap_filter(request, runtime_state); +} + std::shared_ptr ParquetStatisticsUtils::MakeZoneMap( const ParquetColumnStatistics& statistics) { return make_zonemap_from_statistics(statistics); } ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, const cctz::time_zone* timezone) { + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone) { ParquetColumnStatistics result; - if (statistics == nullptr) { + if (statistics == nullptr || column_value_count < 0) { return result; } - result.has_null = !statistics->HasNullCount() || statistics->null_count() > 0; - result.has_not_null = statistics->num_values() > 0 || statistics->HasMinMax(); - result.has_null_count = statistics->HasNullCount(); - if (!result.has_not_null || !statistics->HasMinMax()) { + if (statistics->__isset.null_count && statistics->null_count > column_value_count) { + // An impossible null count makes all derived min/max and all-null flags untrustworthy; + // disable pruning instead of turning corrupt footer metadata into false negatives. return result; } - DORIS_CHECK(column_schema.type != nullptr); - switch (statistics->physical_type()) { - case ::parquet::Type::BOOLEAN: - result.has_min_max = set_decoded_min_max<::parquet::BooleanType>( - statistics, column_schema, DecodedValueKind::BOOL, &result, timezone); - return result; - case ::parquet::Type::INT32: - result.has_min_max = set_decoded_min_max<::parquet::Int32Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::INT64: - result.has_min_max = set_decoded_min_max<::parquet::Int64Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::FLOAT: - result.has_min_max = set_decoded_min_max<::parquet::FloatType>( - statistics, column_schema, DecodedValueKind::FLOAT, &result, timezone); - return result; - case ::parquet::Type::DOUBLE: - result.has_min_max = set_decoded_min_max<::parquet::DoubleType>( - statistics, column_schema, DecodedValueKind::DOUBLE, &result, timezone); - return result; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result.has_min_max = set_string_min_max(statistics, column_schema, &result, timezone); - return result; - default: - return result; + const bool has_null_count = statistics->__isset.null_count && statistics->null_count >= 0; + const int64_t null_count = has_null_count ? statistics->null_count : 0; + const bool has_not_null = has_null_count ? column_value_count > null_count : true; + const std::string* min_value = statistics->__isset.min_value + ? &statistics->min_value + : (statistics->__isset.min ? &statistics->min : nullptr); + const std::string* max_value = statistics->__isset.max_value + ? &statistics->max_value + : (statistics->__isset.max ? &statistics->max : nullptr); + + tparquet::ColumnIndex index; + index.__set_null_pages({!has_not_null}); + index.__set_null_counts({null_count}); + if (min_value != nullptr && max_value != nullptr) { + index.__set_min_values({*min_value}); + index.__set_max_values({*max_value}); + } + // Footer statistics and page indexes share the same little-endian physical encoding. Reusing + // one decoder keeps native row-group and page pruning identical for logical types and NaNs. + if (!build_native_page_statistics(index, column_schema, 0, &result, timezone)) { + return {}; + } + if (!has_null_count) { + result.has_null_count = false; + result.has_null = true; } + return result; } -bool ParquetStatisticsUtils::BloomFilterExcludes(const ParquetColumnSchema& column_schema, - int slot_index, const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - return bloom_filter_excludes(column_schema, slot_index, conjuncts, bloom_filter); +bool ParquetStatisticsUtils::NativeBloomFilterExcludes( + const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, const segment_v2::BloomFilter& bloom_filter) { + if (!bloom_filter_supported(column_schema)) { + return false; + } + NativeParquetBloomFilterAdapter adapter(column_schema, bloom_filter); + BloomFilterEvalContext ctx; + ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { + .data_type = column_schema.type, + .bloom_filter = &adapter, + }); + return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; } namespace { -ParquetRowGroupPruneReason dictionary_prune_reason( - const ::parquet::RowGroupMetaData& row_group, ::parquet::ParquetFileReader* file_reader, - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request) { - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_dictionary_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { - continue; - } - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_dictionary_pruning(*column_schema, *column_chunk) || - !is_dictionary_encoded_chunk(*column_chunk)) { - continue; - } - - ParquetDictionaryWords dict_words; - if (!read_dictionary_words(file_reader, row_group_idx, column_schema->leaf_column_id, - *column_schema, &dict_words)) { - continue; - } - DictionaryEvalContext ctx; - ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { - .data_type = column_schema->type, - .values = dictionary_fields_from_words(dict_words), - }); - if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch) { - return ParquetRowGroupPruneReason::DICTIONARY; +void collect_filtered_leaf_ids(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::set* leaf_column_ids) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (column_schema.leaf_column_id >= 0) { + leaf_column_ids->insert(column_schema.leaf_column_id); } + return; } - return ParquetRowGroupPruneReason::NONE; -} - -ParquetRowGroupPruneReason bloom_filter_prune_reason( - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request, RowGroupBloomFilterCache* bloom_filter_cache, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_cache == nullptr) { - return ParquetRowGroupPruneReason::NONE; - } - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr || - !bloom_filter_supported(*column_schema)) { - continue; - } - auto* bloom_filter = bloom_filter_cache->get(row_group_idx, column_schema->leaf_column_id, - pruning_stats); - if (bloom_filter == nullptr) { + for (const auto& child_schema : column_schema.children) { + if (!format::is_child_projected(projection, child_schema->local_id)) { continue; } - if (ParquetStatisticsUtils::BloomFilterExcludes(*column_schema, slot_index, conjuncts, - *bloom_filter)) { - return ParquetRowGroupPruneReason::BLOOM_FILTER; - } + collect_filtered_leaf_ids(*child_schema, + format::find_child_projection(projection, child_schema->local_id), + leaf_column_ids); } - return ParquetRowGroupPruneReason::NONE; } -void init_bloom_filter_cache(::parquet::ParquetFileReader* file_reader, bool enable_bloom_filter, - RowGroupBloomFilterCache* bloom_filter_cache) { - DORIS_CHECK(bloom_filter_cache != nullptr); - if (!enable_bloom_filter || file_reader == nullptr) { - return; - } - try { - bloom_filter_cache->bloom_filter_reader = &file_reader->GetBloomFilterReader(); - } catch (const ::parquet::ParquetException&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } catch (const std::exception&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } +bool native_metadata_predicate_is_type_safe(const ParquetColumnSchema& column_schema) { + DORIS_CHECK(column_schema.type != nullptr); + // Raw VARBINARY file slots may feed table-side STRING casts. Footer/page metadata is still in + // the pre-cast domain, so using it for a rewritten table predicate can cause false negatives. + return remove_nullable(column_schema.type)->get_primitive_type() != TYPE_VARBINARY; } -bool check_statistics(const ::parquet::RowGroupMetaData& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { +bool check_native_statistics(const tparquet::FileMetaData& metadata, + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) { const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts); if (slot_indexes.empty()) { return false; } - ZoneMapEvalContext ctx; for (const int slot_index : slot_indexes) { const auto file_column_id = file_column_id_by_block_position(request, slot_index); @@ -927,270 +632,320 @@ bool check_statistics(const ::parquet::RowGroupMetaData& row_group, continue; } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { continue; } - + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; std::shared_ptr zone_map; - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk != nullptr) { + if (chunk.__isset.meta_data) { + const auto& column_metadata = chunk.meta_data; + std::optional safe_statistics; + if (column_metadata.__isset.statistics) { + safe_statistics = detail::sanitize_native_footer_statistics( + column_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order(metadata, + column_schema->leaf_column_id)); + } zone_map = ParquetStatisticsUtils::MakeZoneMap( ParquetStatisticsUtils::TransformColumnStatistics( - *column_schema, column_chunk->statistics(), timezone)); + *column_schema, + safe_statistics.has_value() ? &*safe_statistics : nullptr, + column_metadata.num_values, timezone)); } add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map)); } - const auto result = VExprContext::evaluate_zonemap_filter(request.conjuncts, ctx); accumulate_zonemap_stats(ctx, pruning_stats); return result == ZoneMapFilterResult::kNoMatch; } -Status select_row_groups_by_metadata_impl( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - int64_t row_group_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &row_group_filter_time_sink - : &pruning_stats->row_group_filter_time); - if (selected_row_groups == nullptr) { - return Status::InvalidArgument("selected_row_groups is null"); - } - selected_row_groups->clear(); +bool is_native_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; +} - const int num_row_groups = metadata.num_row_groups(); - const auto candidate_size = candidate_row_groups == nullptr - ? static_cast(num_row_groups) - : candidate_row_groups->size(); - if (pruning_stats != nullptr) { - // Scan-range ownership is decided before metadata pruning. Count only row groups owned by - // this split so a file divided into multiple splits does not report the full-file total and - // out-of-split groups once per split. - pruning_stats->total_row_groups = cast_set(candidate_size); +bool is_native_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; +} + +bool is_native_dictionary_encoded_chunk(const tparquet::ColumnMetaData& metadata) { + if (!metadata.__isset.dictionary_page_offset || metadata.dictionary_page_offset < 0) { + return false; } - selected_row_groups->reserve(candidate_size); - RowGroupBloomFilterCache bloom_filter_cache; - init_bloom_filter_cache(file_reader, enable_bloom_filter, &bloom_filter_cache); - for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { - const int row_group_idx = candidate_row_groups == nullptr - ? static_cast(candidate_idx) - : (*candidate_row_groups)[candidate_idx]; - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(row_group_idx < num_row_groups); - auto row_group = metadata.RowGroup(row_group_idx); - if (row_group == nullptr) { - selected_row_groups->push_back(row_group_idx); - continue; + if (metadata.__isset.encoding_stats && !metadata.encoding_stats.empty()) { + bool has_dictionary_data_page = false; + for (const auto& encoding_stat : metadata.encoding_stats) { + if ((encoding_stat.page_type != tparquet::PageType::DATA_PAGE && + encoding_stat.page_type != tparquet::PageType::DATA_PAGE_V2) || + encoding_stat.count <= 0) { + continue; + } + if (!is_native_dictionary_data_encoding(encoding_stat.encoding)) { + return false; + } + has_dictionary_data_page = true; } - ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; - if (has_expr_zonemap_filter(request, runtime_state) && - check_statistics(*row_group, file_schema, request, pruning_stats, timezone)) { - prune_reason = ParquetRowGroupPruneReason::STATISTICS; + return has_dictionary_data_page; + } + bool has_dictionary_encoding = false; + for (const auto encoding : metadata.encodings) { + if (is_native_dictionary_data_encoding(encoding)) { + has_dictionary_encoding = true; + } else if (!is_native_level_encoding(encoding)) { + return false; } + } + return has_dictionary_encoding; +} - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = dictionary_prune_reason(*row_group, file_reader, row_group_idx, - file_schema, request); - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = bloom_filter_prune_reason(row_group_idx, file_schema, request, - &bloom_filter_cache, pruning_stats); - } +const format::LocalColumnIndex* find_request_projection(const format::FileScanRequest& request, + format::LocalColumnId file_column_id) { + for (const auto& projection : request.predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; } - - if (prune_reason != ParquetRowGroupPruneReason::NONE) { - if (pruning_stats != nullptr) { - pruning_stats->filtered_group_rows += row_group->num_rows(); - if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { - ++pruning_stats->filtered_row_groups_by_statistics; - } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { - ++pruning_stats->filtered_row_groups_by_dictionary; - } else if (prune_reason == ParquetRowGroupPruneReason::BLOOM_FILTER) { - ++pruning_stats->filtered_row_groups_by_bloom_filter; - } - } - continue; + } + for (const auto& projection : request.non_predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; } - selected_row_groups->push_back(row_group_idx); } - return Status::OK(); + return nullptr; } -} // namespace - -Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, +ParquetRowGroupPruneReason native_dictionary_prune_reason( + const tparquet::RowGroup& row_group, int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - return select_row_groups_by_metadata_impl( - metadata, file_reader, file_schema, request, candidate_row_groups, selected_row_groups, - enable_bloom_filter, pruning_stats, timezone, runtime_state); -} - -namespace { - -template -bool set_page_decoded_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - DecodedValueKind value_kind, ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - const auto typed_index = - std::static_pointer_cast<::parquet::TypedColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; + const format::FileScanRequest& request, const cctz::time_zone* timezone, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile) { + if (file_context == nullptr || file_context->native_metadata == nullptr) { + return ParquetRowGroupPruneReason::NONE; } - const typename ParquetDType::c_type min_value = typed_index->min_values()[page_idx]; - const typename ParquetDType::c_type max_value = typed_index->max_values()[page_idx]; - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_dictionary_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + const auto* projection = find_request_projection(request, *file_column_id); + if (column_schema == nullptr || projection == nullptr || column_schema->type == nullptr || + !column_schema->type_descriptor.is_string_like || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + if (!native_metadata_predicate_is_type_safe(*column_schema)) { + // The file-local VARBINARY may feed a table-side STRING cast. Pruning before that cast + // can compare different Field kinds and incorrectly discard a matching row group. + continue; + } + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data || + (chunk.meta_data.type != tparquet::Type::BYTE_ARRAY && + chunk.meta_data.type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) || + !is_native_dictionary_encoded_chunk(chunk.meta_data)) { + continue; + } + std::unique_ptr reader; + const std::vector ranges {{0, row_group.num_rows}}; + const std::unordered_map offset_indexes; + // Metadata pruning uses the real native reader, so its page work must be attributed to the + // scan profile even when the row group is eliminated before execution readers are built. + const auto status = NativeColumnReader::create( + *column_schema, projection, file_context->native_file, + file_context->native_metadata, row_group_idx, ranges, offset_indexes, timezone, + file_context->native_io_ctx, nullptr, file_context->native_page_cache_enabled, + file_context->native_page_cache_file_key, true, column_reader_profile, &reader); + if (!status.ok() || reader == nullptr) { + continue; + } + auto dictionary_result = reader->dictionary_values(); + if (!dictionary_result.has_value()) { + continue; + } + auto dictionary = std::move(dictionary_result).value(); + std::vector values(dictionary->size()); + for (size_t value_idx = 0; value_idx < dictionary->size(); ++value_idx) { + dictionary->get(value_idx, values[value_idx]); + } + DictionaryEvalContext ctx; + ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { + .data_type = column_schema->type, + .values = std::move(values), + }); + if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == + ZoneMapFilterResult::kNoMatch) { + return ParquetRowGroupPruneReason::DICTIONARY; } } - if (!valid_min_max(min_value, max_value)) { - // A NaN invalidates only this page's bounds, not the ColumnIndex itself. Keep the page - // conservatively by returning usable null-count statistics with has_min_max=false, while - // allowing later pages with finite bounds to remain eligible for pruning. - return true; - } - if (!set_decoded_field(column_schema, value_kind, min_value, &page_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &page_statistics->max_value, - timezone)) { - return false; - } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; - } - page_statistics->has_min_max = true; - return true; + return ParquetRowGroupPruneReason::NONE; } -bool set_page_string_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: { - const auto typed_index = - std::static_pointer_cast<::parquet::ByteArrayColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; - } - const auto min = ::parquet::ByteArrayToString(typed_index->min_values()[page_idx]); - const auto max = ::parquet::ByteArrayToString(typed_index->max_values()[page_idx]); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; +ParquetRowGroupPruneReason native_bloom_filter_prune_reason( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, ParquetFileContext* file_context, + ParquetPruningStats* pruning_stats) { + if (file_context == nullptr || file_context->native_file == nullptr) { + return ParquetRowGroupPruneReason::NONE; + } + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + !bloom_filter_supported(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; } - page_statistics->has_min_max = true; - return true; - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data) { + continue; } - const auto typed_index = std::static_pointer_cast<::parquet::FLBAColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; + std::unique_ptr bloom_filter; + Status status; + { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->bloom_filter_read_time); + status = read_native_bloom_filter(chunk.meta_data, file_context->native_file, + file_context->native_io_ctx, &bloom_filter); } - const std::string min( - reinterpret_cast(typed_index->min_values()[page_idx].ptr), - type_length); - const std::string max( - reinterpret_cast(typed_index->max_values()[page_idx].ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; + if (!status.ok() || bloom_filter == nullptr) { + continue; } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; + if (ParquetStatisticsUtils::NativeBloomFilterExcludes(*column_schema, slot_index, conjuncts, + *bloom_filter)) { + return ParquetRowGroupPruneReason::BLOOM_FILTER; } - page_statistics->has_min_max = true; - return true; - } - default: - return false; } + return ParquetRowGroupPruneReason::NONE; } -bool set_page_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - DORIS_CHECK(column_schema.type != nullptr); - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BOOLEAN: - return set_page_decoded_min_max<::parquet::BooleanType>(column_index, column_schema, - page_idx, DecodedValueKind::BOOL, - page_statistics, timezone); - case ::parquet::Type::INT32: - return set_page_decoded_min_max<::parquet::Int32Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::INT64: - return set_page_decoded_min_max<::parquet::Int64Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::FLOAT: - return set_page_decoded_min_max<::parquet::FloatType>(column_index, column_schema, page_idx, - DecodedValueKind::FLOAT, - page_statistics, timezone); - case ::parquet::Type::DOUBLE: - return set_page_decoded_min_max<::parquet::DoubleType>(column_index, column_schema, - page_idx, DecodedValueKind::DOUBLE, - page_statistics, timezone); - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return set_page_string_min_max(column_index, column_schema, page_idx, page_statistics, - timezone); - default: - return false; +int64_t native_requested_compressed_bytes( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request) { + std::set leaf_column_ids; + auto collect_projection = [&](const format::LocalColumnIndex& projection) { + const int32_t local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return; + } + collect_filtered_leaf_ids(*file_schema[local_id], &projection, &leaf_column_ids); + }; + for (const auto& projection : request.predicate_columns) { + collect_projection(projection); + } + for (const auto& projection : request.non_predicate_columns) { + collect_projection(projection); } + int64_t bytes = 0; + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (chunk.__isset.meta_data && chunk.meta_data.total_compressed_size > 0) { + bytes += chunk.meta_data.total_compressed_size; + } + } + return bytes; } -bool build_page_statistics(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - DORIS_CHECK(page_statistics != nullptr); - *page_statistics = ParquetColumnStatistics {}; +} // namespace - const auto& null_pages = column_index->null_pages(); - if (!column_index->has_null_counts() || page_idx >= null_pages.size() || - page_idx >= column_index->null_counts().size()) { - return false; +Status select_row_groups_by_metadata( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector* candidate_row_groups, + std::vector* selected_row_groups, bool enable_bloom_filter, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile, + ParquetMetadataProbeMode probe_mode) { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->row_group_filter_time); + if (selected_row_groups == nullptr) { + return Status::InvalidArgument("selected_row_groups is null"); } - - page_statistics->has_null_count = true; - page_statistics->has_null = column_index->null_counts()[page_idx] > 0; - page_statistics->has_not_null = !null_pages[page_idx]; - if (!page_statistics->has_not_null) { - return true; + selected_row_groups->clear(); + const size_t candidate_size = candidate_row_groups == nullptr ? metadata.row_groups.size() + : candidate_row_groups->size(); + if (pruning_stats != nullptr) { + pruning_stats->total_row_groups = cast_set(candidate_size); } - return set_page_min_max(column_index, column_schema, page_idx, page_statistics, timezone); + selected_row_groups->reserve(candidate_size); + for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { + const int row_group_idx = candidate_row_groups == nullptr + ? static_cast(candidate_idx) + : (*candidate_row_groups)[candidate_idx]; + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + // Candidate ids originate in external split metadata; a corrupt id must not terminate + // the BE while planning an otherwise recoverable file scan. + return Status::Corruption("Invalid Parquet row group candidate {} for {} row groups", + row_group_idx, metadata.row_groups.size()); + } + const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.num_rows == 0) { + // Native metadata probes construct positive row ranges; empty groups contribute no + // rows and must be discarded before dictionary, statistics, or Bloom reader setup. + continue; + } + ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; + if (probe_mode != ParquetMetadataProbeMode::EXPENSIVE_ONLY && + has_expr_zonemap_filter(request, runtime_state) && + check_native_statistics(metadata, row_group, file_schema, request, pruning_stats, + timezone)) { + prune_reason = ParquetRowGroupPruneReason::STATISTICS; + } + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE) { + prune_reason = + native_dictionary_prune_reason(row_group, row_group_idx, file_schema, request, + timezone, file_context, column_reader_profile); + } + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE && enable_bloom_filter) { + prune_reason = native_bloom_filter_prune_reason(row_group, file_schema, request, + file_context, pruning_stats); + } + if (prune_reason == ParquetRowGroupPruneReason::NONE) { + selected_row_groups->push_back(row_group_idx); + continue; + } + if (pruning_stats != nullptr) { + pruning_stats->filtered_group_rows += row_group.num_rows; + pruning_stats->filtered_bytes += + native_requested_compressed_bytes(row_group, file_schema, request); + if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { + ++pruning_stats->filtered_row_groups_by_statistics; + } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { + ++pruning_stats->filtered_row_groups_by_dictionary; + } else { + ++pruning_stats->filtered_row_groups_by_bloom_filter; + } + } + } + return Status::OK(); } +namespace { + std::vector intersect_ranges(const std::vector& left, const std::vector& right) { std::vector result; @@ -1223,19 +978,6 @@ int64_t count_range_rows(const std::vector& ranges) { return rows; } -RowRange page_row_range(const ::parquet::OffsetIndex& offset_index, size_t page_idx, - int64_t row_group_rows) { - const auto& page_locations = offset_index.page_locations(); - const int64_t start = page_locations[page_idx].first_row_index; - const int64_t end = page_idx + 1 == page_locations.size() - ? row_group_rows - : page_locations[page_idx + 1].first_row_index; - DORIS_CHECK(start >= 0); - DORIS_CHECK(end >= start); - DORIS_CHECK(end <= row_group_rows); - return RowRange {start, end - start}; -} - void append_row_range(const RowRange& range, std::vector* ranges) { if (range.length == 0) { return; @@ -1250,85 +992,6 @@ void append_row_range(const RowRange& range, std::vector* ranges) { ranges->push_back(range); } -std::optional< - std::pair, std::shared_ptr<::parquet::OffsetIndex>>> -load_page_indexes_for_slot(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, - const ParquetColumnSchema** column_schema) { - DORIS_CHECK(column_schema != nullptr); - *column_schema = nullptr; - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - return std::nullopt; - } - *column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (*column_schema == nullptr || (*column_schema)->descriptor == nullptr) { - return std::nullopt; - } - - try { - auto column_index = row_group->GetColumnIndex((*column_schema)->leaf_column_id); - auto offset_index = row_group->GetOffsetIndex((*column_schema)->leaf_column_id); - if (column_index == nullptr || offset_index == nullptr || - column_index->null_pages().size() != offset_index->page_locations().size()) { - return std::nullopt; - } - return std::make_pair(std::move(column_index), std::move(offset_index)); - } catch (const ::parquet::ParquetException&) { - return std::nullopt; - } catch (const std::exception&) { - return std::nullopt; - } -} - -bool select_ranges_for_expr_zonemap( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, const VExprContextSPtrs& conjuncts, - int64_t row_group_rows, std::vector* ranges, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { - DORIS_CHECK(ranges != nullptr); - if (conjuncts.empty()) { - return false; - } - const ParquetColumnSchema* column_schema = nullptr; - const auto page_indexes = - load_page_indexes_for_slot(row_group, file_schema, request, slot_index, &column_schema); - if (!page_indexes.has_value()) { - return false; - } - const auto& [column_index, offset_index] = *page_indexes; - - ranges->clear(); - ZoneMapEvalStats page_stats; - const auto page_count = offset_index->page_locations().size(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - ParquetColumnStatistics page_statistics; - if (!ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *column_schema, page_idx, &page_statistics, timezone)) { - ranges->clear(); - return false; - } - - ZoneMapEvalContext ctx; - add_slot_zonemap(&ctx, slot_index, column_schema->type, - ParquetStatisticsUtils::MakeZoneMap(page_statistics)); - const auto result = VExprContext::evaluate_zonemap_filter(conjuncts, ctx); - page_stats.merge_page_eval_stats(ctx.stats); - if (result == ZoneMapFilterResult::kNoMatch) { - continue; - } - append_row_range(page_row_range(*offset_index, page_idx, row_group_rows), ranges); - } - if (pruning_stats != nullptr) { - pruning_stats->expr_zonemap_unusable_evals += page_stats.unusable_zonemap_eval_count; - pruning_stats->in_zonemap_point_check_count += page_stats.in_zonemap_point_check_count; - pruning_stats->in_zonemap_range_only_count += page_stats.in_zonemap_range_only_count; - } - return true; -} - bool ranges_intersect(const std::vector& ranges, const RowRange& range) { const int64_t range_end = range.start + range.length; for (const auto& selected_range : ranges) { @@ -1388,127 +1051,148 @@ void collect_request_leaf_schemas( } } -bool build_page_skip_plan_for_leaf( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const ParquetColumnSchema& column_schema, const std::vector& selected_ranges, - int64_t row_group_rows, ParquetPageSkipPlan* page_skip_plan) { - DORIS_CHECK(page_skip_plan != nullptr); - *page_skip_plan = ParquetPageSkipPlan {}; - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.leaf_column_id < 0 || - column_schema.descriptor->max_repetition_level() != 0) { +template +bool set_native_page_scalar_min_max(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + DecodedValueKind kind, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + if (page_idx >= column_index.min_values.size() || page_idx >= column_index.max_values.size() || + column_index.min_values[page_idx].size() != sizeof(ValueType) || + column_index.max_values[page_idx].size() != sizeof(ValueType)) { return false; } - - std::shared_ptr<::parquet::OffsetIndex> offset_index; - try { - offset_index = row_group->GetOffsetIndex(column_schema.leaf_column_id); - } catch (const ::parquet::ParquetException&) { + const auto min_value = unaligned_load(column_index.min_values[page_idx].data()); + const auto max_value = unaligned_load(column_index.max_values[page_idx].data()); + if constexpr (std::is_same_v) { + if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { + return false; + } + } + if (!valid_min_max(min_value, max_value)) { + return true; + } + if (!set_decoded_field(column_schema, kind, min_value, &page_statistics->min_value, timezone) || + !set_decoded_field(column_schema, kind, max_value, &page_statistics->max_value, timezone)) { return false; - } catch (const std::exception&) { + } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; +} + +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + DORIS_CHECK(page_statistics != nullptr); + *page_statistics = {}; + if (!column_index.__isset.null_counts || page_idx >= column_index.null_pages.size() || + page_idx >= column_index.null_counts.size()) { return false; } - if (offset_index == nullptr) { + const int64_t null_count = column_index.null_counts[page_idx]; + if (null_count < 0 || (column_index.null_pages[page_idx] && null_count == 0)) { + // Contradictory optional index metadata must disable pruning; treating it as an empty + // null set can make IS NULL/IS NOT NULL discard rows without reading the data page. return false; } - - const auto page_count = offset_index->page_locations().size(); - page_skip_plan->leaf_column_id = column_schema.leaf_column_id; - page_skip_plan->skipped_pages.resize(page_count); - page_skip_plan->skipped_page_compressed_sizes.resize(page_count); - const auto& page_locations = offset_index->page_locations(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - const RowRange row_range = page_row_range(*offset_index, page_idx, row_group_rows); - if (row_range.length == 0 || ranges_intersect(selected_ranges, row_range)) { - continue; + page_statistics->has_null_count = true; + page_statistics->has_null = null_count > 0; + page_statistics->has_not_null = !column_index.null_pages[page_idx]; + if (!page_statistics->has_not_null) { + return true; + } + switch (column_schema.type_descriptor.physical_type) { + case tparquet::Type::BOOLEAN: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::BOOL, page_statistics, + timezone); + case tparquet::Type::INT32: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case tparquet::Type::INT64: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case tparquet::Type::FLOAT: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::FLOAT, page_statistics, + timezone); + case tparquet::Type::DOUBLE: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::DOUBLE, page_statistics, + timezone); + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { + if (page_idx >= column_index.min_values.size() || + page_idx >= column_index.max_values.size()) { + return false; + } + const auto& min_value = column_index.min_values[page_idx]; + const auto& max_value = column_index.max_values[page_idx]; + const bool fixed = + column_schema.type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; + if (fixed && + (column_schema.type_descriptor.fixed_length <= 0 || + min_value.size() != static_cast(column_schema.type_descriptor.fixed_length) || + max_value.size() != static_cast(column_schema.type_descriptor.fixed_length))) { + return false; } - page_skip_plan->skipped_pages[page_idx] = 1; - page_skip_plan->skipped_page_compressed_sizes[page_idx] = - page_locations[page_idx].compressed_page_size; - append_row_range(row_range, &page_skip_plan->skipped_ranges); + const auto kind = fixed ? DecodedValueKind::FIXED_BINARY : DecodedValueKind::BINARY; + if (!set_decoded_binary_field(column_schema, kind, + StringRef(min_value.data(), min_value.size()), + &page_statistics->min_value, timezone) || + !set_decoded_binary_field(column_schema, kind, + StringRef(max_value.data(), max_value.size()), + &page_statistics->max_value, timezone)) { + return false; + } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; } - if (page_skip_plan->empty()) { - *page_skip_plan = ParquetPageSkipPlan {}; + default: return false; } - return true; } -void build_page_skip_plans(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, - const std::vector& selected_ranges, int64_t row_group_rows, - std::map* page_skip_plans) { - DORIS_CHECK(page_skip_plans != nullptr); - page_skip_plans->clear(); - std::vector leaf_schemas; - collect_request_leaf_schemas(file_schema, request, &leaf_schemas); - for (const auto* leaf_schema : leaf_schemas) { - DORIS_CHECK(leaf_schema != nullptr); - ParquetPageSkipPlan page_skip_plan; - if (build_page_skip_plan_for_leaf(row_group, *leaf_schema, selected_ranges, row_group_rows, - &page_skip_plan)) { - page_skip_plans->emplace(page_skip_plan.leaf_column_id, std::move(page_skip_plan)); - } - } +RowRange native_page_row_range(const tparquet::OffsetIndex& offset_index, size_t page_idx, + int64_t row_group_rows) { + const auto& locations = offset_index.page_locations; + const int64_t start = locations[page_idx].first_row_index; + const int64_t end = page_idx + 1 == locations.size() ? row_group_rows + : locations[page_idx + 1].first_row_index; + return {.start = start, .length = end - start}; } } // namespace -bool ParquetStatisticsUtils::TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - return build_page_statistics(column_index, column_schema, page_idx, page_statistics, timezone); -} - -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, +Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, + const std::unordered_map& page_indexes, const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + const format::FileScanRequest& request, int64_t row_group_rows, std::vector* selected_ranges, std::map* page_skip_plans, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState* runtime_state) { - int64_t page_index_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &page_index_filter_time_sink + int64_t filter_time_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &filter_time_sink : &pruning_stats->page_index_filter_time); DORIS_CHECK(selected_ranges != nullptr); selected_ranges->clear(); + selected_ranges->push_back({.start = 0, .length = row_group_rows}); if (page_skip_plans != nullptr) { page_skip_plans->clear(); } - if (row_group_rows <= 0) { - return Status::OK(); - } - selected_ranges->push_back(RowRange {0, row_group_rows}); - if (!config::enable_parquet_page_index || !has_expr_zonemap_filter(request, runtime_state) || - file_reader == nullptr) { + if (row_group_rows <= 0 || !config::enable_parquet_page_index || + !has_expr_zonemap_filter(request, runtime_state) || page_indexes.empty()) { return Status::OK(); } - - std::shared_ptr<::parquet::PageIndexReader> page_index_reader; - std::shared_ptr<::parquet::RowGroupPageIndexReader> row_group_index_reader; - try { - if (pruning_stats != nullptr) { - ++pruning_stats->page_index_read_calls; - } - { - int64_t read_page_index_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &read_page_index_time_sink - : &pruning_stats->read_page_index_time); - page_index_reader = file_reader->GetPageIndexReader(); - if (page_index_reader == nullptr) { - return Status::OK(); - } - row_group_index_reader = page_index_reader->RowGroup(row_group_idx); - } - } catch (const ::parquet::ParquetException&) { - return Status::OK(); - } catch (const std::exception&) { - return Status::OK(); - } - if (row_group_index_reader == nullptr) { - return Status::OK(); + if (pruning_stats != nullptr) { + ++pruning_stats->page_index_read_calls; } std::map conjuncts_by_slot; @@ -1518,19 +1202,53 @@ Status select_row_group_ranges_by_page_index( conjuncts_by_slot[slot_index].push_back(conjunct); } } - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + !detail::has_supported_type_defined_order(metadata, column_schema->leaf_column_id)) { + continue; + } + const auto index_it = page_indexes.find(column_schema->leaf_column_id); + if (index_it == page_indexes.end()) { + continue; + } + const auto& indexes = index_it->second; std::vector filter_ranges; - if (!select_ranges_for_expr_zonemap(row_group_index_reader, file_schema, request, - slot_index, conjuncts, row_group_rows, &filter_ranges, - pruning_stats, timezone)) { + bool usable = true; + for (size_t page_idx = 0; page_idx < indexes.offset_index.page_locations.size(); + ++page_idx) { + ParquetColumnStatistics statistics; + if (!build_native_page_statistics(indexes.column_index, *column_schema, page_idx, + &statistics, timezone)) { + usable = false; + break; + } + ZoneMapEvalContext ctx; + add_slot_zonemap(&ctx, slot_index, column_schema->type, + ParquetStatisticsUtils::MakeZoneMap(statistics)); + if (VExprContext::evaluate_zonemap_filter(conjuncts, ctx) != + ZoneMapFilterResult::kNoMatch) { + append_row_range( + native_page_row_range(indexes.offset_index, page_idx, row_group_rows), + &filter_ranges); + } + if (pruning_stats != nullptr) { + pruning_stats->expr_zonemap_unusable_evals += ctx.stats.unusable_zonemap_eval_count; + pruning_stats->in_zonemap_point_check_count += + ctx.stats.in_zonemap_point_check_count; + pruning_stats->in_zonemap_range_only_count += ctx.stats.in_zonemap_range_only_count; + } + } + if (!usable) { continue; } *selected_ranges = intersect_ranges(*selected_ranges, filter_ranges); if (selected_ranges->empty()) { - if (page_skip_plans != nullptr) { - page_skip_plans->clear(); - } if (pruning_stats != nullptr) { pruning_stats->filtered_page_rows += row_group_rows; ++pruning_stats->filtered_row_groups_by_page_index; @@ -1538,14 +1256,37 @@ Status select_row_group_ranges_by_page_index( return Status::OK(); } } + if (page_skip_plans != nullptr) { - build_page_skip_plans(row_group_index_reader, file_schema, request, *selected_ranges, - row_group_rows, page_skip_plans); + std::vector leaves; + collect_request_leaf_schemas(file_schema, request, &leaves); + for (const auto* leaf : leaves) { + const auto index_it = page_indexes.find(leaf->leaf_column_id); + if (index_it == page_indexes.end() || leaf->max_repetition_level != 0) { + continue; + } + const auto& offset_index = index_it->second.offset_index; + ParquetPageSkipPlan skip_plan; + skip_plan.leaf_column_id = leaf->leaf_column_id; + skip_plan.skipped_pages.resize(offset_index.page_locations.size()); + skip_plan.skipped_page_compressed_sizes.resize(offset_index.page_locations.size()); + for (size_t page_idx = 0; page_idx < offset_index.page_locations.size(); ++page_idx) { + const auto range = native_page_row_range(offset_index, page_idx, row_group_rows); + if (range.length == 0 || ranges_intersect(*selected_ranges, range)) { + continue; + } + skip_plan.skipped_pages[page_idx] = 1; + skip_plan.skipped_page_compressed_sizes[page_idx] = + offset_index.page_locations[page_idx].compressed_page_size; + append_row_range(range, &skip_plan.skipped_ranges); + } + if (!skip_plan.empty()) { + page_skip_plans->emplace(skip_plan.leaf_column_id, std::move(skip_plan)); + } + } } if (pruning_stats != nullptr) { - const int64_t selected_rows = count_range_rows(*selected_ranges); - DORIS_CHECK(selected_rows <= row_group_rows); - pruning_stats->filtered_page_rows += row_group_rows - selected_rows; + pruning_stats->filtered_page_rows += row_group_rows - count_range_rows(*selected_ranges); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 9e94562bf2d37b..72381548656f9d 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include "common/status.h" @@ -27,16 +30,9 @@ #include "core/string_ref.h" #include "exprs/vexpr_fwd.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/selection_vector.h" -namespace parquet { -class BloomFilter; -class ColumnIndex; -class FileMetaData; -class ParquetFileReader; -class Statistics; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -44,6 +40,7 @@ class time_zone; namespace doris { class RuntimeState; namespace segment_v2 { +class BloomFilter; struct ZoneMap; } // namespace segment_v2 } // namespace doris @@ -51,36 +48,21 @@ struct ZoneMap; namespace doris::format::parquet { struct ParquetColumnSchema; - -// ============================================================================ -// ============================================================================ - -struct ParquetDictionaryWords { - std::vector values; - std::vector refs; - - void clear() { - values.clear(); - refs.clear(); - } - - void build_refs() { - refs.clear(); - refs.reserve(values.size()); - for (const auto& value : values) { - refs.emplace_back(value.data(), value.size()); - } - } -}; - -// Reads the PLAIN dictionary page for BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY columns and owns copied -// dictionary bytes in `values`. Both row-group pruning and row-level dictionary predicates use this -// helper so they agree on dictionary id -> Doris string value mapping. -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words); - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words); +struct ParquetFileContext; +struct ParquetTypeDescriptor; + +namespace detail { +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size); +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order); +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id); +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order); +} // namespace detail // ============================================================================ // ============================================================================ @@ -93,6 +75,7 @@ struct ParquetPruningStats { int64_t filtered_row_groups_by_bloom_filter = 0; // row groups pruned by bloom filter int64_t filtered_row_groups_by_page_index = 0; // row groups fully pruned by page index int64_t filtered_group_rows = 0; // rows in pruned row groups + int64_t filtered_bytes = 0; // requested bytes in pruned row groups int64_t filtered_page_rows = 0; // rows pruned by page index int64_t selected_row_ranges = 0; // selected row range count int64_t page_index_read_calls = 0; // Page Index read count @@ -100,6 +83,7 @@ struct ParquetPruningStats { int64_t row_group_filter_time = 0; // row-group pruning time (ns) int64_t page_index_filter_time = 0; // page-index pruning time (ns) int64_t read_page_index_time = 0; // page-index read time (ns) + int64_t parse_page_index_time = 0; // lazy page-index materialization time (ns) int64_t expr_zonemap_unusable_evals = 0; // VExpr ZoneMap unusable evaluations int64_t in_zonemap_point_check_count = 0; // VExpr IN ZoneMap point checks int64_t in_zonemap_range_only_count = 0; // VExpr IN ZoneMap range-only checks @@ -116,6 +100,20 @@ struct ParquetColumnStatistics { bool has_any_statistics() const { return has_null_count || has_min_max; } }; +struct NativeParquetPageIndex { + tparquet::ColumnIndex column_index; + tparquet::OffsetIndex offset_index; +}; + +enum class ParquetMetadataProbeMode { + ALL, + FOOTER_ONLY, + EXPENSIVE_ONLY, +}; + +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state); + // ============================================================================ // ============================================================================ // VExpr ZoneMap(TransformColumnStatistics + evaluate_zonemap_filter) @@ -128,32 +126,29 @@ struct ParquetStatisticsUtils { const ParquetColumnStatistics& statistics); static ParquetColumnStatistics TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, - const cctz::time_zone* timezone = nullptr); - - static bool TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone = nullptr); - - static bool BloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter); + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone = nullptr); + + static bool NativeBloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, + const segment_v2::BloomFilter& bloom_filter); }; Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector* candidate_row_groups, std::vector* selected_row_groups, bool enable_bloom_filter, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); + const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}, + ParquetMetadataProbeMode probe_mode = ParquetMetadataProbeMode::ALL); -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, +Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, + const std::unordered_map& page_indexes, const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + const format::FileScanRequest& request, int64_t row_group_rows, std::vector* selected_ranges, std::map* page_skip_plans, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, const RuntimeState* runtime_state = nullptr); diff --git a/be/src/format_v2/parquet/parquet_type.cpp b/be/src/format_v2/parquet/parquet_type.cpp index 8411d53f0fba91..be5ecd1c8b0772 100644 --- a/be/src/format_v2/parquet/parquet_type.cpp +++ b/be/src/format_v2/parquet/parquet_type.cpp @@ -17,340 +17,31 @@ #include "format_v2/parquet/parquet_type.h" -#include - -#include -#include - -#include "core/data_type/data_type_factory.hpp" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/primitive_type.h" - namespace doris::format::parquet { -namespace { - -DataTypePtr create_type(PrimitiveType type, bool nullable, int precision = 0, int scale = 0) { - return DataTypeFactory::instance().create_data_type(type, nullable, precision, scale); -} - -PrimitiveType decimal_primitive_type(int precision) { - return precision > 38 ? TYPE_DECIMAL256 : TYPE_DECIMAL128I; -} - -void mark_decimal(const ::parquet::ColumnDescriptor* column, int precision, int scale, - ParquetTypeDescriptor* result) { - result->is_decimal = true; - result->decimal_precision = precision; - result->decimal_scale = scale; - switch (column->physical_type()) { - case ::parquet::Type::INT32: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; - break; - case ::parquet::Type::INT64: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; - break; - default: - result->extra_type_info = ParquetExtraTypeInfo::NONE; - break; - } -} - -void mark_integer(int bit_width, bool is_signed, ParquetTypeDescriptor* result) { - result->integer_bit_width = bit_width; - result->is_unsigned_integer = !is_signed; -} - -DataTypePtr converted_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const bool nullable = column->max_definition_level() > 0; - switch (column->converted_type()) { - case ::parquet::ConvertedType::UTF8: - case ::parquet::ConvertedType::ENUM: - case ::parquet::ConvertedType::JSON: - case ::parquet::ConvertedType::BSON: - return create_type(TYPE_STRING, nullable); - case ::parquet::ConvertedType::DECIMAL: - mark_decimal(column, column->type_precision(), column->type_scale(), result); - return create_type(decimal_primitive_type(column->type_precision()), nullable, - column->type_precision(), column->type_scale()); - case ::parquet::ConvertedType::DATE: - return create_type(TYPE_DATEV2, nullable); - case ::parquet::ConvertedType::TIME_MILLIS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIME_MICROS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIMESTAMP_MILLIS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 3); - case ::parquet::ConvertedType::TIMESTAMP_MICROS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 6); - // Parquet stores signed and unsigned integer logical annotations on signed physical carriers: - // INT_8/UINT_8/INT_16/UINT_16/INT_32/UINT_32 use physical INT32, and - // INT_64/UINT_64 use physical INT64. Doris maps unsigned integers to the next wider - // signed type so all values in the unsigned range can be represented. - case ::parquet::ConvertedType::INT_8: - mark_integer(8, true, result); - return create_type(TYPE_TINYINT, nullable); - case ::parquet::ConvertedType::UINT_8: - mark_integer(8, false, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::INT_16: - mark_integer(16, true, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::UINT_16: - mark_integer(16, false, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::INT_32: - mark_integer(32, true, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::UINT_32: - mark_integer(32, false, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::INT_64: - mark_integer(64, true, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::UINT_64: - mark_integer(64, false, result); - return create_type(TYPE_LARGEINT, nullable); - case ::parquet::ConvertedType::NONE: - default: - return nullptr; - } -} - -DataTypePtr logical_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const auto& logical_type = column->logical_type(); - if (logical_type == nullptr || !logical_type->is_valid() || logical_type->is_none()) { - return nullptr; - } - const bool nullable = column->max_definition_level() > 0; - if (logical_type->is_string() || logical_type->is_enum() || logical_type->is_JSON() || - logical_type->is_BSON() || logical_type->is_UUID()) { - return create_type(TYPE_STRING, nullable); - } - if (logical_type->is_decimal()) { - const auto& decimal_type = static_cast(*logical_type); - mark_decimal(column, decimal_type.precision(), decimal_type.scale(), result); - return create_type(decimal_primitive_type(decimal_type.precision()), nullable, - decimal_type.precision(), decimal_type.scale()); - } - if (logical_type->is_date()) { - return create_type(TYPE_DATEV2, nullable); - } - if (logical_type->is_time()) { - const auto& time_type = static_cast(*logical_type); - if (time_type.is_adjusted_to_utc()) { - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - } - int scale = 0; - if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else { - return nullptr; - } - return create_type(TYPE_TIMEV2, nullable, 0, scale); - } - if (logical_type->is_timestamp()) { - const auto& timestamp_type = - static_cast(*logical_type); - int scale = 0; - if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::NANOS) { - scale = 6; - result->time_unit = ParquetTimeUnit::NANOS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_NS; - } else { - return nullptr; - } - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = timestamp_type.is_adjusted_to_utc(); - return create_type(TYPE_DATETIMEV2, nullable, 0, scale); - } - if (logical_type->is_int()) { - const auto& int_type = static_cast(*logical_type); - mark_integer(int_type.bit_width(), int_type.is_signed(), result); - switch (int_type.bit_width()) { - case 8: - return create_type(int_type.is_signed() ? TYPE_TINYINT : TYPE_SMALLINT, nullable); - case 16: - return create_type(int_type.is_signed() ? TYPE_SMALLINT : TYPE_INT, nullable); - case 32: - return create_type(int_type.is_signed() ? TYPE_INT : TYPE_BIGINT, nullable); - case 64: - return create_type(int_type.is_signed() ? TYPE_BIGINT : TYPE_LARGEINT, nullable); - default: - return nullptr; - } - } - if (logical_type->is_float16()) { - if (column->physical_type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY || - column->type_length() != 2) { - return nullptr; - } - result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; - return create_type(TYPE_FLOAT, nullable); - } - return nullptr; -} - -DataTypePtr physical_type_to_doris_type(const ::parquet::ColumnDescriptor* column) { - const bool nullable = column->max_definition_level() > 0; - DataTypePtr type; - switch (column->physical_type()) { - case ::parquet::Type::BOOLEAN: - type = std::make_shared(); - break; - case ::parquet::Type::INT32: - type = std::make_shared(); - break; - case ::parquet::Type::INT64: - type = std::make_shared(); - break; - case ::parquet::Type::FLOAT: - type = std::make_shared(); - break; - case ::parquet::Type::DOUBLE: - type = std::make_shared(); - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - type = std::make_shared(); - break; - case ::parquet::Type::INT96: - type = create_type(TYPE_DATETIMEV2, nullable, 0, 6); - break; - default: - return nullptr; - } - return nullable ? make_nullable(type) : type; -} - -bool record_reader_physical_type_supported(::parquet::Type::type physical_type) { - switch (physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::INT96: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return true; - default: - return false; - } -} - -} // namespace - -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column) { - if (column == nullptr) { - return {}; - } - auto path = column->path(); - if (path) { - return path->ToDotString(); - } - return column->name(); -} - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column) { - ParquetTypeDescriptor result; - if (column == nullptr) { - return result; - } - - result.physical_type = column->physical_type(); - result.converted_type = column->converted_type(); - result.fixed_length = column->type_length(); - result.physical_doris_type = physical_type_to_doris_type(column); - - if (auto logical_type = logical_type_to_doris_type(column, &result); logical_type != nullptr) { - result.doris_type = logical_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - result.supports_record_reader = false; - } else if (auto converted_type = converted_type_to_doris_type(column, &result); - converted_type != nullptr) { - result.doris_type = converted_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - result.supports_record_reader = false; - } else { - result.doris_type = result.physical_doris_type; - if (result.physical_type == ::parquet::Type::INT96) { - result.extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; - } - } - - result.is_string_like = !result.is_decimal && - result.extra_type_info != ParquetExtraTypeInfo::FLOAT16 && - (result.physical_type == ::parquet::Type::BYTE_ARRAY || - result.physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - - if (!record_reader_physical_type_supported(result.physical_type)) { - result.supports_record_reader = false; - } - return result; -} - -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor) { - return type_descriptor.supports_record_reader; -} DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor) { switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: + case tparquet::Type::BOOLEAN: return DecodedValueKind::BOOL; - case ::parquet::Type::INT32: + case tparquet::Type::INT32: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 32) { return DecodedValueKind::UINT32; } return DecodedValueKind::INT32; - case ::parquet::Type::INT64: + case tparquet::Type::INT64: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 64) { return DecodedValueKind::UINT64; } return DecodedValueKind::INT64; - case ::parquet::Type::INT96: + case tparquet::Type::INT96: return DecodedValueKind::INT96; - case ::parquet::Type::FLOAT: + case tparquet::Type::FLOAT: return DecodedValueKind::FLOAT; - case ::parquet::Type::DOUBLE: + case tparquet::Type::DOUBLE: return DecodedValueKind::DOUBLE; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return DecodedValueKind::FIXED_BINARY; - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BYTE_ARRAY: default: return DecodedValueKind::BINARY; } diff --git a/be/src/format_v2/parquet/parquet_type.h b/be/src/format_v2/parquet/parquet_type.h index a4d99abc0e982a..7ab3d2b3b39d8d 100644 --- a/be/src/format_v2/parquet/parquet_type.h +++ b/be/src/format_v2/parquet/parquet_type.h @@ -15,17 +15,13 @@ #pragma once -#include +#include #include #include "core/data_type/data_type.h" #include "core/data_type_serde/decoded_column_view.h" -namespace parquet { -class ColumnDescriptor; -} // namespace parquet - namespace doris::format::parquet { // ============================================================================ @@ -53,14 +49,15 @@ enum class ParquetTimeUnit { // ============================================================================ // ============================================================================ struct ParquetTypeDescriptor { + // Keep the V2 semantic tree on generated Thrift enums; using parquet-cpp descriptors here + // would reintroduce a second metadata model and make native planning bypassable. DataTypePtr doris_type; // Physical fallback used only to keep file schema construction alive when the logical type is // unsupported. Column reader creation still rejects unsupported_reason before decoding. DataTypePtr physical_doris_type; ParquetExtraTypeInfo extra_type_info = ParquetExtraTypeInfo::NONE; ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; - ::parquet::Type::type physical_type = ::parquet::Type::UNDEFINED; - ::parquet::ConvertedType::type converted_type = ::parquet::ConvertedType::UNDEFINED; + tparquet::Type::type physical_type = tparquet::Type::INT32; int integer_bit_width = -1; // bit width for INT_8/16/32/64 int decimal_precision = -1; // precision for DECIMAL(p,s) int decimal_scale = -1; // scale for DECIMAL(p,s) @@ -70,16 +67,9 @@ struct ParquetTypeDescriptor { bool is_timestamp = false; // whether this is a timestamp type bool timestamp_is_adjusted_to_utc = false; // whether the timestamp is UTC-normalized bool is_string_like = false; // binary type that is neither decimal nor FLOAT16 - bool supports_record_reader = true; // whether Arrow RecordReader can read this type std::string unsupported_reason; // non-empty when this Parquet logical type is unsupported }; -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column); - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column); - -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor); - DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor); } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.cpp b/be/src/format_v2/parquet/reader/column_reader.cpp index 352fbbd7c3d215..f23559e5dbf9f0 100644 --- a/be/src/format_v2/parquet/reader/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/column_reader.cpp @@ -15,145 +15,25 @@ #include "format_v2/parquet/reader/column_reader.h" -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include #include -#include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "runtime/runtime_profile.h" namespace doris::format::parquet { -namespace { - -class DataPageSkipFilter { -public: - DataPageSkipFilter(const ParquetPageSkipPlan* page_skip_plan, - ParquetPageSkipProfile page_skip_profile) - : _page_skip_plan(page_skip_plan), _page_skip_profile(page_skip_profile) { - DORIS_CHECK(_page_skip_plan != nullptr); - } - - bool operator()(const ::parquet::DataPageStats&) { - // Arrow invokes this callback once for each DATA_PAGE/DATA_PAGE_V2 and never for - // dictionary pages, so this ordinal matches Parquet OffsetIndex page locations. - const size_t page_idx = _next_data_page_idx++; - const bool skip = _page_skip_plan->should_skip_page(page_idx); - if (!skip) { - return false; - } - update_skip_profile(page_idx); - return true; - } - -private: - void update_skip_profile(size_t page_idx) const { - if (_page_skip_profile.skipped_pages != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); - } - if (_page_skip_profile.skipped_bytes != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_bytes, - _page_skip_plan->skipped_page_compressed_size(page_idx)); - } - } - - const ParquetPageSkipPlan* _page_skip_plan = nullptr; - ParquetPageSkipProfile _page_skip_profile; - size_t _next_data_page_idx = 0; -}; - -const ParquetPageSkipPlan* find_page_skip_plan( - const std::map* page_skip_plans, int leaf_column_id) { - if (page_skip_plans == nullptr) { - return nullptr; - } - const auto plan_it = page_skip_plans->find(leaf_column_id); - return plan_it == page_skip_plans->end() ? nullptr : &plan_it->second; -} - -void install_data_page_filter(std::unique_ptr<::parquet::PageReader>& page_reader, - const std::map* page_skip_plans, - int leaf_column_id, ParquetPageSkipProfile page_skip_profile) { - DORIS_CHECK(page_reader != nullptr); - const ParquetPageSkipPlan* page_skip_plan = - find_page_skip_plan(page_skip_plans, leaf_column_id); - if (page_skip_plan == nullptr) { - return; - } - page_reader->set_data_page_filter(DataPageSkipFilter(page_skip_plan, page_skip_profile)); -} - -bool supports_nested_scalar_record_reader(const ParquetColumnSchema& column_schema) { - if (column_schema.type_descriptor.supports_record_reader) { - return true; - } - const auto& type_descriptor = column_schema.type_descriptor; - if ((type_descriptor.extra_type_info != ParquetExtraTypeInfo::NONE && - type_descriptor.extra_type_info != ParquetExtraTypeInfo::FLOAT16) || - type_descriptor.is_decimal || type_descriptor.is_timestamp || - type_descriptor.is_string_like) { - return false; - } - if (type_descriptor.converted_type != ::parquet::ConvertedType::NONE && - type_descriptor.converted_type != ::parquet::ConvertedType::UNDEFINED) { - return false; - } - switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - return true; - default: - return false; - } - return true; -} -} // namespace +ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, + ParquetColumnReaderProfile profile) + : _profile(profile), + _field_id(schema.local_id), + _leaf_column_id(schema.leaf_column_id), + _type(std::move(type)), + _name(schema.name) {} Status ParquetColumnReader::skip(int64_t rows) { return Status::NotSupported("Parquet column skip is not implemented, rows={}", rows); } -void ParquetColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - int64_t child_cursor = nested_build_level_cursor(); - const auto& child_rep_levels = nested_repetition_levels(); - const int64_t child_levels_written = nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!is_or_has_repeated_child() || child_rep_level <= parent_repetition_level) { - break; - } - } - set_nested_build_level_cursor(child_cursor); -} - void ParquetColumnReader::update_reader_read_rows(int64_t rows) const { if (_profile.reader_read_rows != nullptr) { COUNTER_UPDATE(_profile.reader_read_rows, rows); @@ -166,21 +46,16 @@ void ParquetColumnReader::update_reader_skip_rows(int64_t rows) const { } } -Status ParquetColumnReader::select(const SelectionVector& sel, uint16_t selected_rows, +Status ParquetColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, MutableColumnPtr& column) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Parquet selected read result is null for column {}", - name()); - } - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); + DORIS_CHECK(column); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); - const auto ranges = selection_to_ranges(sel, selected_rows); + const auto ranges = selection_to_ranges(selection, selected_rows); int64_t cursor = 0; for (const auto& range : ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument("Invalid parquet selection range [{}, {}) for column {}", - range.start, range.start + range.length, name()); - } + DORIS_CHECK(range.start >= cursor); + DORIS_CHECK(range.start + range.length <= batch_rows); RETURN_IF_ERROR(skip(range.start - cursor)); int64_t range_rows_read = 0; @@ -206,453 +81,15 @@ Status ParquetColumnReader::select_with_dictionary_filter(const SelectionVector& name()); } -ParquetColumnReaderFactory::ParquetColumnReaderFactory( - std::shared_ptr<::parquet::RowGroupReader> row_group, int num_leaf_columns, - const std::map* page_skip_plans, - ParquetPageSkipProfile page_skip_profile, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile column_reader_profile) - : _row_group(std::move(row_group)), - _record_readers(static_cast(num_leaf_columns)), - _dictionary_record_readers(static_cast(num_leaf_columns)), - _page_skip_plans(page_skip_plans), - _page_skip_profile(page_skip_profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _column_reader_profile(column_reader_profile) {} - -std::unique_ptr ParquetColumnReaderFactory::create_row_position_column_reader( - int64_t row_group_first_row) const { - return std::make_unique(row_group_first_row, _column_reader_profile); -} - -std::unique_ptr ParquetColumnReaderFactory::create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const { - return std::make_unique(context, row_group_first_row, - _column_reader_profile); -} - -Status ParquetColumnReaderFactory::make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, bool use_page_skip_plan, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - const auto* page_skip_plan = - use_page_skip_plan ? find_page_skip_plan(_page_skip_plans, column_schema.leaf_column_id) - : nullptr; - *reader = std::make_unique(column_schema, std::move(record_reader), - page_skip_plan, _timezone, _enable_strict_mode, - _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_scalar_column_reader( - const ParquetColumnSchema& column_schema, bool is_nested, bool read_dictionary, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (!column_schema.type_descriptor.unsupported_reason.empty()) { - return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, - column_schema.type_descriptor.unsupported_reason); - } - if (is_nested && column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { - return Status::InvalidArgument("Parquet nested scalar reader requires primitive column {}", - column_schema.name); - } - if (column_schema.leaf_column_id < 0 || - column_schema.leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - column_schema.leaf_column_id, column_schema.name); - } - if (column_schema.descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", - column_schema.name); - } - if (!is_nested && (column_schema.descriptor->max_repetition_level() != 0 || - column_schema.descriptor->max_definition_level() > 1)) { - return Status::NotSupported( - "Current parquet scalar reader only supports flat primitive columns; column {} is " - "not supported", - column_schema.name); - } - if (is_nested && !supports_nested_scalar_record_reader(column_schema)) { - return Status::NotSupported( - "Current parquet nested scalar reader does not support column {}", - column_schema.name); - } - if (!is_nested && !column_schema.type_descriptor.supports_record_reader) { - return Status::NotSupported("Current parquet scalar reader does not support column {}", - column_schema.name); - } - std::shared_ptr<::parquet::internal::RecordReader> record_reader; - // Nested readers implement skip() by materializing rows into a scratch column. If Arrow - // page filtering is also installed, those scratch reads can consume the next selected row - // after a page-index range gap. Keep page filtering on flat scalar readers only. - RETURN_IF_ERROR(get_record_reader(column_schema.leaf_column_id, column_schema.descriptor, - column_schema.name, !is_nested, read_dictionary, - &record_reader)); - return make_scalar_column_reader(column_schema, std::move(record_reader), !is_nested, reader); -} - -// 1. RowGroupReader::GetColumnPageReader(leaf_column_id) -> Arrow PageReader -Status ParquetColumnReaderFactory::get_record_reader( - int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, const std::string& name, - bool install_page_filter, bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (_row_group == nullptr) { - return Status::InternalError("Parquet row group reader is not initialized for column {}", - name); - } - if (leaf_column_id < 0 || leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - leaf_column_id, name); - } - if (descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", name); - } - auto& record_readers = read_dictionary ? _dictionary_record_readers : _record_readers; - if (record_readers[leaf_column_id] == nullptr) { - try { - auto page_reader = _row_group->GetColumnPageReader(leaf_column_id); - if (install_page_filter) { - install_data_page_filter(page_reader, _page_skip_plans, leaf_column_id, - _page_skip_profile); - } - const auto level_info = ::parquet::internal::LevelInfo::ComputeLevelInfo(descriptor); - record_readers[leaf_column_id] = ::parquet::internal::RecordReader::Make( - descriptor, level_info, ::arrow::default_memory_pool(), - /*read_dictionary=*/read_dictionary, - /*read_dense_for_nullable=*/false); - record_readers[leaf_column_id]->SetPageReader(std::move(page_reader)); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } - } - if (record_readers[leaf_column_id] == nullptr) { - return Status::Corruption("Failed to create parquet record reader for column {}", name); - } - *reader = record_readers[leaf_column_id]; - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_struct_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - std::vector> child_readers; - child_readers.reserve(column_schema.children.size()); - std::vector child_output_indices; - child_output_indices.reserve(column_schema.children.size()); - DataTypes projected_child_types; - Strings projected_child_names; - for (size_t child_idx = 0; child_idx < column_schema.children.size(); ++child_idx) { - const auto& child_schema = column_schema.children[child_idx]; - const auto* child_projection = - format::find_child_projection(projection, child_schema->local_id); - if (!format::is_child_projected(projection, child_schema->local_id)) { - continue; - } - std::unique_ptr child_reader; - RETURN_IF_ERROR( - create_column_reader(*child_schema, child_projection, true, false, &child_reader)); - child_output_indices.push_back(static_cast(projected_child_types.size())); - projected_child_types.push_back(make_nullable(child_reader->type())); - projected_child_names.push_back(child_reader->name()); - child_readers.push_back(std::move(child_reader)); - } - if (format::is_partial_projection(projection) && - projected_child_types.size() != projection->children.size()) { - return Status::InvalidArgument( - "Parquet STRUCT projection for column {} contains invalid child", - column_schema.name); - } - if (projected_child_types.empty() && !column_schema.children.empty()) { - return Status::NotSupported("Parquet STRUCT projection for column {} contains no children", - column_schema.name); - } - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(projection)) { - type = std::make_shared(projected_child_types, projected_child_names); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique( - column_schema, std::move(type), std::move(child_readers), - std::move(child_output_indices), _column_reader_profile); +Status ParquetColumnReader::select_with_plain_filter(const SelectionVector&, uint16_t, int64_t, + const VExprSPtrs&, int, + IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *used_filter = false; return Status::OK(); } -Status ParquetColumnReaderFactory::create_list_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for column {}", - column_schema.name); - } - std::unique_ptr element_reader; - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - if (format::is_partial_projection(projection) && element_projection == nullptr) { - return Status::NotSupported("Parquet LIST projection for column {} contains no element", - column_schema.name); - } - RETURN_IF_ERROR( - create_column_reader(element_schema, element_projection, true, false, &element_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(element_projection)) { - type = std::make_shared(element_reader->type()); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique(column_schema, std::move(type), - std::move(element_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_map_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP layout for column {}", - column_schema.name); - } - const auto& key_schema = *column_schema.children[0]; - const auto& value_schema = *column_schema.children[1]; - const auto* value_projection = format::find_child_projection(projection, value_schema.local_id); - if (format::is_partial_projection(projection)) { - if (value_projection == nullptr) { - return Status::NotSupported("Parquet MAP projection for column {} contains no value", - column_schema.name); - } - for (const auto& child_projection : projection->children) { - if (child_projection.local_id() == key_schema.local_id) { - continue; - } - if (child_projection.local_id() != value_schema.local_id) { - return Status::InvalidArgument( - "Parquet MAP projection for column {} contains invalid child", - column_schema.name); - } - } - } - std::unique_ptr key_reader; - // MAP materialization always needs the full key stream. It owns entry existence, offsets and - // key equality semantics, so MAP projection is defined only as value-subtree pruning. - RETURN_IF_ERROR(create_column_reader(key_schema, nullptr, true, false, &key_reader)); - std::unique_ptr value_reader; - RETURN_IF_ERROR( - create_column_reader(value_schema, value_projection, true, false, &value_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(value_projection)) { - type = std::make_shared(make_nullable(key_reader->type()), - make_nullable(value_reader->type())); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = - std::make_unique(column_schema, std::move(type), std::move(key_reader), - std::move(value_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, - bool read_dictionary) const { - return create_column_reader(column_schema, projection, false, read_dictionary, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - return create_count_shape_reader_impl(column_schema, projection, false, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader_impl( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, is_nested, false, reader); - case ParquetColumnSchemaKind::STRUCT: { - if (column_schema.children.empty()) { - return Status::NotSupported("Parquet COUNT shape reader found empty STRUCT column {}", - column_schema.name); - } - const ParquetColumnSchema* child_schema = nullptr; - const format::LocalColumnIndex* child_projection = nullptr; - if (format::is_partial_projection(projection)) { - const auto child_id = projection->children[0].local_id(); - const auto child_it = std::ranges::find_if( - column_schema.children, - [&](const auto& child) { return child->local_id == child_id; }); - if (child_it == column_schema.children.end()) { - return Status::InvalidArgument( - "Parquet COUNT projection for column {} contains invalid child", - column_schema.name); - } - child_schema = child_it->get(); - child_projection = &projection->children[0]; - } else { - child_schema = column_schema.children[0].get(); - } - DORIS_CHECK(child_schema != nullptr); - return create_count_shape_reader_impl(*child_schema, child_projection, true, reader); - } - case ParquetColumnSchemaKind::LIST: { - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for COUNT column {}", - column_schema.name); - } - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - return create_count_shape_reader_impl(element_schema, element_projection, true, reader); - } - case ParquetColumnSchemaKind::MAP: { - if (column_schema.children.empty()) { - return Status::NotSupported("Unsupported parquet MAP layout for COUNT column {}", - column_schema.name); - } - // The key stream defines MAP entry existence and offsets. Counting top-level MAP NULL-ness - // from it avoids creating a value reader, which is the expensive path for files with huge - // MAP value strings. - return create_count_shape_reader_impl(*column_schema.children[0], nullptr, true, reader); - } - } - return Status::NotSupported("Unsupported parquet column schema kind for COUNT column {}", - column_schema.name); -} - -Status ParquetColumnReaderFactory::create_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, bool read_dictionary, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (is_nested) { - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet scalar projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, true, false, reader); - } - return create_scalar_column_reader(column_schema, false, read_dictionary, reader); - case ParquetColumnSchemaKind::STRUCT: - return create_struct_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::LIST: - return create_list_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::MAP: - return create_map_column_reader(column_schema, projection, reader); - } - return Status::NotSupported("Unsupported parquet column schema kind for column {}", - column_schema.name); -} - -ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, - ParquetColumnReaderProfile profile) - : _profile(profile), - _field_id(schema.local_id), - _leaf_column_id(schema.leaf_column_id), - _nullable_definition_level(schema.nullable_definition_level), - _repeated_repetition_level(schema.repeated_repetition_level), - _definition_level(schema.definition_level), - _repetition_level(schema.repetition_level), - _repeated_ancestor_definition_level(schema.repeated_ancestor_definition_level), - _type(std::move(type)), - _name(schema.name) {} - -Status ParquetColumnReader::load_nested_batch(int64_t) { - return Status::NotSupported("Parquet nested batch load is not supported for column {}", _name); -} - -Status ParquetColumnReader::load_nested_levels_batch(int64_t) { - return Status::NotSupported("Parquet nested levels batch load is not supported for column {}", - _name); -} - -Status ParquetColumnReader::build_nested_column(int64_t, MutableColumnPtr&, int64_t*) { - return Status::NotSupported("Parquet nested column build is not supported for column {}", - _name); -} - -Status ParquetColumnReader::consume_nested_column(int64_t, int64_t*) { - return Status::NotSupported("Parquet nested column consume is not supported for column {}", - _name); -} - -Status ParquetColumnReader::skip_nested_rows(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - // A nested parent row may expand to many child values. Capping the number of parent rows per - // loaded batch bounds that amplification for large holes. The consume interface advances the - // loaded definition/repetition levels recursively without constructing a discarded Column. - constexpr int64_t MAX_NESTED_SKIP_BATCH_SIZE = 4096; - int64_t remaining_rows = rows; - while (remaining_rows > 0) { - const int64_t batch_rows = std::min(remaining_rows, MAX_NESTED_SKIP_BATCH_SIZE); - RETURN_IF_ERROR(load_nested_levels_batch(batch_rows)); - int64_t rows_consumed = 0; - RETURN_IF_ERROR(consume_nested_column(batch_rows, &rows_consumed)); - if (rows_consumed != batch_rows) { - return Status::Corruption( - "Failed to skip nested parquet column {}: skipped {} of {} rows in batch", - _name, rows_consumed, batch_rows); - } - remaining_rows -= batch_rows; - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -const std::vector& ParquetColumnReader::nested_definition_levels() const { - static const std::vector empty; - return empty; -} - -const std::vector& ParquetColumnReader::nested_repetition_levels() const { - static const std::vector empty; - return empty; -} - -int64_t ParquetColumnReader::nested_levels_written() const { - return 0; -} - -bool ParquetColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index 51dbd44c11c226..bd4b73e3aa5172 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -16,216 +16,79 @@ #pragma once #include -#include -#include #include -#include #include "common/status.h" -#include "core/column/column_nullable.h" +#include "core/column/column.h" #include "core/data_type/data_type.h" -#include "format_v2/column_data.h" +#include "exprs/vexpr_fwd.h" #include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" #include "format_v2/parquet/selection_vector.h" -#include "runtime/runtime_profile.h" - -namespace parquet { -class ColumnDescriptor; -class RowGroupReader; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris { -class IColumn; -} // namespace doris namespace doris::format::parquet { struct ParquetColumnSchema; +// Scan-time column contract for FileScannerV2. +// +// Physical Parquet columns are implemented by NativeColumnReader, which owns Doris' native page +// decoder and writes directly into the destination Doris column. Synthetic row-position/global-id +// readers implement the same cursor contract. Arrow RecordReader, ParquetLeafBatch, decoded value +// views, and the former nested build/consume protocol intentionally do not belong to this API. class ParquetColumnReader { public: virtual ~ParquetColumnReader() = default; virtual int file_column_id() const { return _field_id; } - virtual int parquet_leaf_column_id() const { return _leaf_column_id; } - - int16_t nullable_definition_level() const { return _nullable_definition_level; } - int16_t repeated_repetition_level() const { return _repeated_repetition_level; } - virtual const DataTypePtr& type() const { return _type; } virtual const std::string& name() const { return _name; } const ParquetColumnReaderProfile& profile() const { return _profile; } + // Consume rows from the row-group cursor and append them to column. virtual Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) = 0; + // Consume rows without materializing values. virtual Status skip(int64_t rows); - virtual Status select(const SelectionVector& sel, uint16_t selected_rows, int64_t batch_rows, - MutableColumnPtr& column); + // Consume batch_rows and append only selection[0, selected_rows). The default implementation + // coalesces adjacent indices into ranges; native readers override it with one decoder call. + virtual Status select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column); - virtual Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, + virtual Status select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter); - virtual Status load_nested_batch(int64_t rows); - - // Shape-only load interface for COUNT(col) and skip. Implementations guarantee only that - // nested_definition_levels(), nested_repetition_levels(), and nested_levels_written() are - // available; value indices and payload columns may be absent. Callers may inspect the levels or - // call consume_nested_column(), but must not call build_nested_column() afterwards. For example, - // skipping ARRAY uses this method to find ARRAY boundaries without constructing a - // ColumnString. The underlying Arrow reader may still decode a page because it has no public - // levels-only API. Normal scans that need output values use load_nested_batch() instead. - virtual Status load_nested_levels_batch(int64_t rows); - - virtual Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read); - - // Consume logical values from a batch previously loaded by load_nested_batch() or - // load_nested_levels_batch() without appending them to an output Column. Implementations must - // advance exactly the same nested level cursors and perform the same shape/null/alignment - // validation as build_nested_column(). The levels-only form is preferred for skip paths because - // it avoids transferring leaf payloads into Doris Columns when they will be discarded. - // - // `length_upper_bound` is expressed at this reader's logical level, not in physical leaf - // values. For example, consuming two rows from ARRAY [[1, 2], []] consumes two parent ARRAY - // rows but only two element values. A MAP implementation must also consume key/value streams - // in lockstep, while a nullable STRUCT consumes no child value for a null parent. - // - // Callers must not use the ordinary skip() after either load call: the leaf stream has already - // advanced into an in-memory nested batch, and doing so would advance it twice. - // `values_consumed` may be smaller than the requested bound only when the loaded batch ends. - virtual Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed); - - virtual const std::vector& nested_definition_levels() const; - virtual const std::vector& nested_repetition_levels() const; - virtual int64_t nested_levels_written() const; - virtual bool is_or_has_repeated_child() const; - virtual void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level); - - int64_t nested_build_level_cursor() const { return _nested_build_level_cursor; } - void set_nested_build_level_cursor(int64_t cursor) { - DORIS_CHECK(cursor >= 0); - _nested_build_level_cursor = cursor; + // Consume batch_rows and evaluate eligible fixed-width PLAIN values without constructing a + // predicate column. Implementations must leave the cursor untouched when used_filter=false. + virtual Status select_with_plain_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const VExprSPtrs& conjuncts, int column_id, + IColumn::Filter* row_filter, bool* used_filter); + + // Native statistics are cumulative and can be recursively aggregated for complex columns. + // Flush once at the scheduler batch boundary instead of snapshotting after each operation. + virtual void flush_profile() {} + virtual bool crossed_page_since_last_batch() { return false; } + virtual Result dictionary_values() { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); } - void reset_nested_build_level_cursor() { _nested_build_level_cursor = 0; } protected: - ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, + ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, ParquetColumnReaderProfile profile = {}); ParquetColumnReader() = default; - // Load shape levels and consume skipped parent rows in bounded batches. The bound limits level - // memory when a parent expands to many children; the levels-only load plus - // consume_nested_column() avoids payload materialization and output Columns. - Status skip_nested_rows(int64_t rows); + void update_reader_read_rows(int64_t rows) const; void update_reader_skip_rows(int64_t rows) const; ParquetColumnReaderProfile _profile; - const int _field_id = -1; // child ordinal in the parent node - const int _leaf_column_id = -1; // Parquet physical leaf column id (-1 = non-leaf) - const int16_t _nullable_definition_level = - 0; // definition-level threshold where this node becomes nullable - const int16_t _repeated_repetition_level = - 0; // repetition level of the nearest repeated ancestor - const int16_t _definition_level = 0; // definition level accumulated to this node - const int16_t _repetition_level = 0; // repetition level accumulated to this node - const int16_t _repeated_ancestor_definition_level = - 0; // definition level of the nearest repeated ancestor - const DataTypePtr _type; // Doris target type - const std::string _name; // column name for error messages - int64_t _nested_build_level_cursor = 0; // nested build cursor (current level position) + const int _field_id = -1; + const int _leaf_column_id = -1; + const DataTypePtr _type; + const std::string _name; }; -class ParquetColumnReaderFactory { -public: - ParquetColumnReaderFactory(std::shared_ptr<::parquet::RowGroupReader> row_group, - int num_leaf_columns, - const std::map* page_skip_plans = nullptr, - ParquetPageSkipProfile page_skip_profile = {}, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false, - ParquetColumnReaderProfile column_reader_profile = {}); - - Status create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, bool read_dictionary = false) const; - - // Create a scalar reader for one representative leaf that carries the top-level column shape. - // This is used by COUNT(col): the caller needs definition/repetition levels to decide whether - // the top-level value is NULL, but must not materialize heavy payload leaves. MAP deliberately - // uses the key leaf because the key stream owns entry existence and avoids reading value pages. - Status create_count_shape_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create(const ParquetColumnSchema& column_schema, - std::unique_ptr* reader) const { - return create(column_schema, nullptr, reader); - } - - std::unique_ptr create_row_position_column_reader( - int64_t row_group_first_row) const; - std::unique_ptr create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const; - -private: - Status create_scalar_column_reader(const ParquetColumnSchema& column_schema, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - - Status create_struct_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_list_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_map_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - Status create_count_shape_reader_impl(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - bool is_nested, - std::unique_ptr* reader) const; - - Status get_record_reader(int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, - const std::string& name, bool install_page_filter, - bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const; - - Status make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - bool use_page_skip_plan, std::unique_ptr* reader) const; - - std::shared_ptr<::parquet::RowGroupReader> _row_group; // Arrow RowGroup reader - mutable std::vector> - _record_readers; // RecordReader cache by leaf_column_id - mutable std::vector> - _dictionary_record_readers; // dictionary-exposing RecordReader cache by leaf_column_id - const std::map* _page_skip_plans = - nullptr; // page-index pruning result - ParquetPageSkipProfile _page_skip_profile; // page skip profile - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - ParquetColumnReaderProfile _column_reader_profile; // column reader profile -}; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp new file mode 100644 index 00000000000000..171107107f1a38 --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/count_column_reader.h" + +#include +#include +#include + +#include "common/config.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet { +namespace { + +Status find_count_leaf(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const ParquetColumnSchema** leaf) { + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + if (format::is_partial_projection(projection)) { + return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", + schema.name); + } + *leaf = &schema; + return Status::OK(); + case ParquetColumnSchemaKind::STRUCT: { + DORIS_CHECK(!schema.children.empty()); + if (!format::is_partial_projection(projection)) { + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + DORIS_CHECK(!projection->children.empty()); + const auto child_id = projection->children.front().local_id(); + const auto child = std::ranges::find_if(schema.children, [child_id](const auto& candidate) { + return candidate->local_id == child_id; + }); + if (child == schema.children.end()) { + return Status::InvalidArgument( + "Parquet COUNT projection for column {} contains invalid child", schema.name); + } + return find_count_leaf(**child, &projection->children.front(), leaf); + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto& element = *schema.children.front(); + return find_count_leaf(element, format::find_child_projection(projection, element.local_id), + leaf); + } + case ParquetColumnSchemaKind::MAP: + // The key leaf owns entry existence and top-level MAP shape. Choosing it also avoids + // reading a potentially huge value BYTE_ARRAY for COUNT(map_col). + DORIS_CHECK(!schema.children.empty()); + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + return Status::InternalError("Unknown Parquet schema kind for column {}", schema.name); +} + +NativeFieldSchema* find_physical_leaf(NativeFieldSchema* field, int physical_column_index) { + DORIS_CHECK(field != nullptr); + if (field->children.empty()) { + return field->physical_column_index == physical_column_index ? field : nullptr; + } + for (auto& child : field->children) { + if (auto* result = find_physical_leaf(&child, physical_column_index); result != nullptr) { + return result; + } + } + return nullptr; +} + +} // namespace + +CountColumnReader::CountColumnReader(std::string name, + std::unique_ptr level_reader, + ParquetColumnReaderProfile profile) + : _level_reader(std::move(level_reader)), _profile(profile), _name(std::move(name)) {} + +CountColumnReader::~CountColumnReader() { + sync_profile(); +} + +Status CountColumnReader::create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(reader != nullptr); + const ParquetColumnSchema* leaf_schema = nullptr; + RETURN_IF_ERROR(find_count_leaf(root_schema, projection, &leaf_schema)); + DORIS_CHECK(leaf_schema != nullptr); + DORIS_CHECK(leaf_schema->leaf_column_id >= 0); + + const auto& thrift_metadata = metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::InvalidArgument("Invalid Parquet COUNT row group {}", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + if (leaf_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet COUNT leaf {} for row group {}", + leaf_schema->leaf_column_id, row_group_id); + } + + DORIS_CHECK(root_schema.local_id >= 0); + auto* root_field = + const_cast(metadata->schema().get_column(root_schema.local_id)); + DORIS_CHECK(root_field != nullptr); + auto* leaf_field = find_physical_leaf(root_field, leaf_schema->leaf_column_id); + if (leaf_field == nullptr) { + return Status::Corruption("Cannot resolve Parquet COUNT physical leaf {} for column {}", + leaf_schema->leaf_column_id, root_schema.name); + } + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + std::unique_ptr level_reader; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + RETURN_IF_ERROR(native::LevelReader::create( + std::move(file), row_group.columns[leaf_schema->leaf_column_id], leaf_field, + row_group.num_rows, std::min(max_group_buffer, max_column_buffer), io_ctx, + enable_page_cache, page_cache_file_key, compat, &level_reader)); + reader->reset(new CountColumnReader(leaf_schema->name, std::move(level_reader), profile)); + return Status::OK(); +} + +Status CountColumnReader::skip(int64_t rows) { + DORIS_CHECK(rows >= 0); + if (rows == 0) { + return Status::OK(); + } + { + SCOPED_TIMER(_profile.level_only_skip_time); + RETURN_IF_ERROR(_level_reader->skip_rows(static_cast(rows))); + } + if (_profile.reader_skip_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_skip_rows, rows); + } + sync_profile(); + return Status::OK(); +} + +Status CountColumnReader::read_levels(int64_t rows, int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(rows_read != nullptr); + _definition_levels.clear(); + _repetition_levels.clear(); + _levels_written = 0; + size_t native_rows_read = 0; + { + SCOPED_TIMER(_profile.level_only_read_time); + RETURN_IF_ERROR(_level_reader->read_rows(static_cast(rows), &_repetition_levels, + &_definition_levels, &native_rows_read)); + } + *rows_read = static_cast(native_rows_read); + if (*rows_read != rows || _definition_levels.size() != _repetition_levels.size()) { + return Status::Corruption( + "Parquet COUNT level reader returned {} rows and {}/{} levels for {}", *rows_read, + _definition_levels.size(), _repetition_levels.size(), _name); + } + _levels_written = static_cast(_definition_levels.size()); + if (_levels_written < *rows_read) { + return Status::Corruption("Parquet COUNT returned {} levels for {} rows in column {}", + _levels_written, *rows_read, _name); + } + if (_profile.reader_read_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_read_rows, *rows_read); + } + sync_profile(); + return Status::OK(); +} + +void CountColumnReader::sync_profile() { + if (_level_reader == nullptr) { + return; + } + const auto stats = _level_reader->statistics(); + const auto& reported = _reported_native_stats; +#define UPDATE_NATIVE_PROFILE(counter, field) \ + if (_profile.counter != nullptr) { \ + COUNTER_UPDATE(_profile.counter, stats.field - reported.field); \ + } + UPDATE_NATIVE_PROFILE(decompress_time, decompress_time); + UPDATE_NATIVE_PROFILE(decompress_count, decompress_cnt); + UPDATE_NATIVE_PROFILE(decode_header_time, decode_header_time); + UPDATE_NATIVE_PROFILE(decode_value_time, decode_value_time); + UPDATE_NATIVE_PROFILE(decode_dictionary_time, decode_dict_time); + UPDATE_NATIVE_PROFILE(decode_level_time, decode_level_time); + UPDATE_NATIVE_PROFILE(skip_page_header_count, skip_page_header_num); + UPDATE_NATIVE_PROFILE(parse_page_header_count, parse_page_header_num); + UPDATE_NATIVE_PROFILE(read_page_header_time, read_page_header_time); + UPDATE_NATIVE_PROFILE(page_read_count, page_read_counter); + UPDATE_NATIVE_PROFILE(page_cache_write_count, page_cache_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_write_count, page_cache_compressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_write_count, + page_cache_decompressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_hit_count, page_cache_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_miss_count, page_cache_missing_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_hit_count, page_cache_compressed_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_hit_count, page_cache_decompressed_hit_counter); +#undef UPDATE_NATIVE_PROFILE + _reported_native_stats = stats; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.h b/be/src/format_v2/parquet/reader/count_column_reader.h new file mode 100644 index 00000000000000..789da8534d2ce3 --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris { +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet { +class NativeParquetMetadata; +struct ParquetColumnSchema; +// Shape-only COUNT(nullable_col) reader. It uses v2's native LevelReader, so BYTE_ARRAY payloads +// are skipped in the encoding stream and never copied into Arrow builders or Doris strings. +class CountColumnReader { +public: + ~CountColumnReader(); + + static Status create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + Status skip(int64_t rows); + Status read_levels(int64_t rows, int64_t* rows_read); + + const std::vector& definition_levels() const { return _definition_levels; } + const std::vector& repetition_levels() const { return _repetition_levels; } + int64_t levels_written() const { return _levels_written; } + +private: + CountColumnReader(std::string name, std::unique_ptr level_reader, + ParquetColumnReaderProfile profile); + void sync_profile(); + + std::unique_ptr _level_reader; + ParquetColumnReaderProfile _profile; + std::string _name; + std::vector _definition_levels; + std::vector _repetition_levels; + native::ColumnChunkReaderStatistics _reported_native_stats; + int64_t _levels_written = 0; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.cpp b/be/src/format_v2/parquet/reader/list_column_reader.cpp deleted file mode 100644 index c042fc99b512aa..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.cpp +++ /dev/null @@ -1,230 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/list_column_reader.h" - -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status ListColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status ListColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status ListColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_batch(rows); -} - -Status ListColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_levels_batch(rows); -} - -Status ListColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet list build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status ListColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet list process result pointer for column {}", - _name); - } - DORIS_CHECK(_element_reader != nullptr); - ColumnArray* array_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr nested_column; - if (column != nullptr) { - array_column = array_column_from_output(*column); - DORIS_CHECK(array_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - nested_column = array_column->get_data_ptr()->assert_mutable(); - const auto& element_output_type = - assert_cast(*remove_nullable(_type)).get_nested_type(); - remove_nullable_wrapper_if_not_expected(element_output_type, &nested_column); - } - - const auto& def_levels = _element_reader->nested_definition_levels(); - const auto& rep_levels = _element_reader->nested_repetition_levels(); - const int64_t levels_written = _element_reader->nested_levels_written(); - std::vector entry_counts; - NullMap parent_nulls; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - if (rep_level == _repetition_level) { - if (entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet LIST column {}", - _name); - } - if (def_level >= _definition_level) { - ++entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet LIST column {} contains null for non-nullable LIST", - _name); - } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - int64_t child_value_count = 0; - if (!_element_reader->is_or_has_repeated_child()) { - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - } - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(total_entries), nested_column, &child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(total_entries), &child_value_count)); - } - } else { - uint64_t pending_entries = 0; - auto flush_pending_entries = [&]() -> Status { - if (pending_entries == 0) { - return Status::OK(); - } - int64_t span_child_value_count = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(pending_entries), nested_column, - &span_child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(pending_entries), &span_child_value_count)); - } - if (span_child_value_count != static_cast(pending_entries)) { - return Status::Corruption( - "Parquet LIST column {} built {} child values, expected {}", _name, - span_child_value_count, pending_entries); - } - child_value_count += span_child_value_count; - pending_entries = 0; - return Status::OK(); - }; - - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - if (entry_count > 0) { - pending_entries += entry_count; - continue; - } - RETURN_IF_ERROR(flush_pending_entries()); - _element_reader->advance_nested_build_level_cursor_past_parent(_repetition_level); - } - RETURN_IF_ERROR(flush_pending_entries()); - } - if (child_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet LIST column {} built {} child values, expected {}", - _name, child_value_count, total_entries); - } - if (column != nullptr) { - array_column->get_data_ptr() = std::move(nested_column); - append_offsets(array_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& ListColumnReader::nested_definition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_definition_levels(); -} - -const std::vector& ListColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_repetition_levels(); -} - -int64_t ListColumnReader::nested_levels_written() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_levels_written(); -} - -bool ListColumnReader::is_or_has_repeated_child() const { - return true; -} - -void ListColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_element_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _element_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.h b/be/src/format_v2/parquet/reader/list_column_reader.h deleted file mode 100644 index d64be546e394d7..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class ListColumnReader final : public ParquetColumnReader { -public: - ListColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr element_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _element_reader(std::move(element_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr - _element_reader; // element reader (recursive; may be Scalar/Struct/List/Map) -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.cpp b/be/src/format_v2/parquet/reader/map_column_reader.cpp deleted file mode 100644 index 8217d0c013abc0..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.cpp +++ /dev/null @@ -1,285 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/map_column_reader.h" - -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status MapColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status MapColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status MapColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_batch(rows)); - return _value_reader->load_nested_batch(rows); -} - -Status MapColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_levels_batch(rows)); - return _value_reader->load_nested_levels_batch(rows); -} - -Status MapColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet map build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status MapColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet map process result pointer for column {}", - _name); - } - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ColumnMap* map_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr key_column; - MutableColumnPtr value_column; - if (column != nullptr) { - map_column = map_column_from_output(*column); - DORIS_CHECK(map_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - key_column = map_column->get_keys_ptr()->assert_mutable(); - value_column = map_column->get_values_ptr()->assert_mutable(); - const auto& map_output_type = assert_cast(*remove_nullable(_type)); - remove_nullable_wrapper_if_not_expected(map_output_type.get_key_type(), &key_column); - remove_nullable_wrapper_if_not_expected(map_output_type.get_value_type(), &value_column); - } - - const auto& def_levels = _key_reader->nested_definition_levels(); - const auto& rep_levels = _key_reader->nested_repetition_levels(); - const int64_t levels_written = _key_reader->nested_levels_written(); - - std::vector entry_counts; - std::vector map_level_indices; - NullMap parent_nulls; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - const int64_t current_level_idx = level_idx; - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - map_level_indices.push_back(current_level_idx); - if (rep_level == _repetition_level) { - if (entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet MAP column {}", - _name); - } - if (def_level >= _definition_level) { - ++entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable MAP", - _name); - } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - } - int64_t key_value_count = 0; - size_t key_start = 0; - if (column != nullptr) { - key_start = key_column->size(); - RETURN_IF_ERROR(_key_reader->build_nested_column(static_cast(total_entries), - key_column, &key_value_count)); - } else if (auto* scalar_key_reader = dynamic_cast(_key_reader.get())) { - // MAP keys are required even if a projected Doris key type is nullable. Validate each - // actual entry directly from the key level stream while advancing past empty/null maps. - for (const int64_t key_level_idx : map_level_indices) { - if (def_levels[key_level_idx] >= _definition_level) { - RETURN_IF_ERROR(scalar_key_reader->validate_nested_value(key_level_idx, true)); - ++key_value_count; - } - } - scalar_key_reader->set_nested_build_level_cursor(level_idx); - } else { - RETURN_IF_ERROR(_key_reader->consume_nested_column(static_cast(total_entries), - &key_value_count)); - } - if (key_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} keys, expected {}", _name, - key_value_count, total_entries); - } - if (column != nullptr) { - if (const auto* nullable_key_column = check_and_get_column(*key_column); - nullable_key_column != nullptr && - nullable_key_column->has_null(key_start, nullable_key_column->size())) { - return Status::Corruption("Parquet MAP column {} contains null key", _name); - } - } - int64_t value_count = 0; - if (auto* scalar_value_reader = dynamic_cast(_value_reader.get())) { - const auto& value_def_levels = scalar_value_reader->nested_definition_levels(); - const auto& value_rep_levels = scalar_value_reader->nested_repetition_levels(); - const int64_t value_levels_written = scalar_value_reader->nested_levels_written(); - int64_t value_level_idx = scalar_value_reader->nested_build_level_cursor(); - for (const int64_t key_level_idx : map_level_indices) { - while (value_level_idx < value_levels_written && - (value_rep_levels[value_level_idx] > _repetition_level || - value_def_levels[value_level_idx] < min_parent_definition_level || - (value_rep_levels[value_level_idx] >= _repetition_level && - value_def_levels[value_level_idx] < _repeated_ancestor_definition_level))) { - ++value_level_idx; - } - if (value_level_idx >= value_levels_written) { - return Status::Corruption( - "Parquet MAP column {} value stream ended before key stream", _name); - } - // MAP is encoded as a repeated key/value struct. The key stream owns entry existence, - // but the value stream still has one shape slot for every consumed MAP slot. Consume - // value slots in lockstep with key slots so shape-only slots from empty/null maps do - // not become scalar values. - if (value_rep_levels[value_level_idx] != rep_levels[key_level_idx]) { - return Status::Corruption( - "Parquet MAP column {} value repetition level is not aligned with key " - "stream", - _name); - } - if (def_levels[key_level_idx] >= _definition_level) { - if (column != nullptr) { - RETURN_IF_ERROR(scalar_value_reader->append_nested_value(value_level_idx, - value_column)); - } else { - RETURN_IF_ERROR( - scalar_value_reader->validate_nested_value(value_level_idx, false)); - } - ++value_count; - } - ++value_level_idx; - } - scalar_value_reader->set_nested_build_level_cursor(value_level_idx); - } else { - // Complex MAP values own their nested shape below the entry slot, so they recursively - // process exactly one child value for each MAP entry. - if (column != nullptr) { - RETURN_IF_ERROR(_value_reader->build_nested_column(static_cast(total_entries), - value_column, &value_count)); - } else { - RETURN_IF_ERROR(_value_reader->consume_nested_column( - static_cast(total_entries), &value_count)); - } - } - if (value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} values, expected {}", _name, - value_count, total_entries); - } - - if (column != nullptr) { - map_column->get_keys_ptr() = std::move(key_column); - map_column->get_values_ptr() = std::move(value_column); - append_offsets(map_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& MapColumnReader::nested_definition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_definition_levels(); -} - -const std::vector& MapColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_repetition_levels(); -} - -int64_t MapColumnReader::nested_levels_written() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_levels_written(); -} - -bool MapColumnReader::is_or_has_repeated_child() const { - return true; -} - -void MapColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _key_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _value_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.h b/be/src/format_v2/parquet/reader/map_column_reader.h deleted file mode 100644 index 1a8ca9c70d8c5b..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.h +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -// 2. build_nested_column() -> -class MapColumnReader final : public ParquetColumnReader { -public: - MapColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr key_reader, - std::unique_ptr value_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _key_reader(std::move(key_reader)), - _value_reader(std::move(value_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr _key_reader; // key column reader (always read fully) - std::unique_ptr - _value_reader; // value column reader (can be pruned by projection) -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp new file mode 100644 index 00000000000000..3421ecdd296892 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" + +#include + +#include "util/hash_util.hpp" + +namespace doris::format::parquet::native { + +namespace { +Status set_hash_strategy(segment_v2::HashStrategyPB strategy, + std::function* hash_func) { + if (strategy != segment_v2::HashStrategyPB::XX_HASH_64) { + return Status::InvalidArgument("Invalid Parquet Bloom filter hash strategy {}", strategy); + } + *hash_func = [](const void* buf, int64_t len, uint64_t, void* out) { + *reinterpret_cast(out) = + HashUtil::xxhash64_compat_with_seed(reinterpret_cast(buf), len, 0); + }; + return Status::OK(); +} +} // namespace + +Status BlockSplitBloomFilter::init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) { + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _num_bytes = filter_size; + _size = _num_bytes; + _data = new char[_size]; + memset(_data, 0, _size); + _has_null = nullptr; + _is_write = true; + segment_v2::g_write_bloom_filter_num << 1; + segment_v2::g_write_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +Status BlockSplitBloomFilter::init(const char* buf, size_t size, + segment_v2::HashStrategyPB strategy) { + if (buf == nullptr || size <= 1) { + return Status::InvalidArgument("Invalid Parquet Bloom filter buffer of size {}", size); + } + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _data = new char[size]; + memcpy(_data, buf, size); + _size = size; + _num_bytes = size; + _has_null = nullptr; + segment_v2::g_read_bloom_filter_num << 1; + segment_v2::g_read_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +void BlockSplitBloomFilter::add_bytes(const char* buf, size_t size) { + DCHECK(buf != nullptr); + add_hash(hash(buf, size)); +} + +bool BlockSplitBloomFilter::test_bytes(const char* buf, size_t size) const { + return test_hash(hash(buf, size)); +} + +void BlockSplitBloomFilter::set_has_null(bool has_null) { + DCHECK(!has_null) << "Parquet Bloom filters do not track nulls"; +} + +void BlockSplitBloomFilter::set_masks(uint32_t key, BlockMask* block_mask) { + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + block_mask->item[i] = uint32_t {1} << ((key * SALT[i]) >> 27); + } +} + +void BlockSplitBloomFilter::add_hash(uint64_t hash) { + DCHECK_GE(_num_bytes, BYTES_PER_BLOCK); + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + bitset[bucket_index * BITS_SET_PER_BLOCK + i] |= block_mask.item[i]; + } +} + +bool BlockSplitBloomFilter::test_hash(uint64_t hash) const { + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + const auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + if ((bitset[bucket_index * BITS_SET_PER_BLOCK + i] & block_mask.item[i]) == 0) { + return false; + } + } + return true; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h new file mode 100644 index 00000000000000..38dc97712ac46c --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/bloom_filter/bloom_filter.h" + +namespace doris::format::parquet::native { + +// Keep the split-block implementation inside v2; the native scan path must not link through the +// legacy Parquet reader merely to evaluate footer Bloom filters. +class BlockSplitBloomFilter final : public segment_v2::BloomFilter { +public: + Status init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) override; + Status init(const char* buf, size_t size, segment_v2::HashStrategyPB strategy) override; + void add_bytes(const char* buf, size_t size) override; + bool test_bytes(const char* buf, size_t size) const override; + void set_has_null(bool has_null) override; + bool has_null() const override { return false; } + void add_hash(uint64_t hash) override; + bool test_hash(uint64_t hash) const override; + +private: + static constexpr int BYTES_PER_BLOCK = 32; + static constexpr int BITS_SET_PER_BLOCK = 8; + static constexpr uint32_t SALT[BITS_SET_PER_BLOCK] = {0x47b6137bU, 0x44974d91U, 0x8824ad5bU, + 0xa2b7289dU, 0x705495c7U, 0x2df1424bU, + 0x9efc4947U, 0x5c6bfb31U}; + + struct BlockMask { + uint32_t item[BITS_SET_PER_BLOCK]; + }; + + static void set_masks(uint32_t key, BlockMask* block_mask); +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp new file mode 100644 index 00000000000000..06e4b239f3abb4 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" + +#include + +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "util/bit_util.h" + +namespace doris::format::parquet::native { +Status BoolPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + std::array values; + size_t decoded = 0; + while (decoded < num_values) { + const size_t batch_size = std::min(values.size(), num_values - decoded); + for (size_t row = 0; row < batch_size; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain decoder"); + } + values[row] = static_cast(value); + } + RETURN_IF_ERROR(consumer.consume(values.data(), batch_size, sizeof(uint8_t))); + decoded += batch_size; + } + return Status::OK(); +} + +Status BoolPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + selected_values_.resize(selection.selected_values); + size_t range_index = 0; + size_t output = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + selected_values_[output++] = static_cast(value); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(selected_values_.data(), output, sizeof(uint8_t)); +} + +Status BoolPlainDecoder::skip_values(size_t num_values) { + int skip_cached = + std::min(num_unpacked_values_ - unpacked_value_idx_, cast_set(num_values)); + unpacked_value_idx_ += skip_cached; + if (skip_cached == num_values) { + return Status::OK(); + } + int num_remaining = cast_set(num_values - skip_cached); + int num_to_skip = BitUtil::RoundDownToPowerOf2(num_remaining, 32); + if (num_to_skip > 0) { + // A failed bulk skip must not be reported as success for a truncated boolean page. + if (!bool_values_.SkipBatch(1, num_to_skip)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } + } + num_remaining -= num_to_skip; + if (num_remaining > 0) { + DCHECK_LE(num_remaining, UNPACKED_BUFFER_LEN); + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ < num_remaining)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } + unpacked_value_idx_ = num_remaining; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h new file mode 100644 index 00000000000000..ee49a236b847ce --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris { +class ColumnSelectVector; +} // namespace doris + +namespace doris::format::parquet::native { +/// Decoder bit-packed boolean-encoded values. +/// Implementation from https://github.com/apache/impala/blob/master/be/src/exec/parquet/parquet-bool-decoder.h +//bit-packed-run-len and rle-run-len must be in the range [1, 2^31 - 1]. +// This means that a Parquet implementation can always store the run length in a signed 32-bit integer +class BoolPlainDecoder final : public Decoder { +public: + BoolPlainDecoder() = default; + ~BoolPlainDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + bool_values_.Reset((const uint8_t*)data->data, data->size); + num_unpacked_values_ = 0; + unpacked_value_idx_ = 0; + _offset = 0; + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&selected_values_, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return selected_values_.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return selected_values_.size() * sizeof(uint8_t); + } + +protected: + inline bool _decode_value(bool* value) { + if (LIKELY(unpacked_value_idx_ < num_unpacked_values_)) { + *value = unpacked_values_[unpacked_value_idx_++]; + } else { + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ == 0)) { + return false; + } + *value = unpacked_values_[0]; + unpacked_value_idx_ = 1; + } + return true; + } + + /// A buffer to store unpacked values. Must be a multiple of 32 size to use the + /// batch-oriented interface of BatchedBitReader. We use uint8_t instead of bool because + /// bit unpacking is only supported for unsigned integers. The values are converted to + /// bool when returned to the user. + static const int UNPACKED_BUFFER_LEN = 128; + uint8_t unpacked_values_[UNPACKED_BUFFER_LEN]; + + /// The number of valid values in 'unpacked_values_'. + int num_unpacked_values_ = 0; + + /// The next value to return from 'unpacked_values_'. + int unpacked_value_idx_ = 0; + + /// Bit packed decoder, used if 'encoding_' is PLAIN. + BatchedBitReader bool_values_; + + std::vector selected_values_; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp new file mode 100644 index 00000000000000..d541550cef180f --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" + +#include + +#include +#include +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +Status BoolRLEDecoder::set_data(Slice* slice) { + _data = slice; + _num_bytes = slice->size; + _offset = 0; + if (_num_bytes < 4) { + return Status::IOError("Received invalid length : " + std::to_string(_num_bytes) + + " (corrupt data page?)"); + } + // Load the first 4 bytes in little-endian, which indicates the length + const auto* data = reinterpret_cast(_data->data); + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > static_cast(_num_bytes - 4)) { + return Status::IOError("Received invalid number of bytes : " + std::to_string(num_bytes) + + " (corrupt data page?)"); + } + _num_bytes = num_bytes; + auto decoder_data = data + 4; + _decoder = RleBatchDecoder(const_cast(decoder_data), num_bytes, 1); + return Status::OK(); +} + +Status BoolRLEDecoder::skip_values(size_t num_values) { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + // GetBatch reports truncation; RleDecoder::Skip assumes a valid run and can spin forever. + if (_decoder.GetBatch(_values.data(), static_cast(batch_size)) != batch_size) { + return Status::IOError("Can't skip enough booleans in Parquet RLE decoder"); + } + skipped += batch_size; + } + return Status::OK(); +} + +Status BoolRLEDecoder::decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) { + _values.resize(num_values); + if (_decoder.GetBatch(_values.data(), cast_set(num_values)) != num_values) { + return Status::IOError("Can't read enough booleans in Parquet RLE decoder"); + } + return consumer.consume(_values.data(), _values.size(), sizeof(uint8_t)); +} + +Status BoolRLEDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + _values.resize(selection.total_values); + if (_decoder.GetBatch(_values.data(), cast_set(selection.total_values)) != + selection.total_values) { + return Status::IOError("Can't read enough booleans in Parquet RLE selection decoder"); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, + range.count * sizeof(uint8_t)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_values.data(), output, sizeof(uint8_t)); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h new file mode 100644 index 00000000000000..55d7a331bb27ca --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/rle_encoding.h" + +namespace doris { +class ColumnSelectVector; +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +class BoolRLEDecoder final : public Decoder { +public: + BoolRLEDecoder() = default; + ~BoolRLEDecoder() override = default; + + Status set_data(Slice* slice) override; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { return _values.capacity() * sizeof(uint8_t); } + size_t active_scratch_bytes() const override { return _values.size() * sizeof(uint8_t); } + +private: + RleBatchDecoder _decoder; + std::vector _values; + size_t _num_bytes; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp new file mode 100644 index 00000000000000..cc3dc66b3cacdc --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" + +#include "core/custom_allocator.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { +Status ByteArrayDictDecoder::set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + _dict_items.clear(); + _dict = std::move(dict); + if (_dict == nullptr) { + return Status::Corruption("Wrong dictionary data for byte array type, dict is null."); + } + if (UNLIKELY(length < 0)) { + return Status::Corruption("Wrong data length in dictionary"); + } + const size_t dict_length = cast_set(length); + // Every BYTE_ARRAY entry needs a four-byte length prefix; bound metadata-driven reserve first. + if (UNLIKELY(num_values > dict_length / sizeof(uint32_t))) { + return Status::Corruption("Dictionary value count exceeds byte array payload"); + } + _dict_items.reserve(num_values); + size_t offset_cursor = 0; + char* dict_item_address = reinterpret_cast(_dict.get()); + for (size_t i = 0; i < num_values; ++i) { + if (UNLIKELY(offset_cursor > dict_length || + dict_length - offset_cursor < sizeof(uint32_t))) { + return Status::Corruption("Wrong data length in dictionary"); + } + uint32_t l = decode_fixed32_le(_dict.get() + offset_cursor); + offset_cursor += sizeof(uint32_t); + if (UNLIKELY(l > dict_length - offset_cursor)) { + return Status::Corruption("Wrong data length in dictionary"); + } + _dict_items.emplace_back(dict_item_address + offset_cursor, l); + offset_cursor += l; + } + if (offset_cursor != dict_length) { + return Status::Corruption("Wrong dictionary data for byte array type"); + } + ++_dictionary_generation; + return Status::OK(); +} + +Status ByteArrayDictDecoder::decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return binary_consumer.consume(_dict_items.data(), _dict_items.size()); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h new file mode 100644 index 00000000000000..790d9f2e4b65ba --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/string_ref.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteArrayDictDecoder final : public BaseDictDecoder { +public: + ByteArrayDictDecoder() = default; + ~ByteArrayDictDecoder() override = default; + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override; + + size_t dictionary_size() const override { return _dict_items.size(); } + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override; + +protected: + // StringRef entries point into BaseDictDecoder::_dict. The dictionary buffer lives for the + // entire column chunk, so retaining a second copy of every byte is unnecessary. + DorisVector _dict_items; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp new file mode 100644 index 00000000000000..064bcffec8dd1f --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" + +#include +#include +#include + +#include "core/column/column.h" +#include "core/data_type/data_type_nullable.h" +#include "core/string_ref.h" + +namespace doris::format::parquet::native { +namespace { +Status read_length(const Slice* data, size_t* offset, uint32_t* length) { + if (UNLIKELY(*offset > data->size || data->size - *offset < sizeof(uint32_t))) { + return Status::IOError("Can't read byte array length from plain decoder"); + } + *length = decode_fixed32_le(reinterpret_cast(data->data) + *offset); + *offset += sizeof(uint32_t); + return Status::OK(); +} +} // namespace + +Status ByteArrayPlainDecoder::decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) { + _payload_offsets.clear(); + _payload_offsets.reserve(num_values); + _value_offsets.clear(); + _value_offsets.reserve(num_values + 1); + _value_offsets.push_back(0); + for (size_t row = 0; row < num_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain decoder"); + } + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY batch exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); + _offset += length; + } + _value_spans.clear(); + if (num_values != 0) { + _value_spans.push_back({.first = 0, .count = num_values}); + } + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), num_values, _value_spans); +} + +Status ByteArrayPlainDecoder::decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + _payload_offsets.clear(); + _payload_offsets.reserve(selection.selected_values); + _value_offsets.clear(); + _value_offsets.reserve(selection.selected_values + 1); + _value_offsets.push_back(0); + _value_spans.clear(); + _value_spans.reserve(selection.ranges.size()); + size_t range_index = 0; + size_t selected_in_range = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + if (selected_in_range != 0) { + _value_spans.push_back({.first = _payload_offsets.size() - selected_in_range, + .count = selected_in_range}); + selected_in_range = 0; + } + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY selection exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); + ++selected_in_range; + } + _offset += length; + } + if (selected_in_range != 0) { + _value_spans.push_back( + {.first = _payload_offsets.size() - selected_in_range, .count = selected_in_range}); + } + DORIS_CHECK_EQ(_payload_offsets.size(), selection.selected_values); + if (_payload_offsets.empty()) { + return Status::OK(); + } + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), _payload_offsets.size(), + _value_spans); +} + +Status ByteArrayPlainDecoder::skip_values(size_t num_values) { + for (int i = 0; i < num_values; ++i) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't skip enough bytes in plain decoder"); + } + _offset += length; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h new file mode 100644 index 00000000000000..45bc82fe823f1a --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/types.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_util.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris { +template +class ColumnDecimal; +} // namespace doris + +namespace doris::format::parquet::native { +class ByteArrayPlainDecoder final : public Decoder { +public: + ByteArrayPlainDecoder() = default; + ~ByteArrayPlainDecoder() override = default; + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override; + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_payload_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_spans, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return (_payload_offsets.capacity() + _value_offsets.capacity()) * sizeof(uint32_t) + + _value_spans.capacity() * sizeof(ParquetSelectionRange); + } + size_t active_scratch_bytes() const override { + return (_payload_offsets.size() + _value_offsets.size()) * sizeof(uint32_t) + + _value_spans.size() * sizeof(ParquetSelectionRange); + } + +private: + std::vector _payload_offsets; + std::vector _value_offsets; + std::vector _value_spans; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp new file mode 100644 index 00000000000000..caf3b675e2dfd7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" + +#include +#include + +#include "core/column/column_fixed_length_object.h" +#include "util/byte_stream_split.h" + +namespace doris::format::parquet::native { +Status ByteStreamSplitDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + return Status::IOError("Out-of-bounds access in Parquet byte-stream-split decoder"); + } + const int64_t stride = static_cast(_data->size / _type_length); + _decoded_values.resize(byte_size); + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + _offset / _type_length, num_values, stride, _decoded_values.data()); + _offset += byte_size; + return consumer.consume(_decoded_values.data(), num_values, static_cast(_type_length)); +} + +Status ByteStreamSplitDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet byte-stream-split selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError( + "Out-of-bounds access in Parquet byte-stream-split selection decoder"); + } + DORIS_CHECK_EQ(_data->size % value_width, 0); + const int64_t stride = static_cast(_data->size / value_width); + _decoded_values.resize(selection.selected_values * value_width); + size_t output = 0; + const size_t first_row = _offset / value_width; + for (const auto& range : selection.ranges) { + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + first_row + range.first, range.count, stride, + _decoded_values.data() + output * value_width); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + _offset += input_bytes; + return consumer.consume(_decoded_values.data(), output, value_width); +} + +Status ByteStreamSplitDecoder::skip_values(size_t num_values) { + // Check in row units before multiplication so a corrupt skip count cannot wrap back in-bounds. + if (UNLIKELY(_type_length <= 0 || _offset > _data->size || + num_values > (_data->size - _offset) / _type_length)) { + return Status::IOError( + "Out-of-bounds access in parquet data decoder: offset = {}, size = {}", _offset, + _data->size); + } + _offset += _type_length * num_values; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h new file mode 100644 index 00000000000000..dfe1d954b61641 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteStreamSplitDecoder final : public Decoder { +public: + ByteStreamSplitDecoder() = default; + ~ByteStreamSplitDecoder() override = default; + + Status set_data(Slice* data) override { + if (UNLIKELY(_type_length <= 0 || data == nullptr || + data->size % static_cast(_type_length) != 0)) { + // Every byte lane must contain the same number of complete physical values. + return Status::Corruption("Invalid Parquet byte-stream-split page length"); + } + return Decoder::set_data(data); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_decoded_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _decoded_values.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return _decoded_values.size() * sizeof(uint8_t); + } + +private: + std::vector _decoded_values; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp new file mode 100644 index 00000000000000..34e940df72f780 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -0,0 +1,1728 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_chunk_reader.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "core/column/column.h" +#include "core/column/column_decimal.h" +#include "core/column/column_dictionary.h" +#include "core/column/column_varbinary.h" +#include "core/column/column_vector.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vexpr.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/bit_util.h" +#include "util/block_compression.h" +#include "util/cpu_info.h" +#include "util/unaligned.h" + +namespace cctz { +class time_zone; +} // namespace cctz +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { + +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available) { + return session_cache_enabled && !storage_cache_disabled && cache_available && header_available; +} + +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed) { + const bool is_v2 = header.__isset.data_page_header_v2; + const bool page_is_compressed = + codec != tparquet::CompressionCodec::UNCOMPRESSED && + (!is_v2 || header.data_page_header_v2.is_compressed || data_page_v2_always_compressed); + if (!page_is_compressed && + UNLIKELY(header.compressed_page_size != header.uncompressed_page_size)) { + // An uncompressed payload has one physical representation, so accepting two lengths makes + // cold reads and decompressed cache hits consume different byte boundaries. + return Status::Corruption( + "Uncompressed Parquet page sizes differ: compressed={}, uncompressed={}", + header.compressed_page_size, header.uncompressed_page_size); + } + return Status::OK(); +} + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by) { + if (created_by.empty()) { + return {}; + } + const ::parquet::ApplicationVersion version(created_by); + return {.parquet_816_padding = + version.VersionLt(::parquet::ApplicationVersion::PARQUET_816_FIXED_VERSION()), + .data_page_v2_always_compressed = version.VersionLt( + ::parquet::ApplicationVersion::PARQUET_CPP_10353_FIXED_VERSION())}; +} + +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range) { + DORIS_CHECK(range != nullptr); + int64_t start = metadata.data_page_offset; + if (metadata.__isset.dictionary_page_offset && metadata.dictionary_page_offset > 0 && + metadata.dictionary_page_offset < start) { + // Some writers use dictionary_page_offset=0 as an absence sentinel. Range validation must + // follow has_dict_page() or the native reader starts at the Parquet magic bytes. + start = metadata.dictionary_page_offset; + } + const int64_t length = metadata.total_compressed_size; + if (UNLIKELY(start < 0 || length < 0)) { + return Status::Corruption("Parquet column chunk has a negative offset or length"); + } + const uint64_t unsigned_start = static_cast(start); + const uint64_t unsigned_length = static_cast(length); + if (UNLIKELY(unsigned_start > file_size || unsigned_length > file_size - unsigned_start)) { + // Thrift range fields are signed and untrusted; validate before converting them to the + // unsigned stream-reader coordinates so overflow cannot wrap back into the file. + return Status::Corruption("Parquet column chunk [{}, {}) exceeds file size {}", start, + unsigned_start + unsigned_length, file_size); + } + size_t bounded_length = static_cast(unsigned_length); + if (parquet_816_padding) { + // parquet-mr before PARQUET-816 under-reported the chunk by up to 100 bytes. Padding stays + // file-bounded and is only enabled for the affected writer versions. + bounded_length += std::min(100, file_size - unsigned_start - unsigned_length); + } + range->offset = static_cast(unsigned_start); + range->length = bounded_length; + return Status::OK(); +} + +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t data_page_offset, int64_t row_count) { + if (index.page_locations.empty() || data_page_offset < 0 || row_count < 0 || + index.page_locations.front().first_row_index != 0 || + index.page_locations.front().offset != data_page_offset || + chunk_range.length > std::numeric_limits::max() - chunk_range.offset) { + return false; + } + // Row indexes alone cannot detect a uniformly shifted OffsetIndex. Anchor its first location + // to the owning metadata so page-to-row mapping cannot silently move by one physical page. + const uint64_t chunk_begin = chunk_range.offset; + const uint64_t chunk_end = chunk_begin + chunk_range.length; + uint64_t previous_end = chunk_begin; + int64_t previous_row = -1; + for (const auto& location : index.page_locations) { + if (location.first_row_index <= previous_row || location.first_row_index >= row_count || + location.offset < 0 || location.compressed_page_size <= 0) { + return false; + } + const uint64_t begin = static_cast(location.offset); + const uint64_t size = static_cast(location.compressed_page_size); + if (begin < chunk_begin || begin < previous_end || begin > chunk_end || + size > chunk_end - begin) { + return false; + } + previous_row = location.first_row_index; + previous_end = begin + size; + } + return true; +} + +namespace { + +class EmptyValueSectionDecoder final : public Decoder { +public: + Status skip_values(size_t num_values) override { + if (UNLIKELY(num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } + return Status::OK(); + } +}; + +Status append_v2_int96_datetime(ColumnDateTimeV2::Container& data, + const ParquetInt96Timestamp& value, + const cctz::time_zone& timezone) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_PER_DAY = 86400000000LL; + static constexpr int64_t MICROS_PER_SECOND = 1000000LL; + + // Arrow normalized out-of-day INT96 nanos before the native V2 path replaced it. Preserve that + // legacy-writer compatibility here; rejecting the carrier loses valid pre-epoch/year-0 values + // already accepted by Doris external-table scans. + const __int128 days = static_cast<__int128>(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + // Truncate the signed nanos field before day normalization. Flooring a negative value first + // changes the historical result by one microsecond. + const __int128 timestamp_micros = days * MICROS_PER_DAY + value.nanos_of_day / 1000; + if (timestamp_micros < std::numeric_limits::min() || + timestamp_micros > std::numeric_limits::max()) { + return Status::DataQualityError("Parquet INT96 timestamp overflows microseconds"); + } + + const int64_t micros = static_cast(timestamp_micros); + int64_t epoch_seconds = micros / MICROS_PER_SECOND; + int64_t micros_of_second = micros % MICROS_PER_SECOND; + if (micros_of_second < 0) { + micros_of_second += MICROS_PER_SECOND; + --epoch_seconds; + } + DateV2Value datetime; + datetime.from_unixtime(epoch_seconds, timezone); + datetime.set_microsecond(static_cast(micros_of_second)); + if (!datetime.is_valid_date()) { + return Status::DataQualityError("Parquet INT96 timestamp is outside the Doris range"); + } + data.push_back(datetime); + return Status::OK(); +} + +class V2Int96DateTimeConsumer final : public ParquetFixedValueConsumer { +public: + V2Int96DateTimeConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state) + : _data(assert_cast(column).get_data()), _state(state) { + static const auto utc = cctz::utc_time_zone(); + _timezone = context.timezone == nullptr ? &utc : context.timezone; + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + const size_t old_size = _data.size(); + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load( + values + row * sizeof(ParquetInt96Timestamp)); + const auto status = append_v2_int96_datetime(_data, value, *_timezone); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + ParquetMaterializationState* _state; + const cctz::time_zone* _timezone = nullptr; +}; + +class RejectV2Int96BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + return Status::NotSupported("INT96 cannot be decoded from binary Parquet values"); + } +}; + +Status read_v2_int96_datetime(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) { + V2Int96DateTimeConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + V2Int96DateTimeConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectV2Int96BinaryConsumer binary_consumer; + const auto dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + return state.materialize_dictionary(column); +} + +Status read_native_or_serde(IColumn& column, const DataTypeSerDe& serde, + ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) { + if (context.physical_type == ParquetPhysicalType::INT96 && + check_and_get_column(&column) != nullptr) { + return read_v2_int96_datetime(column, source, context, num_values, state); + } + return serde.read_column_from_parquet(column, source, context, num_values, state); +} + +Status translate_value_encoding(tparquet::Encoding::type encoding, + ParquetValueEncoding* translated) { + DORIS_CHECK(translated != nullptr); + switch (encoding) { + case tparquet::Encoding::PLAIN: + *translated = ParquetValueEncoding::PLAIN; + return Status::OK(); + case tparquet::Encoding::RLE_DICTIONARY: + case tparquet::Encoding::PLAIN_DICTIONARY: + *translated = ParquetValueEncoding::DICTIONARY; + return Status::OK(); + case tparquet::Encoding::RLE: + *translated = ParquetValueEncoding::RLE; + return Status::OK(); + case tparquet::Encoding::BIT_PACKED: + *translated = ParquetValueEncoding::BIT_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_BINARY_PACKED: + *translated = ParquetValueEncoding::DELTA_BINARY_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_LENGTH_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::DELTA_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::BYTE_STREAM_SPLIT: + *translated = ParquetValueEncoding::BYTE_STREAM_SPLIT; + return Status::OK(); + default: + return Status::NotSupported("Unsupported Parquet encoding {}", + tparquet::to_string(encoding)); + } +} + +template +Status decode_selected_values(IColumn& column, const DataTypeSerDe& serde, Decoder& decoder, + const ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector, + int64_t* materialization_time) { + SCOPED_RAW_TIMER(materialization_time); + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + RETURN_IF_ERROR( + read_native_or_serde(column, serde, decoder, context, run_length, state)); + break; + case ColumnSelectVector::NULL_DATA: + column.insert_many_defaults(run_length); + break; + case ColumnSelectVector::FILTERED_CONTENT: + RETURN_IF_ERROR(decoder.skip_values(run_length)); + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + return Status::OK(); +} + +// Presents one sparse page request as an ordinary sequential source to DataTypeSerDe. SerDe is +// entered once per page fragment; the concrete decoder decides whether to gather selected spans, +// batch-decode and compact, or use the cursor-preserving range fallback. +class SelectedDecodeSource final : public ParquetDecodeSource { +public: + SelectedDecodeSource(Decoder& decoder, const ParquetSelection& selection) + : _decoder(decoder), _selection(selection) {} + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_fixed_values(_selection, consumer); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_binary_values(_selection, consumer); + } + + Status skip_values(size_t num_values) override { + return Status::NotSupported("Selected Parquet source cannot be skipped, values={}", + num_values); + } + + bool has_dictionary() const override { return _decoder.has_dictionary(); } + uint64_t dictionary_generation() const override { return _decoder.dictionary_generation(); } + size_t dictionary_size() const override { return _decoder.dictionary_size(); } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return _decoder.decode_dictionary(fixed_consumer, binary_consumer); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_indices(_selection, indices); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_values(_selection, consumer); + } + + bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const override { + // Avoid touching a dictionary larger than L2 for rows already removed by sparse selection. + // Cache-resident or dense batches instead fuse RLE decode and gather, eliminating the + // page-sized intermediate ID vector. + return _selection.selected_values < _selection.total_values && + dictionary_bytes > static_cast(CpuInfo::get_l2_cache_size()); + } + +private: + Decoder& _decoder; + const ParquetSelection& _selection; +}; + +Status decode_selected_non_null_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = select_vector.num_values(); + selection.selected_values = 0; + + size_t cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + DORIS_CHECK(read_type == ColumnSelectVector::CONTENT || + read_type == ColumnSelectVector::FILTERED_CONTENT); + if (read_type == ColumnSelectVector::CONTENT) { + selection.ranges.push_back({.first = cursor, .count = run_length}); + selection.selected_values += run_length; + } + cursor += run_length; + } + DORIS_CHECK_EQ(cursor, selection.total_values); + if (selection.selected_values == 0) { + return decoder.skip_values(selection.total_values); + } + + SCOPED_RAW_TIMER(materialization_time); + SelectedDecodeSource selected_source(decoder, selection); + return read_native_or_serde(column, serde, selected_source, context, selection.selected_values, + state); +} + +template +bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) { +#define VISIT_COLUMN(TYPE) \ + if (auto* typed = check_and_get_column(&column)) { \ + visitor(*typed); \ + return true; \ + } + VISIT_COLUMN(ColumnUInt8) + VISIT_COLUMN(ColumnInt8) + VISIT_COLUMN(ColumnInt16) + VISIT_COLUMN(ColumnInt32) + VISIT_COLUMN(ColumnInt64) + VISIT_COLUMN(ColumnInt128) + VISIT_COLUMN(ColumnDate) + VISIT_COLUMN(ColumnDateTime) + VISIT_COLUMN(ColumnDateV2) + VISIT_COLUMN(ColumnDateTimeV2) + VISIT_COLUMN(ColumnFloat32) + VISIT_COLUMN(ColumnFloat64) + VISIT_COLUMN(ColumnIPv4) + VISIT_COLUMN(ColumnIPv6) + VISIT_COLUMN(ColumnTimeV2) + VISIT_COLUMN(ColumnTimeStampTz) + VISIT_COLUMN(ColumnOffset32) + VISIT_COLUMN(ColumnOffset64) + VISIT_COLUMN(ColumnDecimal32) + VISIT_COLUMN(ColumnDecimal64) + VISIT_COLUMN(ColumnDecimal128V2) + VISIT_COLUMN(ColumnDecimal128V3) + VISIT_COLUMN(ColumnDecimal256) + VISIT_COLUMN(ColumnString) + VISIT_COLUMN(ColumnString64) + VISIT_COLUMN(ColumnVarbinary) + VISIT_COLUMN(ColumnDictI32) +#undef VISIT_COLUMN + return false; +} + +template +void expand_nullable_pod_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + auto& data = column.get_data(); + DORIS_CHECK_EQ(data.size(), old_size + compact_values); + data.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + data[old_size + output] = typename ColumnType::value_type {}; + } else { + DORIS_CHECK(source > 0); + --source; + data[old_size + output] = std::move(data[old_size + source]); + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_string_values(ColumnStr& column, size_t old_size, + size_t compact_values, const NullMap& selected_nulls) { + auto& offsets = column.get_offsets(); + DORIS_CHECK_EQ(offsets.size(), old_size + compact_values); + const Offset prefix_end = old_size == 0 ? 0 : offsets[old_size - 1]; + offsets.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] == 0) { + DORIS_CHECK(source > 0); + --source; + offsets[old_size + output] = offsets[old_size + source]; + } else { + offsets[old_size + output] = source == 0 ? prefix_end : offsets[old_size + source - 1]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_pod_values(column, old_size, compact_values, selected_nulls); +} + +template +void expand_nullable_values(ColumnStr& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_string_values(column, old_size, compact_values, selected_nulls); +} + +void remap_nullable_conversion_failures(IColumn::Filter* conversion_failure_null_map, + size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + if (conversion_failure_null_map == nullptr) { + return; + } + DORIS_CHECK(conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + size_t source = compact_values; + // Walk backwards so writing an expanded row cannot overwrite an unread compact failure bit. + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + (*conversion_failure_null_map)[old_size + output] = 1; + } else { + DORIS_CHECK(source > 0); + --source; + (*conversion_failure_null_map)[old_size + output] = + (*conversion_failure_null_map)[old_size + source]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +Status decode_selected_nullable_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, NullMap& selected_nulls, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = 0; + selection.selected_values = 0; + selected_nulls.clear(); + selected_nulls.reserve(select_vector.num_values() - select_vector.num_filtered()); + + size_t physical_cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls.resize_fill(selected_nulls.size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls.resize_fill(selected_nulls.size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls.size(), + select_vector.num_values() - select_vector.num_filtered()); + + const size_t old_size = column.size(); + SCOPED_RAW_TIMER(materialization_time); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(decoder.skip_values(selection.total_values)); + column.insert_many_defaults(selected_nulls.size()); + return Status::OK(); + } + + if (state.conversion_failure_null_map != nullptr) { + DORIS_CHECK(state.conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + memset(state.conversion_failure_null_map->data() + old_size, 0, selection.selected_values); + } + SelectedDecodeSource selected_source(decoder, selection); + const auto status = read_native_or_serde(column, serde, selected_source, context, + selection.selected_values, state); + if (!status.ok()) { + if (state.conversion_failure_null_map != nullptr) { + memcpy(state.conversion_failure_null_map->data() + old_size, selected_nulls.data(), + selected_nulls.size()); + } + return status; + } + DORIS_CHECK_EQ(column.size(), old_size + selection.selected_values); + + remap_nullable_conversion_failures(state.conversion_failure_null_map, old_size, + selection.selected_values, selected_nulls); + const bool expanded = visit_nullable_expandable_column(column, [&](auto& typed_column) { + // Decode into the final nested column compactly, then expand in place. This preserves the + // V2 no-intermediate-column invariant while restoring the nullable sparse row layout. + expand_nullable_values(typed_column, old_size, selection.selected_values, selected_nulls); + }); + DORIS_CHECK(expanded); + return Status::OK(); +} + +class PlainPredicateConsumer final : public ParquetFixedValueConsumer { +public: + PlainPredicateConsumer(const VExprSPtrs& conjuncts, DataTypePtr data_type, int column_id, + IColumn::Filter* matches) + : _conjuncts(conjuncts), + _data_type(std::move(data_type)), + _column_id(column_id), + _matches(matches) { + DORIS_CHECK(_matches != nullptr); + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _matches->size(); + _matches->resize_fill(old_size + num_values, 1); + for (const auto& conjunct : _conjuncts) { + RETURN_IF_ERROR(conjunct->execute_on_raw_fixed_values(values, num_values, value_width, + _data_type, _column_id, + _matches->data() + old_size)); + } + return Status::OK(); + } + +private: + const VExprSPtrs& _conjuncts; + DataTypePtr _data_type; + int _column_id; + IColumn::Filter* _matches; +}; + +} // namespace + +template +ColumnChunkReader::ColumnChunkReader( + io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_rows, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range) + : _field_schema(field_schema), + _max_rep_level(field_schema->repetition_level), + _max_def_level(field_schema->definition_level), + _stream_reader(reader), + _metadata(column_chunk->meta_data), + _offset_index(offset_index), + _total_rows(total_rows), + _io_ctx(io_ctx), + _page_read_ctx(page_read_ctx) { + if (chunk_range != nullptr) { + _chunk_range = *chunk_range; + _has_validated_chunk_range = true; + } +} + +template +Status ColumnChunkReader::init() { + size_t start_offset = _has_validated_chunk_range + ? _chunk_range.offset + : (has_dict_page(_metadata) ? _metadata.dictionary_page_offset + : _metadata.data_page_offset); + size_t chunk_size = + _has_validated_chunk_range ? _chunk_range.length : _metadata.total_compressed_size; + // create page reader + _page_reader = create_page_reader( + _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _metadata, + _page_read_ctx, _offset_index); + // get the block compression codec + RETURN_IF_ERROR(get_block_compression_codec(_metadata.codec, &_block_compress_codec)); + _state = INITIALIZED; + RETURN_IF_ERROR(_parse_first_page_header()); + return Status::OK(); +} + +template +Status ColumnChunkReader::skip_nested_values( + const std::vector& def_levels, size_t start_index) { + size_t no_value_cnt = 0; + size_t value_cnt = 0; + + DORIS_CHECK(start_index <= def_levels.size()); + for (size_t idx = start_index; idx < def_levels.size(); idx++) { + level_t def_level = def_levels[idx]; + if (IN_COLLECTION && def_level < _field_schema->repeated_parent_def_level) { + no_value_cnt++; + } else if (def_level < _field_schema->definition_level) { + no_value_cnt++; + } else { + value_cnt++; + } + } + + RETURN_IF_ERROR(skip_values(value_cnt, true)); + RETURN_IF_ERROR(skip_values(no_value_cnt, false)); + return Status::OK(); +} + +template +Status ColumnChunkReader::read_levels( + size_t num_values, std::vector* rep_levels, std::vector* def_levels) { + DORIS_CHECK(rep_levels != nullptr); + DORIS_CHECK(def_levels != nullptr); + if (_remaining_num_values < num_values || _remaining_rep_nums < num_values || + _remaining_def_nums < num_values) { + return Status::Corruption( + "Parquet level reader requested {} slots with only {}/{}/{} remaining", num_values, + _remaining_num_values, _remaining_rep_nums, _remaining_def_nums); + } + + const size_t start_index = def_levels->size(); + rep_levels->resize(rep_levels->size() + num_values, 0); + def_levels->resize(def_levels->size() + num_values, 0); + if (_max_rep_level > 0) { + const size_t decoded = _rep_level_decoder.get_levels( + rep_levels->data() + rep_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet repetition level stream ended after {} of {} slots", + decoded, num_values); + } + } + if (_max_def_level > 0) { + const size_t decoded = _def_level_decoder.get_levels( + def_levels->data() + def_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet definition level stream ended after {} of {} slots", + decoded, num_values); + } + } + _remaining_rep_nums -= num_values; + _remaining_def_nums -= num_values; + return skip_nested_values(*def_levels, start_index); +} + +template +Status ColumnChunkReader::_parse_first_page_header() { + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + _state = INITIALIZED; + return parse_page_header(); + } + if (header->type != tparquet::PageType::DICTIONARY_PAGE) { + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + _state = INITIALIZED; + continue; + } + // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, + // so we should parse the directory page in next_page() + RETURN_IF_ERROR(_decode_dict_page()); + // parse the real first data page + RETURN_IF_ERROR(_page_reader->dict_next_page()); + _state = INITIALIZED; + // A dictionary is the only non-data page with decoder state. Any following index or + // extension pages are skipped by the same pre-data loop. + } +} + +template +Status ColumnChunkReader::parse_page_header() { + if (_state == HEADER_PARSED || _state == DATA_LOADED) { + return Status::OK(); + } + const tparquet::PageHeader* header = nullptr; + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + break; + } + if (header->type == tparquet::PageType::DICTIONARY_PAGE) { + return Status::Corruption("Parquet dictionary page appears after data pages"); + } + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + } + int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values + : header->data_page_header.num_values; + if (page_num_values < 0 || page_num_values > _metadata.num_values || + (!OFFSET_INDEX && + static_cast(page_num_values) > + static_cast(_metadata.num_values) - _chunk_parsed_values)) { + // Page counts are untrusted and feed both level decoders and scratch sizing. Bound each + // page by the column metadata before converting to unsigned counters. + return Status::Corruption("Parquet data page value count {} exceeds column total {}", + page_num_values, _metadata.num_values); + } + if constexpr (!IN_COLLECTION) { + const size_t page_start_row = _page_reader->start_row(); + const size_t page_end_row = _page_reader->end_row(); + if (UNLIKELY(page_end_row < page_start_row || + static_cast(page_num_values) != page_end_row - page_start_row)) { + // Flat columns have exactly one physical value slot per logical row. Rejecting a + // divergent header/OffsetIndex span prevents every later page from shifting rows. + return Status::Corruption( + "Parquet flat data page has {} values for logical row range [{}, {})", + page_num_values, page_start_row, page_end_row); + } + } + _remaining_rep_nums = page_num_values; + _remaining_def_nums = page_num_values; + _remaining_num_values = page_num_values; + + // no offset will parse all header. + if constexpr (OFFSET_INDEX == false) { + _chunk_parsed_values += _remaining_num_values; + } + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status ColumnChunkReader::next_page() { + _state = INITIALIZED; + RETURN_IF_ERROR(_page_reader->next_page()); + return Status::OK(); +} + +template +Status ColumnChunkReader::_get_uncompressed_levels( + const tparquet::DataPageHeaderV2& page_v2, Slice& page_data) { + const size_t rl = page_v2.repetition_levels_byte_length; + const size_t dl = page_v2.definition_levels_byte_length; + if (UNLIKELY(rl > page_data.size || dl > page_data.size - rl)) { + // Validate the physical slice again because a cached entry may itself be truncated. + return Status::Corruption("Parquet data page v2 level bytes exceed available payload"); + } + _v2_rep_levels = Slice(page_data.data, rl); + _v2_def_levels = Slice(page_data.data + rl, dl); + page_data.data += dl + rl; + page_data.size -= dl + rl; + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_data() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::Corruption("Should parse page header"); + } + + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); + int32_t uncompressed_size = header->uncompressed_page_size; + bool page_loaded = false; + + // First, try to reuse a cache handle previously discovered by PageReader + // (header-only lookup) to avoid a second lookup here. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + size_t levels_size = 0; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + levels_size = rl + dl; + if (UNLIKELY(header_size > cached.size || + levels_size > cached.size - header_size)) { + return Status::Corruption("Cached Parquet page is shorter than its v2 levels"); + } + _v2_rep_levels = + Slice(reinterpret_cast(cached.data) + header_size, rl); + _v2_def_levels = + Slice(reinterpret_cast(cached.data) + header_size + rl, dl); + } + // payload_slice points to the bytes after header and levels + if (UNLIKELY(header_size + levels_size > cached.size)) { + return Status::Corruption("Cached Parquet page is shorter than its header"); + } + Slice payload_slice(cached.data + header_size + levels_size, + cached.size - header_size - levels_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + const size_t expected_payload_size = + cache_payload_is_decompressed + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->compressed_page_size) - levels_size; + if (UNLIKELY(payload_slice.size != expected_payload_size)) { + return Status::Corruption("Cached Parquet page payload has size {}, expected {}", + payload_slice.size, expected_payload_size); + } + + if (cache_payload_is_decompressed) { + // Cached payload is already uncompressed + _page_data = payload_slice; + } else { + CHECK(_block_compress_codec); + // Decompress cached payload into _decompress_buf for decoding + size_t uncompressed_payload_size = + header->__isset.data_page_header_v2 + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->uncompressed_page_size); + _reserve_decompress_buf(uncompressed_payload_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_payload_size); + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &_page_data)); + if (UNLIKELY(_page_data.size != uncompressed_payload_size)) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_payload_size); + } + } + // page cache counters were incremented when PageReader did the header-only + // cache lookup. Do not increment again to avoid double-counting. + page_loaded = true; + } + } + + if (!page_loaded) { + const bool prepare_cache_payload = can_prepare_page_cache_payload( + _page_read_ctx.enable_parquet_file_page_cache, config::disable_storage_page_cache, + StoragePageCache::instance() != nullptr, !_page_reader->header_bytes().empty()); + if (_block_compress_codec != nullptr) { + Slice compressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + std::vector level_bytes; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + // uncompressed_size = rl + dl + uncompressed_data_size + // compressed_size = rl + dl + compressed_data_size + uncompressed_size -= header_v2.repetition_levels_byte_length + + header_v2.definition_levels_byte_length; + // copy level bytes (rl + dl) so that we can cache header + levels + uncompressed payload + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (prepare_cache_payload && level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), compressed_data.data, level_sz); + } + // now remove levels from compressed_data for decompression + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, compressed_data)); + } + bool is_v2_compressed = header->__isset.data_page_header_v2 && + (header->data_page_header_v2.is_compressed || + _page_read_ctx.data_page_v2_always_compressed); + bool page_has_compression = header->__isset.data_page_header || is_v2_compressed; + + if (page_has_compression) { + // Decompress payload for immediate decoding + _reserve_decompress_buf(uncompressed_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_size); + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data)); + if (UNLIKELY(_page_data.size != static_cast(uncompressed_size))) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_size); + } + + // Decide whether to cache decompressed payload or compressed payload based on threshold + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (prepare_cache_payload) { + if (cache_payload_decompressed) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + // cache the compressed payload as-is (header | levels | compressed_payload) + _insert_page_into_cache( + level_bytes, Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + } else { + // no compression on this page, use the data directly + _page_data = Slice(compressed_data.data, compressed_data.size); + if (prepare_cache_payload) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } else { + // For uncompressed page, we may still need to extract v2 levels + std::vector level_bytes; + Slice uncompressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(uncompressed_data)); + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (prepare_cache_payload && level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), uncompressed_data.data, level_sz); + } + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, uncompressed_data)); + } + // copy page data out + _page_data = Slice(uncompressed_data.data, uncompressed_data.size); + // Optionally cache uncompressed data for uncompressed pages + if (prepare_cache_payload) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + + // Initialize repetition level and definition level. Skip when level = 0, which means required field. + if (_max_rep_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_rep_level_decoder.init_v2(_v2_rep_levels, _max_rep_level, + _remaining_rep_nums)); + } else { + RETURN_IF_ERROR(_rep_level_decoder.init( + &_page_data, header->data_page_header.repetition_level_encoding, _max_rep_level, + _remaining_rep_nums)); + } + } + if (_max_def_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_def_level_decoder.init_v2(_v2_def_levels, _max_def_level, + _remaining_def_nums)); + } else { + RETURN_IF_ERROR(_def_level_decoder.init( + &_page_data, header->data_page_header.definition_level_encoding, _max_def_level, + _remaining_def_nums)); + } + } + auto encoding = header->__isset.data_page_header_v2 ? header->data_page_header_v2.encoding + : header->data_page_header.encoding; + // change the deprecated encoding to RLE_DICTIONARY + if (encoding == tparquet::Encoding::PLAIN_DICTIONARY) { + encoding = tparquet::Encoding::RLE_DICTIONARY; + } + _current_encoding = encoding; + + // Reuse page decoder + Decoder* encoding_decoder = nullptr; + if (_decoders.find(static_cast(encoding)) != _decoders.end()) { + encoding_decoder = _decoders[static_cast(encoding)].get(); + } else { + std::unique_ptr page_decoder; + RETURN_IF_ERROR(Decoder::get_decoder(_metadata.type, encoding, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + _decoders[static_cast(encoding)] = std::move(page_decoder); + encoding_decoder = _decoders[static_cast(encoding)].get(); + } + _empty_value_section = _page_data.empty() && _max_def_level > 0; + if (_empty_value_section) { + // Nullable all-NULL pages legally contain only definition levels. Keep them decodable for + // every advertised encoding, but make a non-NULL definition level fail before stale decoder + // state from the preceding page can be consumed. + if (_empty_value_decoder == nullptr) { + _empty_value_decoder = std::make_unique(); + } + _page_decoder = _empty_value_decoder.get(); + } else { + _page_decoder = encoding_decoder; + // Encoding headers cannot legitimately advertise more physical values than the data page's + // logical value count; establish the bound before decoders inspect external counts. + _page_decoder->set_expected_values(_remaining_num_values); + RETURN_IF_ERROR(_page_decoder->set_data(&_page_data)); + } + + _state = DATA_LOADED; + return Status::OK(); +} + +template +Status ColumnChunkReader::_decode_dict_page() { + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type); + SCOPED_RAW_TIMER(&_chunk_statistics.decode_dict_time); + + // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. + // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files. + // refer: https://github.com/apache/parquet-format/blob/master/Encodings.md + tparquet::Encoding::type dict_encoding = header->dictionary_page_header.encoding; + if (dict_encoding != tparquet::Encoding::PLAIN_DICTIONARY && + dict_encoding != tparquet::Encoding::PLAIN) { + return Status::InternalError("Unsupported dictionary encoding {}", + tparquet::to_string(dict_encoding)); + } + + // Prepare dictionary data + int32_t uncompressed_size = header->uncompressed_page_size; + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); + auto dict_data = make_unique_buffer(uncompressed_size); + bool dict_loaded = false; + + // Try to load dictionary page from cache + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + // Dictionary page layout in cache: header | payload (compressed or uncompressed) + Slice payload_slice(cached.data + header_size, cached.size - header_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + + if (cache_payload_is_decompressed) { + // Use cached decompressed dictionary data + if (UNLIKELY(payload_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Cached Parquet dictionary payload has size {}, expected {}", + payload_slice.size, uncompressed_size); + } + memcpy(dict_data.get(), payload_slice.data, payload_slice.size); + dict_loaded = true; + } else { + CHECK(_block_compress_codec); + // Decompress cached compressed dictionary data + Slice dict_slice(dict_data.get(), uncompressed_size); + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &dict_slice)); + } + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } + dict_loaded = true; + } + + // When dictionary page is loaded from cache, we need to skip the page data + // to update the offset correctly (similar to calling get_page_data()) + if (dict_loaded) { + _page_reader->skip_page_data(); + } + } + } + + if (!dict_loaded) { + // Load and decompress dictionary page from file + if (_block_compress_codec != nullptr) { + auto dict_num = header->dictionary_page_header.num_values; + if (dict_num == 0 && uncompressed_size != 0) { + return Status::IOError( + "Dictionary page's num_values is {} but uncompressed_size is {}", dict_num, + uncompressed_size); + } + Slice compressed_data; + Slice dict_slice(dict_data.get(), uncompressed_size); + if (dict_num != 0) { + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + // Dictionary probes stop before data pages, so count their decompression here or + // metadata pruning profiles will report no codec work for the scan. + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR( + _block_compress_codec->decompress(compressed_data, &dict_slice)); + } + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } + } + + // Decide whether to cache decompressed or compressed dictionary based on threshold + // If uncompressed_page_size == 0, should_cache_decompressed will return true + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; // Dictionary pages don't have levels + if (cache_payload_decompressed) { + // Cache the decompressed dictionary page + // If dict_num == 0, `dict_slice` will be empty + _insert_page_into_cache(empty_levels, dict_slice); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + DCHECK(!compressed_data.empty()); + // Cache the compressed dictionary page + _insert_page_into_cache(empty_levels, + Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + // `get_page_data` not called, we should skip the page data + // Because `_insert_page_into_cache` will use _page_reader, we should exec `skip_page_data` after `_insert_page_into_cache` + if (dict_num == 0) { + _page_reader->skip_page_data(); + } + } else { + Slice dict_slice; + RETURN_IF_ERROR(_page_reader->get_page_data(dict_slice)); + // The data is stored by BufferedStreamReader, we should copy it out + memcpy(dict_data.get(), dict_slice.data, dict_slice.size); + + // Cache the uncompressed dictionary page + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; + Slice payload(dict_data.get(), uncompressed_size); + _insert_page_into_cache(empty_levels, payload); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + + // Cache page decoder + std::unique_ptr page_decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(_metadata.type, tparquet::Encoding::RLE_DICTIONARY, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + // Set the dictionary data + RETURN_IF_ERROR(page_decoder->set_dict(dict_data, uncompressed_size, + header->dictionary_page_header.num_values)); + _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)] = std::move(page_decoder); + + _has_dict = true; + return Status::OK(); +} + +template +void ColumnChunkReader::_reserve_decompress_buf(size_t size) { + if (size > _decompress_buf_size) { + _decompress_buf_size = BitUtil::next_power_of_two(size); + _decompress_buf = make_unique_buffer(_decompress_buf_size); + } +} + +template +void ColumnChunkReader::_insert_page_into_cache( + const std::vector& level_bytes, const Slice& payload) { + StoragePageCache::CacheKey key = + _page_reader->make_page_cache_key(_page_reader->header_start_offset()); + const std::vector& header_bytes = _page_reader->header_bytes(); + size_t total = header_bytes.size() + level_bytes.size() + payload.size; + auto page = std::make_unique(total, true, segment_v2::DATA_PAGE); + size_t pos = 0; + memcpy(page->data() + pos, header_bytes.data(), header_bytes.size()); + pos += header_bytes.size(); + if (!level_bytes.empty()) { + memcpy(page->data() + pos, level_bytes.data(), level_bytes.size()); + pos += level_bytes.size(); + } + if (payload.size > 0) { + memcpy(page->data() + pos, payload.data, payload.size); + pos += payload.size; + } + page->reset_size(total); + PageCacheHandle handle; + StoragePageCache::instance()->insert(key, page.get(), &handle, segment_v2::DATA_PAGE); + page.release(); + _chunk_statistics.page_cache_write_counter += 1; +} + +template +Status ColumnChunkReader::skip_values(size_t num_values, + bool skip_data) { + if (UNLIKELY(_remaining_num_values < num_values)) { + return Status::IOError("Skip too many values in current page. {} vs. {}", + _remaining_num_values, num_values); + } + if (skip_data) { + if (UNLIKELY(_empty_value_section && num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + RETURN_IF_ERROR(_page_decoder->skip_values(num_values)); + } + // Commit logical page progress only after the physical decoder accepted the whole request. + _remaining_num_values -= num_values; + return Status::OK(); +} + +template +Status ColumnChunkReader::materialize_values( + MutableColumnPtr& doris_column, const DataTypeSerDe& serde, ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector) { + if (select_vector.num_values() == 0) { + return Status::OK(); + } + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + const size_t physical_values = select_vector.num_values() - select_vector.num_nulls(); + if (UNLIKELY(_empty_value_section && physical_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + physical_values); + } + if (UNLIKELY((doris_column->is_column_dictionary() || context.dictionary_index_only) && + !_has_dict && physical_values != 0)) { + return Status::IOError("Not dictionary coded"); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + RETURN_IF_ERROR(translate_value_encoding(_current_encoding, &context.encoding)); + Status status; + if (select_vector.has_filter()) { + if (select_vector.num_nulls() == 0) { + ++_chunk_statistics.hybrid_selection_batches; + status = decode_selected_non_null_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + } else if (visit_nullable_expandable_column(*doris_column, [](auto&) {})) { + // Parquet omits NULL leaf values from the physical stream. For example, the logical + // DATE sequence [d0, NULL, d1, NULL, d2] is physically encoded as [d0, d1, d2]. The + // generic fallback follows the logical selection runs, so those NULLs split one + // contiguous physical span into decode(d0), insert-NULL, decode(d1), insert-NULL, + // decode(d2). On nullable sparse scans this repeatedly enters SerDe and turns decimal + // scaling, date conversion, timestamp/timezone conversion, or dictionary-ID + // materialization into many tiny calls even though the physical values are adjacent. + // + // Build selected non-NULL ranges in physical coordinates instead. In the example this + // decodes [d0, d1, d2] compactly with one SerDe consumer, records [0, 1, 0, 1, 0] as + // the logical NULL layout, and expands the final nested column backwards so unread + // compact values cannot be overwritten. Apply this to every expandable V2 scalar, + // including ordinary PLAIN primitives: otherwise nullable numeric predicates still + // fragment a physical span into millions of SerDe calls and defeat sparse decoding. + ++_chunk_statistics.hybrid_selection_batches; + status = decode_selected_nullable_values( + *doris_column, serde, *_page_decoder, context, state, select_vector, + _nullable_selection_nulls, &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + } else { + ++_chunk_statistics.hybrid_selection_null_fallback_batches; + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); + } + } else { + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, state, + select_vector, + &_chunk_statistics.materialization_time); + } + RETURN_IF_ERROR(status); + _remaining_num_values -= select_vector.num_values(); + return Status::OK(); +} + +template +bool ColumnChunkReader::can_filter_plain_values( + const VExprSPtrs& conjuncts, int column_id) const { + if (conjuncts.empty() || _current_encoding != tparquet::Encoding::PLAIN || + (_metadata.type != tparquet::Type::INT32 && _metadata.type != tparquet::Type::INT64 && + _metadata.type != tparquet::Type::FLOAT && _metadata.type != tparquet::Type::DOUBLE)) { + return false; + } + const auto primitive_type = remove_nullable(_field_schema->data_type)->get_primitive_type(); + const bool has_identity_width = + (_metadata.type == tparquet::Type::INT32 && primitive_type == TYPE_INT) || + (_metadata.type == tparquet::Type::INT64 && primitive_type == TYPE_BIGINT) || + (_metadata.type == tparquet::Type::FLOAT && primitive_type == TYPE_FLOAT) || + (_metadata.type == tparquet::Type::DOUBLE && primitive_type == TYPE_DOUBLE); + if (!has_identity_width) { + // Raw predicates consume the physical Parquet width. Logical conversions such as UINT32 + // to BIGINT must stay on the typed path or a four-byte value is interpreted as eight bytes. + return false; + } + return std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); + }); +} + +template +Status ColumnChunkReader::filter_plain_values( + const VExprSPtrs& conjuncts, int column_id, ColumnSelectVector& select_vector, + NullMap* selected_nulls, IColumn::Filter* physical_matches, IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(selected_nulls != nullptr); + DORIS_CHECK(physical_matches != nullptr); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + *used_filter = false; + row_filter->clear(); + if (!can_filter_plain_values(conjuncts, column_id)) { + return Status::OK(); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + + ParquetSelection selection; + selected_nulls->clear(); + selected_nulls->reserve(select_vector.num_values() - select_vector.num_filtered()); + size_t physical_cursor = 0; + auto build_selection = [&]() { + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == + physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls->resize_fill(selected_nulls->size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls->resize_fill(selected_nulls->size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + }; + if (select_vector.has_filter()) { + build_selection.template operator()(); + } else { + build_selection.template operator()(); + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls->size(), + select_vector.num_values() - select_vector.num_filtered()); + if (UNLIKELY(_empty_value_section && selection.total_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + selection.total_values); + } + + physical_matches->clear(); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(_page_decoder->skip_values(selection.total_values)); + } else { + PlainPredicateConsumer consumer(conjuncts, _field_schema->data_type, column_id, + physical_matches); + RETURN_IF_ERROR(_page_decoder->decode_selected_fixed_values(selection, consumer)); + DORIS_CHECK_EQ(physical_matches->size(), selection.selected_values); + } + + row_filter->reserve(selected_nulls->size()); + size_t physical_row = 0; + for (const uint8_t is_null : *selected_nulls) { + row_filter->push_back(is_null != 0 ? 0 : (*physical_matches)[physical_row++]); + } + DORIS_CHECK_EQ(physical_row, physical_matches->size()); + // Commit logical progress only after both the raw comparison and NULL remapping succeed. + _remaining_num_values -= select_vector.num_values(); + *used_filter = true; + return Status::OK(); +} + +template +Status ColumnChunkReader::seek_to_nested_row(size_t left_row) { + if constexpr (OFFSET_INDEX) { + while (true) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + break; + } else if (has_next_page()) { + RETURN_IF_ERROR(next_page()); + _current_row = _page_reader->start_row(); + } else [[unlikely]] { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + }; + + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + } else { + while (true) { + RETURN_IF_ERROR(parse_page_header()); + if (_page_reader->is_header_v2() || !IN_COLLECTION) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + RETURN_IF_ERROR(load_page_data()); + // this page contain this row. + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + break; + } + + _current_row = _page_reader->end_row(); + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } else { + RETURN_IF_ERROR(load_page_data()); + std::vector rep_levels; + std::vector def_levels; + bool cross_page = false; + + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, left_row - _current_row, + &result_rows, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + bool need_load_next_page = true; + while (cross_page) { + need_load_next_page = false; + rep_levels.clear(); + def_levels.clear(); + RETURN_IF_ERROR(load_cross_page_nested_row(rep_levels, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + } + if (left_row == _current_row) { + break; + } + if (need_load_next_page) { + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", + left_row, _current_row); + } + } + } + }; + } + + return Status::OK(); +} + +template +Status ColumnChunkReader::_skip_nested_rows_in_page(size_t num_rows) { + if (num_rows == 0) { + return Status::OK(); + } + + std::vector rep_levels; + std::vector def_levels; + + bool cross_page = false; + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, num_rows, &result_rows, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + DCHECK(cross_page == false); + if (num_rows != result_rows) [[unlikely]] { + return Status::InternalError("no match skip rows, expect {} vs. real {}", num_rows, + result_rows); + } + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_nested_rows( + std::vector& rep_levels, size_t max_rows, size_t* result_rows, bool* cross_page) { + if (_state != DATA_LOADED) [[unlikely]] { + return Status::IOError("Should load page data first to load nested rows"); + } + *cross_page = false; + *result_rows = 0; + // Reserve only the requested row frontier. One nested row may legitimately contain more + // values and grow the vector incrementally, but a forged page count must not allocate gigabytes + // before the level stream proves those values exist. + const size_t requested_frontier = + max_rows == std::numeric_limits::max() ? max_rows : max_rows + 1; + rep_levels.reserve(rep_levels.size() + + std::min(_remaining_rep_nums, requested_frontier)); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } + if constexpr (!OFFSET_INDEX && IN_COLLECTION) { + // A continuation level is valid across later V1 pages only after this chunk has seen + // a row start; accepting it on the first sequential page invents an orphan parent row. + if (!_nested_row_started && rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + _nested_row_started = true; + } + if (rep_level == 0) { // rep_level 0 indicates start of new row + if (*result_rows == max_rows) { // this page contain max_rows, page no end. + _current_row += max_rows; + _rep_level_rewind_one(); + return Status::OK(); + } + (*result_rows)++; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + _current_row += *result_rows; + + if ((_page_reader->is_header_v2() || OFFSET_INDEX) && + UNLIKELY(_current_row != _page_reader->end_row())) { + // V2 and OffsetIndex advertise an exact logical row span. A page that exhausts its + // repetition levels without that many row starts would otherwise make the caller retry + // the same row forever. + return Status::Corruption( + "Parquet nested data page ended at row {}, expected page end row {}", _current_row, + _page_reader->end_row()); + } + + auto need_check_cross_page = [&]() -> bool { + return !OFFSET_INDEX && IN_COLLECTION && _remaining_rep_nums == 0 && + !_page_reader->is_header_v2() && has_next_page(); + }; + *cross_page = need_check_cross_page(); + return Status::OK(); +}; + +template +Status ColumnChunkReader::load_cross_page_nested_row( + std::vector& rep_levels, bool* cross_page) { + RETURN_IF_ERROR(next_page()); + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + + *cross_page = has_next_page(); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } + if constexpr (!OFFSET_INDEX && IN_COLLECTION) { + if (!_nested_row_started && rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + _nested_row_started = true; + } + if (rep_level == 0) { // rep_level 0 indicates start of new row + *cross_page = false; + _rep_level_rewind_one(); + break; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + return Status::OK(); +} + +template +int32_t ColumnChunkReader::_get_type_length() { + switch (_field_schema->physical_type) { + case tparquet::Type::INT32: + [[fallthrough]]; + case tparquet::Type::FLOAT: + return 4; + case tparquet::Type::INT64: + [[fallthrough]]; + case tparquet::Type::DOUBLE: + return 8; + case tparquet::Type::INT96: + return 12; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + return _field_schema->parquet_schema.type_length; + default: + return -1; + } +} + +/** + * Checks if the given column has a dictionary page. + * + * This function determines the presence of a dictionary page by checking the + * dictionary_page_offset field in the column metadata. The dictionary_page_offset + * must be set and greater than 0, and it must be less than the data_page_offset. + * + * The reason for these checks is based on the implementation in the Java version + * of ORC, where dictionary_page_offset is used to indicate the absence of a dictionary. + * Additionally, Parquet may write an empty row group, in which case the dictionary page + * content would be empty, and thus the dictionary page should not be read. + * + * See https://github.com/apache/arrow/pull/2667/files + */ +bool has_dict_page(const tparquet::ColumnMetaData& column) { + return column.__isset.dictionary_page_offset && column.dictionary_page_offset > 0 && + column.dictionary_page_offset < column.data_page_offset; +} + +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h new file mode 100644 index 00000000000000..02634f96d8038d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -0,0 +1,364 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/common.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "util/slice.h" + +namespace doris { +class BlockCompressionCodec; +class DataTypeSerDe; +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris::format::parquet::native { +using ::doris::ColumnString; + +struct ColumnChunkRange { + size_t offset = 0; + size_t length = 0; +}; + +struct ParquetReaderCompat { + bool parquet_816_padding = false; + bool data_page_v2_always_compressed = false; +}; + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by); +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range); +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t data_page_offset, int64_t row_count); +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available); +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed); + +struct ColumnChunkReaderStatistics { + int64_t decompress_time = 0; + int64_t decompress_cnt = 0; + int64_t decode_header_time = 0; + int64_t decode_value_time = 0; + int64_t materialization_time = 0; + int64_t hybrid_selection_batches = 0; + int64_t hybrid_selection_ranges = 0; + int64_t hybrid_selection_null_fallback_batches = 0; + int64_t decode_dict_time = 0; + int64_t decode_level_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; +}; + +/** + * Read and decode parquet column data into doris block column. + *

Usage:

+ * // Create chunk reader + * ColumnChunkReader chunk_reader(BufferedStreamReader* reader, + * tparquet::ColumnChunk* column_chunk, + * NativeFieldSchema* fieldSchema); + * // Initialize chunk reader + * chunk_reader.init(); + * while (chunk_reader.has_next_page()) { + * // Seek to next page header. Only read and parse the page header, not page data. + * chunk_reader.next_page(); + * // Load data to decoder. Load the page data into underlying container. + * // Or, we can call the chunk_reader.skip_page() to skip current page. + * chunk_reader.load_page_data(); + * // Decode values into column or slice. + * // Or, we can call chunk_reader.skip_values(num_values) to skip some values. + * chunk_reader.materialize_values(column, serde, context, state, selection); + * } + */ +template +class ColumnChunkReader { +public: + ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_row, io::IOContext* io_ctx, + const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range = nullptr); + ~ColumnChunkReader() = default; + + // Initialize chunk reader, will generate the decoder and codec. + Status init(); + + // Whether the chunk reader has a more page to read. + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + return _page_reader->has_next_page(); + } else { + // no offset need parse all page header. + return _chunk_parsed_values < _metadata.num_values; + } + } + + // Skip some values(will not read and parse) in current page if the values are filtered by predicates. + // when skip_data = false, the underlying decoder will not skip data, + // only used when maintaining the consistency of _remaining_num_values. + Status skip_values(size_t num_values, bool skip_data = true); + + // Load page data into the underlying container, + // and initialize the repetition and definition level decoder for current page data. + Status load_page_data(); + Status load_page_data_idempotent() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + return load_page_data(); + } + // The remaining number of values in current page(including null values). Decreased when reading or skipping. + uint32_t remaining_num_values() const { return _remaining_num_values; } + + // Apply one logical selection to the encoded page. Decoder advances physical payload cursors; + // SerDe interprets each selected raw span and materializes the destination Doris column. + Status materialize_values(MutableColumnPtr& doris_column, const DataTypeSerDe& serde, + ParquetDecodeContext& context, ParquetMaterializationState& state, + ColumnSelectVector& select_vector); + + // Evaluate selected fixed-width PLAIN values and return one keep byte per selected logical + // row. NULL comparisons are false and therefore never enter the physical consumer. + Status filter_plain_values(const VExprSPtrs& conjuncts, int column_id, + ColumnSelectVector& select_vector, NullMap* selected_nulls, + IColumn::Filter* physical_matches, IColumn::Filter* row_filter, + bool* used_filter); + bool can_filter_plain_values(const VExprSPtrs& conjuncts, int column_id) const; + + // Get the repetition level decoder of current page. + LevelDecoder& rep_level_decoder() { return _rep_level_decoder; } + // Get the definition level decoder of current page. + LevelDecoder& def_level_decoder() { return _def_level_decoder; } + + level_t max_rep_level() const { return _max_rep_level; } + level_t max_def_level() const { return _max_def_level; } + + bool has_dict() const { return _has_dict; }; + + // Get page decoder + Decoder* get_page_decoder() { return _page_decoder; } + + void release_decoder_scratch(size_t max_retained_bytes) { + for (auto& [encoding, decoder] : _decoders) { + decoder->release_scratch(max_retained_bytes); + } + // Level decoders may batch-convert unsigned RLE values into Doris' signed level_t. + _rep_level_decoder.release_scratch(max_retained_bytes); + _def_level_decoder.release_scratch(max_retained_bytes); + } + + size_t retained_decoder_scratch_bytes() const { + size_t bytes = _rep_level_decoder.retained_scratch_bytes() + + _def_level_decoder.retained_scratch_bytes(); + for (const auto& [encoding, decoder] : _decoders) { + bytes += decoder->retained_scratch_bytes(); + } + return bytes; + } + + size_t active_decoder_scratch_bytes() const { + // Only the current encoding is active. Old decoder instances retain reusable capacity but + // must not make the high-water policy treat their last batch as current working memory. + return (_page_decoder == nullptr ? 0 : _page_decoder->active_scratch_bytes()) + + _rep_level_decoder.active_scratch_bytes() + + _def_level_decoder.active_scratch_bytes(); + } + + tparquet::Encoding::type current_encoding() const { return _current_encoding; } + + ColumnChunkReaderStatistics& chunk_statistics() { + _chunk_statistics.decode_header_time = _page_reader->page_statistics().decode_header_time; + _chunk_statistics.skip_page_header_num = + _page_reader->page_statistics().skip_page_header_num; + _chunk_statistics.parse_page_header_num = + _page_reader->page_statistics().parse_page_header_num; + _chunk_statistics.read_page_header_time = + _page_reader->page_statistics().read_page_header_time; + // PageReader statistics are already cumulative. Snapshot them instead of folding the same + // totals into the chunk on every FileScannerV2 batch. Cache write counters are owned by + // ColumnChunkReader because insertion happens after decompression and therefore remain in + // the chunk accumulator above. + _chunk_statistics.page_read_counter = _page_reader->page_statistics().page_read_counter; + _chunk_statistics.data_page_read_counter = + _page_reader->page_statistics().data_page_read_counter; + _chunk_statistics.page_cache_hit_counter = + _page_reader->page_statistics().page_cache_hit_counter; + _chunk_statistics.page_cache_missing_counter = + _page_reader->page_statistics().page_cache_missing_counter; + _chunk_statistics.page_cache_compressed_hit_counter = + _page_reader->page_statistics().page_cache_compressed_hit_counter; + _chunk_statistics.page_cache_decompressed_hit_counter = + _page_reader->page_statistics().page_cache_decompressed_hit_counter; + return _chunk_statistics; + } + + Decoder* dictionary_decoder() { + return _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)].get(); + } + + size_t page_start_row() const { return _page_reader->start_row(); } + + size_t page_end_row() const { return _page_reader->end_row(); } + + Status parse_page_header(); + Status next_page(); + + Status seek_to_nested_row(size_t left_row); + // Decode level slots without materializing their values. `read_levels` is the flat-column + // counterpart of load_page_nested_rows()+fill_def(): both advance definition/repetition and + // value streams together so a levels-only consumer cannot desynchronize the next data page. + Status read_levels(size_t num_values, std::vector* rep_levels, + std::vector* def_levels); + Status skip_nested_values(const std::vector& def_levels, size_t start_index = 0); + Status fill_def(std::vector& def_values) { + auto before_sz = def_values.size(); + auto append_sz = _remaining_def_nums - _remaining_rep_nums; + def_values.resize(before_sz + append_sz, 0); + if (max_def_level() != 0) { + auto ptr = def_values.data() + before_sz; + const size_t decoded = _def_level_decoder.get_levels(ptr, append_sz); + if (UNLIKELY(decoded != append_sz)) { + def_values.resize(before_sz); + return Status::Corruption( + "Parquet definition level stream ended after {} of {} slots", decoded, + append_sz); + } + } + _remaining_def_nums -= append_sz; + return Status::OK(); + } + + Status load_page_nested_rows(std::vector& rep_levels, size_t max_rows, + size_t* result_rows, bool* cross_page); + Status load_cross_page_nested_row(std::vector& rep_levels, bool* cross_page); + + Slice get_page_data() const { return _page_data; } + const Slice& v2_rep_levels() const { return _v2_rep_levels; } + const Slice& v2_def_levels() const { return _v2_def_levels; } + ColumnChunkReaderStatistics& statistics() { return chunk_statistics(); } + +private: + enum ColumnChunkReaderState { NOT_INIT, INITIALIZED, HEADER_PARSED, DATA_LOADED, PAGE_SKIPPED }; + + // for check dict page. + Status _parse_first_page_header(); + Status _decode_dict_page(); + + void _reserve_decompress_buf(size_t size); + int32_t _get_type_length(); + void _insert_page_into_cache(const std::vector& level_bytes, const Slice& payload); + + Status _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data); + Status _skip_nested_rows_in_page(size_t num_rows); + + level_t _rep_level_get_next() { + if constexpr (IN_COLLECTION) { + return _rep_level_decoder.get_next(); + } + return 0; + } + + void _rep_level_rewind_one() { + if constexpr (IN_COLLECTION) { + _rep_level_decoder.rewind_one(); + } + } + + ColumnChunkReaderState _state = NOT_INIT; + NativeFieldSchema* _field_schema = nullptr; + const level_t _max_rep_level; + const level_t _max_def_level; + + io::BufferedStreamReader* _stream_reader = nullptr; + tparquet::ColumnMetaData _metadata; + const tparquet::OffsetIndex* _offset_index = nullptr; + size_t _current_row = 0; + size_t _total_rows = 0; + io::IOContext* _io_ctx = nullptr; + + std::unique_ptr> _page_reader; + BlockCompressionCodec* _block_compress_codec = nullptr; + + ParquetPageReadContext _page_read_ctx; + ColumnChunkRange _chunk_range; + bool _has_validated_chunk_range = false; + + LevelDecoder _rep_level_decoder; + LevelDecoder _def_level_decoder; + size_t _chunk_parsed_values = 0; + // this page remaining rep/def nums + // if max_rep_level = 0 / max_def_level = 0, this value retail hava value. + uint32_t _remaining_rep_nums = 0; + uint32_t _remaining_def_nums = 0; + // this page remaining values to be processed (for read/skip). + // need parse this page header. + uint32_t _remaining_num_values = 0; + + Slice _page_data; + DorisUniqueBufferPtr _decompress_buf; + size_t _decompress_buf_size = 0; + Slice _v2_rep_levels; + Slice _v2_def_levels; + bool _dict_checked = false; + bool _has_dict = false; + bool _nested_row_started = false; + Decoder* _page_decoder = nullptr; + std::unique_ptr _empty_value_decoder; + bool _empty_value_section = false; + tparquet::Encoding::type _current_encoding = tparquet::Encoding::PLAIN; + // Map: encoding -> Decoder + // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding + std::unordered_map> _decoders; + NullMap _nullable_selection_nulls; + ColumnChunkReaderStatistics _chunk_statistics; +}; + +bool has_dict_page(const tparquet::ColumnMetaData& column); + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp new file mode 100644 index 00000000000000..aa22e4f5f7098d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -0,0 +1,1886 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_reader.h" + +#include +#include +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/column/column.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "exprs/vexpr.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "io/fs/tracing_file_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet::native { +namespace { + +ParquetTimeUnit parquet_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; + } + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; + } + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; + } + return ParquetTimeUnit::UNKNOWN; +} + +template +bool release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + DORIS_CHECK(values != nullptr); + if (values->capacity() * sizeof(T) <= max_retained_bytes) { + return false; + } + std::vector().swap(*values); + return true; +} + +size_t retained_set_bytes(const std::unordered_set& values) { + return values.bucket_count() * sizeof(void*) + values.size() * sizeof(size_t); +} + +Status validate_decimal_physical_type(const NativeFieldSchema& field, int precision, int scale) { + if (precision <= 0 || scale < 0 || scale > precision) { + return Status::Corruption("Parquet decimal field {} has invalid precision {} and scale {}", + field.name, precision, scale); + } + switch (field.physical_type) { + case tparquet::Type::INT32: + if (precision <= 9) { + return Status::OK(); + } + break; + case tparquet::Type::INT64: + if (precision <= 18) { + return Status::OK(); + } + break; + case tparquet::Type::BYTE_ARRAY: + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { + const int length = + field.parquet_schema.__isset.type_length ? field.parquet_schema.type_length : -1; + if (length > 0) { + const int64_t max_precision = (static_cast(length) * 8 - 1) * 30103 / 100000; + if (precision <= max_precision) { + return Status::OK(); + } + } + break; + } + default: + break; + } + return Status::Corruption("Parquet decimal field {} has incompatible physical type {}", + field.name, tparquet::to_string(field.physical_type)); +} + +Status validate_physical_annotation(const NativeFieldSchema& field) { + const auto& schema = field.parquet_schema; + if (field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + (!schema.__isset.type_length || schema.type_length <= 0)) { + return Status::Corruption("Parquet fixed-length field {} has invalid width {}", field.name, + schema.__isset.type_length ? schema.type_length : -1); + } + auto require = [&](bool valid, std::string_view annotation) -> Status { + if (valid) { + return Status::OK(); + } + return Status::Corruption("Parquet {} field {} is incompatible with physical type {}", + annotation, field.name, tparquet::to_string(field.physical_type)); + }; + + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "string"); + } + if (logical.__isset.DECIMAL) { + return validate_decimal_physical_type(field, logical.DECIMAL.precision, + logical.DECIMAL.scale); + } + if (logical.__isset.DATE) { + return require(field.physical_type == tparquet::Type::INT32, "date"); + } + if (logical.__isset.TIME) { + const auto unit = parquet_time_unit(logical.TIME.unit); + return require( + (unit == ParquetTimeUnit::MILLIS && + field.physical_type == tparquet::Type::INT32) || + ((unit == ParquetTimeUnit::MICROS || unit == ParquetTimeUnit::NANOS) && + field.physical_type == tparquet::Type::INT64), + "time"); + } + if (logical.__isset.TIMESTAMP) { + return require( + field.physical_type == tparquet::Type::INT64 && + parquet_time_unit(logical.TIMESTAMP.unit) != ParquetTimeUnit::UNKNOWN, + "timestamp"); + } + if (logical.__isset.INTEGER) { + const int width = logical.INTEGER.bitWidth; + return require(((width == 8 || width == 16 || width == 32) && + field.physical_type == tparquet::Type::INT32) || + (width == 64 && field.physical_type == tparquet::Type::INT64), + "integer"); + } + if (logical.__isset.UUID) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 16, + "UUID"); + } + if (logical.__isset.FLOAT16) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 2, + "FLOAT16"); + } + if (logical.__isset.GEOMETRY || logical.__isset.GEOGRAPHY) { + // Geospatial WKB has no Doris logical mapping here; keep its BYTE_ARRAY payload raw so + // reader validation preserves the physical fallback chosen by native schema inference. + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "geospatial"); + } + if (logical.__isset.UNKNOWN) { + return Status::OK(); + } + return Status::Corruption("Unsupported Parquet logical annotation on field {}", field.name); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "converted string"); + case tparquet::ConvertedType::DECIMAL: + return validate_decimal_physical_type(field, + schema.__isset.precision ? schema.precision : -1, + schema.__isset.scale ? schema.scale : -1); + case tparquet::ConvertedType::DATE: + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + return require(field.physical_type == tparquet::Type::INT32, "converted INT32"); + case tparquet::ConvertedType::TIME_MICROS: + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_64: + return require(field.physical_type == tparquet::Type::INT64, "converted INT64"); + case tparquet::ConvertedType::INTERVAL: + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 12, + "interval"); + default: + return Status::Corruption("Unsupported Parquet converted annotation {} on field {}", + tparquet::to_string(schema.converted_type), field.name); + } +} + +IColumn::Filter* conversion_failure_map(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn::Filter* output_null_map, + IColumn::Filter* compatibility_scratch) { + const auto& schema = field.parquet_schema; + const bool is_utc_timestamp = + field.physical_type == tparquet::Type::INT96 || + (field.physical_type == tparquet::Type::INT64 && + ((schema.__isset.logicalType && schema.logicalType.__isset.TIMESTAMP && + schema.logicalType.TIMESTAMP.isAdjustedToUTC) || + (schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS || + schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MICROS)))); + if (!strict_mode && output_null_map != nullptr && is_utc_timestamp && + remove_nullable(target_type)->get_primitive_type() == TYPE_DATETIMEV2) { + // Legacy UTC timestamp conversion kept out-of-range values as DATETIME defaults. Local + // timestamps intentionally keep non-strict NULL-on-overflow behavior because timezone + // conversion is not part of their representation. + compatibility_scratch->resize_fill(output_null_map->size(), 0); + return compatibility_scratch; + } + return output_null_map; +} + +void mark_local_timestamp_defaults(const NativeFieldSchema& field, const DataTypePtr& target_type, + bool strict_mode, IColumn& data_column, + IColumn::Filter* output_null_map, size_t start_row) { + const auto& schema = field.parquet_schema; + const bool is_local_timestamp = + field.physical_type == tparquet::Type::INT64 && schema.__isset.logicalType && + schema.logicalType.__isset.TIMESTAMP && !schema.logicalType.TIMESTAMP.isAdjustedToUTC; + if (strict_mode || output_null_map == nullptr || !is_local_timestamp || + remove_nullable(target_type)->get_primitive_type() != TYPE_DATETIMEV2) { + return; + } + auto& values = assert_cast(data_column).get_data(); + DORIS_CHECK_EQ(values.size(), output_null_map->size()); + for (size_t row = start_row; row < values.size(); ++row) { + if ((*output_null_map)[row] == 0 && !values[row].is_valid_date()) { + // Local timestamps before Doris' representable calendar can materialize as a zero + // date without a SerDe error. Preserve non-strict scan semantics by nulling only that + // sentinel; physical NULLs and valid local timestamps keep their original map bits. + (*output_null_map)[row] = 1; + } + } +} + +Status init_decode_context(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + DORIS_CHECK(context != nullptr); + // Annotation validation belongs before decoder construction: accepting an impossible pair + // lets the logical SerDe reinterpret a differently sized physical value stream. + RETURN_IF_ERROR(validate_physical_annotation(field)); + switch (field.physical_type) { + case tparquet::Type::BOOLEAN: + context->physical_type = ParquetPhysicalType::BOOLEAN; + break; + case tparquet::Type::INT32: + context->physical_type = ParquetPhysicalType::INT32; + break; + case tparquet::Type::INT64: + context->physical_type = ParquetPhysicalType::INT64; + break; + case tparquet::Type::INT96: + context->physical_type = ParquetPhysicalType::INT96; + break; + case tparquet::Type::FLOAT: + context->physical_type = ParquetPhysicalType::FLOAT; + break; + case tparquet::Type::DOUBLE: + context->physical_type = ParquetPhysicalType::DOUBLE; + break; + case tparquet::Type::BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::BYTE_ARRAY; + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY; + break; + default: + return Status::NotSupported("Unsupported Parquet physical type {}", + tparquet::to_string(field.physical_type)); + } + + const auto& schema = field.parquet_schema; + context->type_length = schema.__isset.type_length ? schema.type_length : -1; + context->decimal_precision = schema.__isset.precision ? schema.precision : -1; + context->decimal_scale = schema.__isset.scale ? schema.scale : -1; + context->timezone = ctz; + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + context->logical_type = ParquetLogicalType::STRING; + } else if (logical.__isset.DECIMAL) { + context->logical_type = ParquetLogicalType::DECIMAL; + context->decimal_precision = logical.DECIMAL.precision; + context->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.DATE) { + context->logical_type = ParquetLogicalType::DATE; + } else if (logical.__isset.TIME) { + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = parquet_time_unit(logical.TIME.unit); + } else if (logical.__isset.TIMESTAMP) { + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = parquet_time_unit(logical.TIMESTAMP.unit); + context->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + } else if (logical.__isset.INTEGER) { + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_bit_width = logical.INTEGER.bitWidth; + context->logical_integer_is_signed = logical.INTEGER.isSigned; + } else if (logical.__isset.UUID) { + context->logical_type = ParquetLogicalType::UUID; + context->logical_uuid = true; + } else if (logical.__isset.FLOAT16) { + context->logical_type = ParquetLogicalType::FLOAT16; + context->logical_float16 = true; + } + if (context->logical_uuid && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 16)) { + return Status::Corruption("Parquet UUID field {} must be FIXED_LEN_BYTE_ARRAY(16)", + field.name); + } + if (context->logical_float16 && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 2)) { + return Status::Corruption("Parquet FLOAT16 field {} must be FIXED_LEN_BYTE_ARRAY(2)", + field.name); + } + return Status::OK(); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + context->logical_type = ParquetLogicalType::STRING; + break; + case tparquet::ConvertedType::DECIMAL: + context->logical_type = ParquetLogicalType::DECIMAL; + break; + case tparquet::ConvertedType::DATE: + context->logical_type = ParquetLogicalType::DATE; + break; + case tparquet::ConvertedType::TIME_MILLIS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MILLIS; + break; + case tparquet::ConvertedType::TIME_MICROS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MICROS; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MILLIS; + // Legacy converted timestamps are defined as UTC-adjusted, unlike an unannotated INT64. + context->timestamp_is_adjusted_to_utc = true; + break; + case tparquet::ConvertedType::TIMESTAMP_MICROS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MICROS; + context->timestamp_is_adjusted_to_utc = true; + break; + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::INT_64: + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_is_signed = + schema.converted_type >= tparquet::ConvertedType::INT_8; + context->logical_integer_bit_width = + schema.converted_type == tparquet::ConvertedType::UINT_8 || + schema.converted_type == tparquet::ConvertedType::INT_8 + ? 8 + : schema.converted_type == tparquet::ConvertedType::UINT_16 || + schema.converted_type == tparquet::ConvertedType::INT_16 + ? 16 + : schema.converted_type == tparquet::ConvertedType::UINT_32 || + schema.converted_type == tparquet::ConvertedType::INT_32 + ? 32 + : 64; + break; + default: + break; + } + return Status::OK(); +} + +} // namespace + +#ifdef BE_TEST +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + return init_decode_context(field, ctz, context); +} + +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode) { + IColumn::Filter output_null_map; + output_null_map.resize_fill(1, 0); + IColumn::Filter compatibility_scratch; + return conversion_failure_map(field, target_type, strict_mode, &output_null_map, + &compatibility_scratch) == &compatibility_scratch; +} + +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row) { + mark_local_timestamp_defaults(field, target_type, strict_mode, data_column, output_null_map, + start_row); +} +#endif + +static void fill_struct_null_map(NativeFieldSchema* field, NullMap& null_map, + const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = def_levels.size(); + DCHECK_EQ(num_levels, rep_levels.size()); + size_t origin_size = null_map.size(); + null_map.resize(origin_size + num_levels); + size_t pos = origin_size; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (def_levels[i] >= field->definition_level) { + null_map[pos++] = 0; + } else { + null_map[pos++] = 1; + } + } + null_map.resize(pos); +} + +static Status fill_array_offset(NativeFieldSchema* field, ColumnArray::Offsets64& offsets_data, + NullMap* null_map_ptr, const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = rep_levels.size(); + if (UNLIKELY(num_levels != def_levels.size())) { + return Status::Corruption("Parquet repetition and definition level counts differ"); + } + size_t origin_size = offsets_data.size(); + offsets_data.resize(origin_size + num_levels); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(origin_size + num_levels); + } + size_t offset_pos = origin_size - 1; + bool parent_opened = false; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (rep_levels[i] == field->repetition_level) { + // A continuation can extend only a parent opened by this aligned logical batch. + if (UNLIKELY(!parent_opened)) { + return Status::Corruption( + "Parquet collection starts with an orphan repetition continuation"); + } + offsets_data[offset_pos]++; + continue; + } + parent_opened = true; + offset_pos++; + offsets_data[offset_pos] = offsets_data[offset_pos - 1]; + if (def_levels[i] >= field->definition_level) { + offsets_data[offset_pos]++; + } + if (null_map_ptr != nullptr) { + if (def_levels[i] >= field->definition_level - 1) { + (*null_map_ptr)[offset_pos] = 0; + } else { + (*null_map_ptr)[offset_pos] = 1; + } + } + } + offsets_data.resize(offset_pos + 1); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(offset_pos + 1); + } + return Status::OK(); +} + +Status ColumnReader::create(io::FileReaderSPtr file, NativeFieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + const std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection, + const std::set& column_ids, + const std::set& filter_column_ids, + const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode) { + size_t total_rows = row_group.num_rows; + if (field->data_type->get_primitive_type() == TYPE_ARRAY) { + std::unique_ptr element_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + element_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, enable_strict_mode)); + auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + element_reader->set_column_in_nested(); + RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field)); + array_reader->_filter_column_ids = filter_column_ids; + reader.reset(array_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_MAP) { + std::unique_ptr key_reader; + std::unique_ptr value_reader; + + if (column_ids.empty() || + column_ids.find(field->children[0].get_column_id()) != column_ids.end()) { + // Create key reader + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + key_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[0]); + key_reader = std::move(skip_reader); + } + + if (column_ids.empty() || + column_ids.find(field->children[1].get_column_id()) != column_ids.end()) { + // Create value reader + RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx, + value_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[1]); + value_reader = std::move(skip_reader); + } + + auto map_reader = MapColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + key_reader->set_column_in_nested(); + value_reader->set_column_in_nested(); + RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field)); + map_reader->_filter_column_ids = filter_column_ids; + reader.reset(map_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) { + std::unordered_map> child_readers; + child_readers.reserve(field->children.size()); + int non_skip_reader_idx = -1; + for (int i = 0; i < field->children.size(); ++i) { + auto& child = field->children[i]; + std::unique_ptr child_reader; + if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) { + RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, + in_collection, column_ids, filter_column_ids, + page_cache_file_key, compat, enable_strict_mode)); + child_readers[child.name] = std::move(child_reader); + // Record the first non-SkippingReader + if (non_skip_reader_idx == -1) { + non_skip_reader_idx = i; + } + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &child); + skip_reader->_filter_column_ids = filter_column_ids; + child_readers[child.name] = std::move(skip_reader); + } + child_readers[child.name]->set_column_in_nested(); + } + // If all children are SkipReadingReader, force the first child to call create + if (non_skip_reader_idx == -1) { + std::unique_ptr child_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, in_collection, + column_ids, filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + child_reader->set_column_in_nested(); + child_readers[field->children[0].name] = std::move(child_reader); + } + auto struct_reader = StructColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field)); + struct_reader->_filter_column_ids = filter_column_ids; + reader.reset(struct_reader.release()); + } else { + auto physical_index = field->physical_column_index; + if (physical_index < 0 || static_cast(physical_index) >= row_group.columns.size()) { + // Keep the leaf-to-chunk invariant checked at this consumer too because unit callers + // can construct a reader without going through NativeParquetMetadata::init_schema(). + return Status::Corruption("Parquet physical column index {} is out of range {}", + physical_index, row_group.columns.size()); + } + const auto offset_it = col_offsets.find(physical_index); + const tparquet::OffsetIndex* offset_index = + offset_it != col_offsets.end() ? &offset_it->second : nullptr; + + const tparquet::ColumnChunk& chunk = row_group.columns[physical_index]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet physical column {} has no chunk metadata", + physical_index); + } + if (in_collection) { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } else { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } + } + return Status::OK(); +} + +void ColumnReader::_generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const { + result_ranges->add(page_row_range); + RowRanges::ranges_intersection(*result_ranges, _row_ranges, result_ranges); +} + +template +Status ScalarColumnReader::init( + io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, RuntimeState* state, + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, + bool enable_strict_mode) { + _field_schema = field; + auto& chunk_meta = _chunk_meta.meta_data; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(chunk_meta, file->size(), compat.parquet_816_padding, + &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_len = chunk_range.length; + size_t prefetch_buffer_size = std::min(chunk_len, max_buf_size); + if ((typeid_cast(file.get()) && + typeid_cast( + ((doris::io::TracingFileReader*)(file.get()))->inner_reader().get())) || + typeid_cast(file.get())) { + // turn off prefetch data when using MergeRangeFileReader + prefetch_buffer_size = 0; + } + _stream_reader = std::make_unique(file, chunk_start, chunk_len, + prefetch_buffer_size); + ParquetPageReadContext ctx( + (state == nullptr) ? true : state->query_options().enable_parquet_file_page_cache, + page_cache_file_key, compat.data_page_v2_always_compressed); + + _chunk_reader = std::make_unique>( + _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx, ctx, + &chunk_range); + _materialization_state.enable_strict_mode = enable_strict_mode; + RETURN_IF_ERROR(_chunk_reader->init()); + RETURN_IF_ERROR(init_decode_context(*field, _ctz, &_decode_context)); + return Status::OK(); +} + +template +void ScalarColumnReader::release_batch_scratch( + size_t max_retained_bytes) { + const size_t retained_bytes = retained_batch_scratch_bytes(); + const size_t active_bytes = active_batch_scratch_bytes(); + if (retained_bytes <= max_retained_bytes || active_bytes > max_retained_bytes) { + _oversized_scratch_idle_batches = 0; + return; + } + // An adaptive probe or one repeated outlier must not pin memory forever, but immediately + // dropping a large steady-state buffer makes every following batch allocate it again. Require + // three ordinary batches before treating an oversized capacity as idle. + constexpr uint8_t OVERSIZED_SCRATCH_IDLE_BATCHES = 3; + if (++_oversized_scratch_idle_batches < OVERSIZED_SCRATCH_IDLE_BATCHES) { + return; + } + _oversized_scratch_idle_batches = 0; + if (_chunk_reader != nullptr) { + // Persistent decoders also own batch-sized value/slice buffers, not only the reader. + _chunk_reader->release_decoder_scratch(max_retained_bytes); + } + bool release_selection = false; + release_selection |= release_vector_if_oversized(&_rep_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_def_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_null_run_lengths, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_nested_filter_map_data, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.dictionary_indices, + max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.selection.ranges, + max_retained_bytes); + if (retained_set_bytes(_ancestor_null_indices) > max_retained_bytes) { + std::unordered_set().swap(_ancestor_null_indices); + release_selection = true; + } + if (release_selection) { + _select_vector = ColumnSelectVector(); + } +} + +template +size_t ScalarColumnReader::retained_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->retained_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.capacity() * sizeof(level_t) + + _def_levels.capacity() * sizeof(level_t) + + _null_run_lengths.capacity() * sizeof(uint16_t) + + _nested_filter_map_data.capacity() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.capacity() * sizeof(uint32_t) + + _materialization_state.selection.ranges.capacity() * sizeof(ParquetSelectionRange) + + retained_set_bytes(_ancestor_null_indices); +} + +template +size_t ScalarColumnReader::active_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->active_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.size() * sizeof(level_t) + + _def_levels.size() * sizeof(level_t) + _null_run_lengths.size() * sizeof(uint16_t) + + _nested_filter_map_data.size() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.size() * sizeof(uint32_t) + + _materialization_state.selection.ranges.size() * sizeof(ParquetSelectionRange) + + _ancestor_null_indices.size() * sizeof(size_t); +} + +#ifdef BE_TEST +template +void ScalarColumnReader::reserve_batch_scratch_for_test( + size_t elements) { + _rep_levels.reserve(elements); + _def_levels.reserve(elements); + _null_run_lengths.reserve(elements); + _nested_filter_map_data.reserve(elements); + _materialization_state.dictionary_indices.reserve(elements); + _materialization_state.selection.ranges.reserve(elements); + _ancestor_null_indices.reserve(elements); +} + +template +size_t ScalarColumnReader::retained_batch_scratch_bytes_for_test() + const { + return retained_batch_scratch_bytes(); +} +#endif + +template +Status ScalarColumnReader::_skip_values(size_t num_values) { + if (num_values == 0) { + return Status::OK(); + } + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t skipped = 0; + size_t null_size = 0; + size_t nonnull_size = 0; + while (skipped < num_values) { + level_t def_level = -1; + size_t loop_skip = def_decoder.get_next_run(&def_level, num_values - skipped); + if (loop_skip == 0) { + return Status::Corruption("Parquet definition level stream ended while skipping"); + } + if (def_level < _field_schema->definition_level) { + null_size += loop_skip; + } else { + nonnull_size += loop_skip; + } + skipped += loop_skip; + } + if (null_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(null_size, false)); + } + if (nonnull_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(nonnull_size, true)); + } + } else { + RETURN_IF_ERROR(_chunk_reader->skip_values(num_values)); + } + return Status::OK(); +} + +template +Status ScalarColumnReader::_read_values(size_t num_values, + ColumnPtr& doris_column, + const DataTypePtr& type, + FilterMap& filter_map, + bool is_dict_filter) { + if (num_values == 0) { + return Status::OK(); + } + MutableColumnPtr data_column; + _null_run_lengths.clear(); + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level; + size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + return Status::Corruption( + "Parquet definition level stream ended while materializing"); + } + + bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + prev_is_null = is_null; + has_read += loop_read; + } + } + } else { + if (_chunk_reader->max_def_level() > 0) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (_null_run_lengths.empty()) { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + } + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, map_data_column, + &filter_map, _filter_map_index)); + _filter_map_index += num_values; + } + DORIS_CHECK(_serde != nullptr); + // Keep selected-row cardinality stable: non-strict conversion failures append a nested + // default and mark this matching nullable output row instead of shortening the column. + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = + conversion_failure_map(*_field_schema, type, _materialization_state.enable_strict_mode, + map_data_column, &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values(data_column, *_serde, _decode_context, + _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + return status; +} + +/** + * Load the nested column data of complex type. + * A row of complex type may be stored across two(or more) pages, and the parameter `align_rows` indicates that + * whether the reader should read the remaining value of the last row in previous page. + */ +template +Status ScalarColumnReader::_read_nested_column( + ColumnPtr& doris_column, const DataTypePtr& type, FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter) { + _rep_levels.clear(); + _def_levels.clear(); + + // Handle nullable columns + MutableColumnPtr data_column; + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + _nested_filter_map_data.clear(); + + auto read_and_fill_data = [&](size_t before_rep_level_sz, size_t filter_map_index) { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + if (filter_map.has_filter()) { + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, before_rep_level_sz, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + } else { + RETURN_IF_ERROR(_nested_filter_map.init( + nullptr, _rep_levels.size() - before_rep_level_sz, false)); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + RETURN_IF_ERROR(gen_nested_null_map(before_rep_level_sz, _rep_levels.size(), + _null_run_lengths, _ancestor_null_indices)); + + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init( + _null_run_lengths, + _rep_levels.size() - before_rep_level_sz - _ancestor_null_indices.size(), + map_data_column, &_nested_filter_map, 0, &_ancestor_null_indices)); + } + + DORIS_CHECK(_serde != nullptr); + // Nested materialization must preserve the same value/null-map row alignment invariant. + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = conversion_failure_map( + *_field_schema, type, _materialization_state.enable_strict_mode, map_data_column, + &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values( + data_column, *_serde, _decode_context, _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + RETURN_IF_ERROR(status); + if (!_ancestor_null_indices.empty()) { + RETURN_IF_ERROR(_chunk_reader->skip_values(_ancestor_null_indices.size(), false)); + } + if (filter_map.has_filter()) { + auto new_rep_sz = before_rep_level_sz; + for (size_t idx = before_rep_level_sz; idx < _rep_levels.size(); idx++) { + if (_nested_filter_map_data[idx - before_rep_level_sz]) { + _rep_levels[new_rep_sz] = _rep_levels[idx]; + _def_levels[new_rep_sz] = _def_levels[idx]; + new_rep_sz++; + } + } + _rep_levels.resize(new_rep_sz); + _def_levels.resize(new_rep_sz); + } + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + size_t right_row = std::min(left_row + batch_size - *read_rows, + (size_t)_row_ranges.get_range_to(_current_range_idx)); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + size_t load_rows = 0; + bool cross_page = false; + size_t before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &load_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && load_rows == 0 && !cross_page)) { + // A bounded V2/indexed page must advance at least one logical row; zero progress would + // leave both range cursors unchanged and spin forever on corrupt repetition levels. + return Status::Corruption("Parquet nested reader made no row progress"); + } + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index)); + _filter_map_index += load_rows; + while (cross_page) { + before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index - 1)); + } + *read_rows += load_rows; + _current_row_index += load_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Status ScalarColumnReader::_read_plain_filter_values( + size_t num_values, const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, + IColumn::Filter* row_filter) { + DORIS_CHECK(row_filter != nullptr); + _null_run_lengths.clear(); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level = -1; + const size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + return Status::Corruption( + "Parquet definition level stream ended while filtering PLAIN values"); + } + const bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + prev_is_null = is_null; + has_read += loop_read; + } + } else { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + } + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, nullptr, &filter_map, + _filter_map_index)); + _filter_map_index += num_values; + bool used_filter = false; + RETURN_IF_ERROR(_chunk_reader->filter_plain_values( + conjuncts, column_id, _select_vector, &_plain_predicate_nulls, + &_plain_predicate_matches, row_filter, &used_filter)); + // Chunk encodings are prevalidated before any definition level is consumed, so a false result + // here would make a materializing fallback observe an advanced level cursor. + DORIS_CHECK(used_filter); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_plain_filter( + const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, size_t batch_size, + IColumn::Filter* row_filter, size_t* read_rows, bool* eof, bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + if (_in_nested || conjuncts.empty()) { + return Status::OK(); + } + if (!std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); + })) { + return Status::OK(); + } + // Levels use RLE/BIT_PACKED, while every value page must be PLAIN. Reject mixed-encoding + // chunks before touching either cursor so the caller can safely use the normal expression path. + const bool plain_only = std::ranges::all_of( + _chunk_meta.meta_data.encodings, [](const tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN || + encoding == tparquet::Encoding::RLE || + encoding == tparquet::Encoding::BIT_PACKED; + }); + if (!plain_only) { + return Status::OK(); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + _current_row_index = right_row; + } else { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + if (!_chunk_reader->can_filter_plain_values(conjuncts, column_id)) { + return Status::OK(); + } + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); ++idx) { + const auto range = read_ranges.get_range(idx); + const size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + const size_t values = + std::min(static_cast(range.to() - range.from()), batch_size - has_read); + IColumn::Filter fragment_filter; + RETURN_IF_ERROR(_read_plain_filter_values(values, conjuncts, column_id, filter_map, + &fragment_filter)); + row_filter->insert(row_filter->end(), fragment_filter.begin(), fragment_filter.end()); + has_read += values; + *read_rows += values; + _current_row_index += values; + if (has_read == batch_size) { + break; + } + } + } + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + *used_filter = true; + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_levels(FilterMap& filter_map, + size_t batch_size, + size_t* read_rows, + bool* eof) { + DORIS_CHECK(_in_nested); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + _rep_levels.clear(); + _def_levels.clear(); + *read_rows = 0; + + auto consume_level_segment = [&](size_t level_start, size_t filter_map_index) -> Status { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + // Advance the encoded value stream without constructing a temporary Doris column. The + // definition levels identify the physical values that actually exist; dictionary skips + // still validate every index. + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(_def_levels, level_start)); + if (!filter_map.has_filter()) { + return Status::OK(); + } + + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, level_start, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + size_t write_index = level_start; + for (size_t read_index = level_start; read_index < _rep_levels.size(); ++read_index) { + if (_nested_filter_map_data[read_index - level_start] != 0) { + _rep_levels[write_index] = _rep_levels[read_index]; + _def_levels[write_index] = _def_levels[read_index]; + ++write_index; + } + } + _rep_levels.resize(write_index); + _def_levels.resize(write_index); + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + const size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + const size_t right_row = + std::min(left_row + batch_size - *read_rows, + static_cast(_row_ranges.get_range_to(_current_range_idx))); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + + size_t loaded_rows = 0; + bool cross_page = false; + size_t level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &loaded_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && loaded_rows == 0 && !cross_page)) { + // Keep the levels-only path under the same forward-progress invariant as value reads. + return Status::Corruption("Parquet nested level reader made no row progress"); + } + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index)); + _filter_map_index += loaded_rows; + while (cross_page) { + level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index - 1)); + } + + *read_rows += loaded_rows; + _current_row_index += loaded_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Result +ScalarColumnReader::convert_dict_column_to_string_column( + const ColumnInt32* dict_column, const DataTypePtr& target_type) { + DORIS_CHECK(dict_column != nullptr); + DORIS_CHECK(target_type != nullptr); + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + DORIS_CHECK(dictionary_decoder != nullptr); + // A Hive STRING can be backed by an unannotated Parquet BYTE_ARRAY that native metadata maps to + // VARBINARY. The dictionary must use the projected table type, exactly like normal data pages. + const DataTypePtr dictionary_type = remove_nullable(target_type); + const DataTypeSerDeSPtr dictionary_serde = dictionary_type->get_serde(); + if (_materialization_state.dictionary_generation != + dictionary_decoder->dictionary_generation()) { + _materialization_state.typed_dictionary = dictionary_type->create_column(); + ParquetDecodeContext dictionary_context = _decode_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + dictionary_context.dictionary_index_only = false; + auto status = dictionary_serde->read_parquet_dictionary( + *_materialization_state.typed_dictionary, *dictionary_decoder, dictionary_context); + if (!status.ok()) { + return ResultError(std::move(status)); + } + DORIS_CHECK_EQ(_materialization_state.typed_dictionary->size(), + dictionary_decoder->dictionary_size()); + _materialization_state.dictionary_generation = dictionary_decoder->dictionary_generation(); + } + + auto result = _materialization_state.typed_dictionary->clone_empty(); + const auto& source_indices = dict_column->get_data(); + auto& indices = _materialization_state.dictionary_indices; + indices.resize(source_indices.size()); + for (size_t row = 0; row < source_indices.size(); ++row) { + if (UNLIKELY(source_indices[row] < 0 || + static_cast(source_indices[row]) >= + _materialization_state.typed_dictionary->size())) { + return ResultError(Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + source_indices[row], row, _materialization_state.typed_dictionary->size())); + } + indices[row] = static_cast(source_indices[row]); + } + result->insert_indices_from(*_materialization_state.typed_dictionary, indices.data(), + indices.data() + indices.size()); + return result; +} + +template +Result ScalarColumnReader::dictionary_values( + const DataTypePtr& target_type) { + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + if (dictionary_decoder == nullptr || dictionary_decoder->dictionary_size() == 0) { + return ResultError(Status::NotSupported("Parquet column has no reusable dictionary")); + } + auto ids = ColumnInt32::create(); + auto& data = ids->get_data(); + data.resize(dictionary_decoder->dictionary_size()); + for (size_t dictionary_id = 0; dictionary_id < data.size(); ++dictionary_id) { + data[dictionary_id] = cast_set(dictionary_id); + } + // Materialize the typed dictionary once and keep it in _materialization_state. Later row-level + // filtering decodes only ids and flattens surviving values from this same dictionary. + return convert_dict_column_to_string_column(ids.get(), target_type); +} + +template +Status ScalarColumnReader::_try_load_dict_page(bool* loaded, + bool* has_dict) { + // _chunk_reader init will load first page header to check whether has dict page + *loaded = true; + *has_dict = _chunk_reader->has_dict(); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_data( + ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, FilterMap& filter_map, + size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + const DataTypePtr materialization_type = remove_nullable(type); + const DataTypePtr file_type = remove_nullable(_field_schema->data_type); + if (!is_dict_filter && !file_type->equals(*materialization_type)) { + // File-to-table casts belong to ColumnMapper. Reaching this reader with a table type would + // also evaluate file-local predicates against values produced under the wrong type. + return Status::InternalError( + "Native Parquet reader type mismatch for file column '{}': file={}, requested={}", + _field_schema->name, file_type->get_name(), materialization_type->get_name()); + } + + const DataTypePtr serde_type = is_dict_filter ? file_type : materialization_type; + if (_serde_type != serde_type.get() || _dictionary_index_only != is_dict_filter) { + _serde_type = serde_type.get(); + _serde = serde_type->get_serde(); + _dictionary_index_only = is_dict_filter; + _materialization_state.reset_dictionary(); + } + _decode_context.dictionary_index_only = is_dict_filter; + + _def_levels.clear(); + _rep_levels.clear(); + *read_rows = 0; + + if (_in_nested) { + return _read_nested_column(doris_column, materialization_type, filter_map, batch_size, + read_rows, eof, is_dict_filter); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + + do { + // generate the row ranges that should be read + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + // skip the whole page + _current_row_index = right_row; + } else { + bool skip_whole_batch = false; + // Determining whether to skip page or batch will increase the calculation time. + // When the filtering effect is greater than 60%, it is possible to skip the page or batch. + if (filter_map.has_filter() && filter_map.filter_ratio() > 0.6) { + // lazy read + size_t remaining_num_values = read_ranges.count(); + if (batch_size >= remaining_num_values && + filter_map.can_filter_all(remaining_num_values, _filter_map_index)) { + // We can skip the whole page if the remaining values are filtered by predicate columns + _filter_map_index += remaining_num_values; + _current_row_index = right_row; + *read_rows = remaining_num_values; + break; + } + skip_whole_batch = batch_size <= remaining_num_values && + filter_map.can_filter_all(batch_size, _filter_map_index); + if (skip_whole_batch) { + _filter_map_index += batch_size; + } + } + // load page data to decode or skip values + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); idx++) { + auto range = read_ranges.get_range(idx); + // generate the skipped values + size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + // generate the read values + size_t read_values = + std::min((size_t)(range.to() - range.from()), batch_size - has_read); + if (skip_whole_batch) { + RETURN_IF_ERROR(_skip_values(read_values)); + } else { + RETURN_IF_ERROR(_read_values(read_values, doris_column, materialization_type, + filter_map, is_dict_filter)); + } + has_read += read_values; + *read_rows += read_values; + _current_row_index += read_values; + if (has_read == batch_size) { + break; + } + } + } + } while (false); + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + + return Status::OK(); +} + +Status ArrayColumnReader::init(std::unique_ptr element_reader, + NativeFieldSchema* field) { + _field_schema = field; + _element_reader = std::move(element_reader); + return Status::OK(); +} + +Status ArrayColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_ARRAY) { + return Status::Corruption( + "Wrong data type for column '{}', expected Array type, actual type: {}.", + _field_schema->name, type->get_name()); + } + + ColumnPtr& element_column = assert_cast(*data_column).get_data_ptr(); + const DataTypePtr& element_type = + (assert_cast(remove_nullable(type).get()))->get_nested_type(); + // read nested column + RETURN_IF_ERROR(_element_reader->read_column_data(element_column, element_type, + root_node->element(), filter_map, batch_size, + read_rows, eof, is_dict_filter)); + if (*read_rows == 0) { + return Status::OK(); + } + + ColumnArray::Offsets64& offsets_data = assert_cast(*data_column).get_offsets(); + // fill offset and null map + RETURN_IF_ERROR(fill_array_offset(_field_schema, offsets_data, null_map_ptr, + _element_reader->get_rep_level(), + _element_reader->get_def_level())); + if (UNLIKELY(element_column->size() != offsets_data.back())) { + return Status::Corruption("Parquet array element count does not match repetition levels"); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::init(std::unique_ptr key_reader, + std::unique_ptr value_reader, NativeFieldSchema* field) { + _field_schema = field; + _key_reader = std::move(key_reader); + _value_reader = std::move(value_reader); + return Status::OK(); +} + +Status MapColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (remove_nullable(type)->get_primitive_type() != PrimitiveType::TYPE_MAP) { + return Status::Corruption( + "Wrong data type for column '{}', expected Map type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& map = assert_cast(*data_column); + const DataTypePtr& key_type = + assert_cast(remove_nullable(type).get())->get_key_type(); + const DataTypePtr& value_type = + assert_cast(remove_nullable(type).get())->get_value_type(); + ColumnPtr& key_column = map.get_keys_ptr(); + ColumnPtr& value_column = map.get_values_ptr(); + + size_t key_rows = 0; + size_t value_rows = 0; + bool key_eof = false; + bool value_eof = false; + int64_t orig_col_column_size = key_column->size(); + + RETURN_IF_ERROR(_key_reader->read_column_data(key_column, key_type, root_node->key(), + filter_map, batch_size, &key_rows, &key_eof, + is_dict_filter)); + + while (value_rows < key_rows && !value_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_value_reader->read_column_data( + value_column, value_type, root_node->value(), filter_map, key_rows - value_rows, + &loop_rows, &value_eof, is_dict_filter, key_column->size() - orig_col_column_size)); + value_rows += loop_rows; + } + if (UNLIKELY(key_rows != value_rows)) { + // MAP children share one logical-row boundary; EOF in one sibling is file corruption. + return Status::Corruption("Parquet map value reader returned {} rows for {} key rows", + value_rows, key_rows); + } + *read_rows = key_rows; + *eof = key_eof; + + if (*read_rows == 0) { + return Status::OK(); + } + + const auto parent_shape = [this](const ColumnReader& reader) { + std::vector> shape; + const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } + shape.reserve(rep_levels.size()); + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + // MAP siblings may have child-local collections and nullness. At this boundary, + // only the outer entry distribution and whether that entry exists must agree. + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); + } + } + return shape; + }; + if (UNLIKELY(_key_reader->get_rep_level().size() != _key_reader->get_def_level().size() || + _value_reader->get_rep_level().size() != _value_reader->get_def_level().size())) { + return Status::Corruption("Parquet map sibling level counts differ"); + } + if (UNLIKELY(parent_shape(*_key_reader) != parent_shape(*_value_reader))) { + return Status::Corruption("Parquet map key/value outer entry shapes differ"); + } + + if (UNLIKELY(key_column->size() != value_column->size())) { + return Status::Corruption("Parquet map key/value entry counts differ: {} vs {}", + key_column->size(), value_column->size()); + } + // Non-strict conversion can persist nullable Doris MAP keys, so the native reader must + // preserve writer output instead of reclassifying an otherwise valid file as corrupt. + const auto& key_rep_levels = _key_reader->get_rep_level(); + // fill offset and null map + // The key leaf is the canonical outer MAP shape. A nested value has additional repetition + // levels for its own collections, so comparing the two leaves would reject valid MAP values. + RETURN_IF_ERROR(fill_array_offset(_field_schema, map.get_offsets(), null_map_ptr, + key_rep_levels, _key_reader->get_def_level())); + if (UNLIKELY(key_column->size() != map.get_offsets().back())) { + return Status::Corruption("Parquet map entry count does not match repetition levels"); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + DORIS_CHECK(dynamic_cast(_key_reader.get()) == nullptr); + return _key_reader->read_column_levels(filter_map, batch_size, read_rows, eof); +} + +Status StructColumnReader::init( + std::unordered_map>&& child_readers, + NativeFieldSchema* field) { + _field_schema = field; + _child_readers = std::move(child_readers); + return Status::OK(); +} + +Status StructColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + _read_column_names.clear(); + for (const auto& child : _field_schema->children) { + auto reader = _child_readers.find(child.name); + DORIS_CHECK(reader != _child_readers.end()); + if (dynamic_cast(reader->second.get()) != nullptr) { + continue; + } + _read_column_names.emplace_back(child.name); + return reader->second->read_column_levels(filter_map, batch_size, read_rows, eof); + } + return Status::InternalError("Struct {} has no physical reader for levels", + _field_schema->name); +} +Status StructColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_STRUCT) { + return Status::Corruption( + "Wrong data type for column '{}', expected Struct type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& doris_struct = assert_cast(*data_column); + const auto* doris_struct_type = assert_cast(remove_nullable(type).get()); + + int64_t not_missing_column_id = -1; + size_t not_missing_orig_column_size = 0; + std::vector missing_column_idxs {}; + std::vector skip_reading_column_idxs {}; + std::vector> reference_parent_shape; + + auto parent_shape = [this](const ColumnReader& reader) { + std::vector> shape; + const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } + shape.reserve(rep_levels.size()); + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { + // Deeper repetitions belong to a nested child; only starts visible at this STRUCT's + // boundary and the optional parent's presence determine how siblings are paired. + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); + } + } + return shape; + }; + + _read_column_names.clear(); + + for (size_t i = 0; i < doris_struct.tuple_size(); ++i) { + ColumnPtr& doris_field = doris_struct.get_column_ptr(i); + auto& doris_type = doris_struct_type->get_element(i); + auto& doris_name = doris_struct_type->get_element_name(i); + if (!root_node->has_child(doris_name)) { + missing_column_idxs.push_back(i); + VLOG_DEBUG << "[ParquetReader] Missing column in schema: column_idx[" << i + << "], doris_name: " << doris_name << " (column not exists in root node)"; + continue; + } + auto file_name = root_node->file_child_name(doris_name); + + // Check if this is a SkipReadingReader - we should skip it when choosing reference column + // because SkipReadingReader doesn't know the actual data size in nested context + bool is_skip_reader = + dynamic_cast(_child_readers[file_name].get()) != nullptr; + + if (is_skip_reader) { + // Store SkipReadingReader columns to fill them later based on reference column size + skip_reading_column_idxs.push_back(i); + continue; + } + + // Only add non-SkipReadingReader columns to _read_column_names + // This ensures get_rep_level() and get_def_level() return valid levels + _read_column_names.emplace_back(file_name); + + size_t field_rows = 0; + bool field_eof = false; + if (not_missing_column_id == -1) { + not_missing_column_id = i; + not_missing_orig_column_size = doris_field->size(); + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, batch_size, + &field_rows, &field_eof, is_dict_filter)); + *read_rows = field_rows; + *eof = field_eof; + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } + reference_parent_shape = parent_shape(*_child_readers[file_name]); + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column + * (since `_read_nested_column` is not called repeatedly). + * + * It is worth mentioning that, theoretically, any sub-column can be chosen to fill the null_map, + * and selecting the shortest one will offer better performance + */ + } else { + while (field_rows < *read_rows && !field_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, + *read_rows - field_rows, &loop_rows, &field_eof, is_dict_filter)); + field_rows += loop_rows; + } + if (UNLIKELY(*read_rows != field_rows)) { + // STRUCT siblings must advance the same logical rows before any result is exposed. + return Status::Corruption("Parquet struct child '{}' returned {} rows, expected {}", + file_name, field_rows, *read_rows); + } + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } + if (UNLIKELY(parent_shape(*_child_readers[file_name]) != reference_parent_shape)) { + return Status::Corruption( + "Parquet struct child '{}' has a different repeated-parent shape", + file_name); + } + // DCHECK_EQ(*eof, field_eof); + } + } + + int64_t missing_column_sz = -1; + + if (not_missing_column_id == -1) { + // All queried columns are missing in the file (e.g., all added after schema change) + // We need to pick a column from _field_schema children that exists in the file for RL/DL reference + std::string reference_file_column_name; + std::unique_ptr* reference_reader = nullptr; + + for (const auto& child : _field_schema->children) { + auto it = _child_readers.find(child.name); + if (it != _child_readers.end()) { + // Skip SkipReadingReader as they don't have valid RL/DL + bool is_skip_reader = dynamic_cast(it->second.get()) != nullptr; + if (!is_skip_reader) { + reference_file_column_name = child.name; + reference_reader = &(it->second); + break; + } + } + } + + if (reference_reader != nullptr) { + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR( + (*reference_reader) + ->read_column_levels(filter_map, batch_size, &field_rows, &field_eof)); + + *read_rows = field_rows; + *eof = field_eof; + _read_column_names.emplace_back(reference_file_column_name); + missing_column_sz = 0; + const auto& rep_levels = (*reference_reader)->get_rep_level(); + const auto& def_levels = (*reference_reader)->get_def_level(); + DORIS_CHECK_EQ(rep_levels.size(), def_levels.size()); + for (size_t level_index = 0; level_index < def_levels.size(); ++level_index) { + if (def_levels[level_index] >= _field_schema->repeated_parent_def_level && + rep_levels[level_index] <= _field_schema->repetition_level) { + ++missing_column_sz; + } + } + } else { + return Status::Corruption( + "Cannot read struct '{}': all queried columns are missing and no reference " + "column found in file", + _field_schema->name); + } + } + + // This missing_column_sz is not *read_rows. Because read_rows returns the number of rows. + // For example: suppose we have a column array>, + // where b is a newly added column, that is, a missing column. + // There are two rows of data in this column, + // [{1,null},{2,null},{3,null}] + // [{4,null},{5,null}] + // When you first read subcolumn a, you read 5 data items and the value of *read_rows is 2. + // You should insert 5 records into subcolumn b instead of 2. + if (missing_column_sz == -1) { + missing_column_sz = doris_struct.get_column(not_missing_column_id).size() - + not_missing_orig_column_size; + } + + // Fill SkipReadingReader columns with the correct amount of data based on reference column + // Let SkipReadingReader handle the data filling through its read_column_data method + for (auto idx : skip_reading_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = const_cast(doris_struct_type->get_element(idx)); + auto& doris_name = const_cast(doris_struct_type->get_element_name(idx)); + auto file_name = root_node->file_child_name(doris_name); + + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, + missing_column_sz, &field_rows, &field_eof, is_dict_filter, missing_column_sz)); + } + + // Fill truly missing columns (not in root_node) with null or default value + for (auto idx : missing_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = doris_struct_type->get_element(idx); + DCHECK(doris_type->is_nullable()); + doris_field = IColumn::mutate(std::move(doris_field)); + auto mutable_column = doris_field->assert_mutable(); + auto* nullable_column = static_cast(mutable_column.get()); + nullable_column->insert_many_defaults(missing_column_sz); + } + + if (null_map_ptr != nullptr) { + fill_struct_null_map(_field_schema, *null_map_ptr, this->get_rep_level(), + this->get_def_level()); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h new file mode 100644 index 00000000000000..a4f0a56c8efff6 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -0,0 +1,654 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/parquet/native_schema_node.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/common.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "storage/segment/row_ranges.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris::io { +struct IOContext; +} // namespace doris::io + +namespace doris::format::parquet::native { +using ::doris::ColumnString; +using segment_v2::RowRange; +using segment_v2::RowRanges; + +#ifdef BE_TEST +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context); +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode); +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row); +#endif + +class ColumnReader { +public: + struct ColumnStatistics { + ColumnStatistics() + : page_index_read_calls(0), + decompress_time(0), + decompress_cnt(0), + decode_header_time(0), + decode_value_time(0), + materialization_time(0), + hybrid_selection_batches(0), + hybrid_selection_ranges(0), + hybrid_selection_null_fallback_batches(0), + decode_dict_time(0), + decode_level_time(0), + decode_null_map_time(0), + skip_page_header_num(0), + parse_page_header_num(0), + read_page_header_time(0), + page_read_counter(0), + page_cache_write_counter(0), + page_cache_compressed_write_counter(0), + page_cache_decompressed_write_counter(0), + page_cache_hit_counter(0), + page_cache_missing_counter(0), + page_cache_compressed_hit_counter(0), + page_cache_decompressed_hit_counter(0) {} + + ColumnStatistics(ColumnChunkReaderStatistics& cs, int64_t null_map_time) + : page_index_read_calls(0), + decompress_time(cs.decompress_time), + decompress_cnt(cs.decompress_cnt), + decode_header_time(cs.decode_header_time), + decode_value_time(cs.decode_value_time), + materialization_time(cs.materialization_time), + hybrid_selection_batches(cs.hybrid_selection_batches), + hybrid_selection_ranges(cs.hybrid_selection_ranges), + hybrid_selection_null_fallback_batches(cs.hybrid_selection_null_fallback_batches), + decode_dict_time(cs.decode_dict_time), + decode_level_time(cs.decode_level_time), + decode_null_map_time(null_map_time), + skip_page_header_num(cs.skip_page_header_num), + parse_page_header_num(cs.parse_page_header_num), + read_page_header_time(cs.read_page_header_time), + page_read_counter(cs.page_read_counter), + page_cache_write_counter(cs.page_cache_write_counter), + page_cache_compressed_write_counter(cs.page_cache_compressed_write_counter), + page_cache_decompressed_write_counter(cs.page_cache_decompressed_write_counter), + page_cache_hit_counter(cs.page_cache_hit_counter), + page_cache_missing_counter(cs.page_cache_missing_counter), + page_cache_compressed_hit_counter(cs.page_cache_compressed_hit_counter), + page_cache_decompressed_hit_counter(cs.page_cache_decompressed_hit_counter), + leaf_page_read_counters {cs.data_page_read_counter} {} + + int64_t page_index_read_calls; + int64_t decompress_time; + int64_t decompress_cnt; + int64_t decode_header_time; + int64_t decode_value_time; + int64_t materialization_time; + int64_t hybrid_selection_batches; + int64_t hybrid_selection_ranges; + int64_t hybrid_selection_null_fallback_batches; + int64_t decode_dict_time; + int64_t decode_level_time; + int64_t decode_null_map_time; + int64_t skip_page_header_num; + int64_t parse_page_header_num; + int64_t read_page_header_time; + int64_t page_read_counter; + int64_t page_cache_write_counter; + int64_t page_cache_compressed_write_counter; + int64_t page_cache_decompressed_write_counter; + int64_t page_cache_hit_counter; + int64_t page_cache_missing_counter; + int64_t page_cache_compressed_hit_counter; + int64_t page_cache_decompressed_hit_counter; + // Preserve per-leaf identity when complex readers aggregate their other counters. + std::vector leaf_page_read_counters; + + void merge(ColumnStatistics& col_statistics) { + page_index_read_calls += col_statistics.page_index_read_calls; + decompress_time += col_statistics.decompress_time; + decompress_cnt += col_statistics.decompress_cnt; + decode_header_time += col_statistics.decode_header_time; + decode_value_time += col_statistics.decode_value_time; + materialization_time += col_statistics.materialization_time; + hybrid_selection_batches += col_statistics.hybrid_selection_batches; + hybrid_selection_ranges += col_statistics.hybrid_selection_ranges; + hybrid_selection_null_fallback_batches += + col_statistics.hybrid_selection_null_fallback_batches; + decode_dict_time += col_statistics.decode_dict_time; + decode_level_time += col_statistics.decode_level_time; + decode_null_map_time += col_statistics.decode_null_map_time; + skip_page_header_num += col_statistics.skip_page_header_num; + parse_page_header_num += col_statistics.parse_page_header_num; + read_page_header_time += col_statistics.read_page_header_time; + page_read_counter += col_statistics.page_read_counter; + leaf_page_read_counters.insert(leaf_page_read_counters.end(), + col_statistics.leaf_page_read_counters.begin(), + col_statistics.leaf_page_read_counters.end()); + page_cache_write_counter += col_statistics.page_cache_write_counter; + page_cache_compressed_write_counter += + col_statistics.page_cache_compressed_write_counter; + page_cache_decompressed_write_counter += + col_statistics.page_cache_decompressed_write_counter; + page_cache_hit_counter += col_statistics.page_cache_hit_counter; + page_cache_missing_counter += col_statistics.page_cache_missing_counter; + page_cache_compressed_hit_counter += col_statistics.page_cache_compressed_hit_counter; + page_cache_decompressed_hit_counter += + col_statistics.page_cache_decompressed_hit_counter; + } + }; + + ColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : _row_ranges(row_ranges), _total_rows(total_rows), _ctz(ctz), _io_ctx(io_ctx) {} + virtual ~ColumnReader() = default; + virtual Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter, + int64_t real_column_size = -1) = 0; + + // Evaluate a predicate-only scalar directly from fixed-width PLAIN page bytes. The default is + // a non-consuming fallback for nested and synthetic readers. + virtual Status read_plain_filter(const VExprSPtrs&, int, FilterMap&, size_t, + IColumn::Filter* row_filter, size_t* read_rows, bool* eof, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + return Status::OK(); + } + + // Consume a nested batch while retaining only definition/repetition levels. This is used when + // schema evolution makes every projected STRUCT child synthetic: the parent still needs one + // physical leaf's shape, but decoding that leaf's strings or other payload would be wasted. + virtual Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) = 0; + + virtual Result convert_dict_column_to_string_column( + const ColumnInt32* dict_column, const DataTypePtr& target_type) { + throw Exception( + Status::FatalError("Method convert_dict_column_to_string_column is not supported")); + } + virtual Result dictionary_values(const DataTypePtr& target_type) { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); + } + + static Status create(io::FileReaderSPtr file, NativeFieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + const std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection = false, + const std::set& column_ids = {}, + const std::set& filter_column_ids = {}, + const std::string& page_cache_file_key = {}, + const ParquetReaderCompat& compat = {}, bool enable_strict_mode = false); + virtual const std::vector& get_rep_level() const = 0; + virtual const std::vector& get_def_level() const = 0; + virtual ColumnStatistics column_statistics() = 0; + virtual void close() = 0; + + // A repeated parent can expand one logical-row batch into millions of leaf values. Keep + // ordinary batch scratch for reuse, but let the top-level adapter release exceptional + // high-water allocations after every parent offset/null-map consumer has finished. + virtual void release_batch_scratch(size_t max_retained_bytes) = 0; + + virtual void reset_filter_map_index() = 0; + + NativeFieldSchema* get_field_schema() const { return _field_schema; } + void set_column_in_nested() { _in_nested = true; } + +protected: + void _generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const; + + NativeFieldSchema* _field_schema = nullptr; + const RowRanges& _row_ranges; + size_t _total_rows = 0; + const cctz::time_zone* _ctz = nullptr; + io::IOContext* _io_ctx = nullptr; + int64_t _current_row_index = 0; + int64_t _decode_null_map_time = 0; + + size_t _filter_map_index = 0; + std::set _filter_column_ids; + + // _in_nested: column in struct/map/array + // IN_COLLECTION : column in map/array + bool _in_nested = false; +}; + +template +class ScalarColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ScalarColumnReader) +public: + ScalarColumnReader(const RowRanges& row_ranges, size_t total_rows, + const tparquet::ColumnChunk& chunk_meta, + const tparquet::OffsetIndex* offset_index, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx), + _chunk_meta(chunk_meta), + _offset_index(offset_index) {} + ~ScalarColumnReader() override { close(); } + Status init(io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, + RuntimeState* state, const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_plain_filter(const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, + size_t batch_size, IColumn::Filter* row_filter, size_t* read_rows, + bool* eof, bool* used_filter) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + Result convert_dict_column_to_string_column( + const ColumnInt32* dict_column, const DataTypePtr& target_type) override; + Result dictionary_values(const DataTypePtr& target_type) override; + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { + return ColumnStatistics(_chunk_reader->chunk_statistics(), _decode_null_map_time); + } + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override; + +#ifdef BE_TEST + void reserve_batch_scratch_for_test(size_t elements); + size_t retained_batch_scratch_bytes_for_test() const; +#endif + + void reset_filter_map_index() override { + _filter_map_index = 0; // nested + } + +private: + tparquet::ColumnChunk _chunk_meta; + const tparquet::OffsetIndex* _offset_index = nullptr; + std::unique_ptr _stream_reader; + std::unique_ptr> _chunk_reader; + // rep def levels buffer. + std::vector _rep_levels; + std::vector _def_levels; + + size_t _current_range_idx = 0; + + Status gen_nested_null_map(size_t level_start_idx, size_t level_end_idx, + std::vector& null_map, + std::unordered_set& ancestor_null_indices) { + size_t has_read = level_start_idx; + null_map.emplace_back(0); + bool prev_is_null = false; + + while (has_read < level_end_idx) { + level_t def_level = _def_levels[has_read++]; + size_t loop_read = 1; + while (has_read < _def_levels.size() && _def_levels[has_read] == def_level) { + has_read++; + loop_read++; + } + + if (def_level < _field_schema->repeated_parent_def_level) { + for (size_t i = 0; i < loop_read; i++) { + ancestor_null_indices.insert(has_read - level_start_idx - loop_read + i); + } + continue; + } + + bool is_null = def_level < _field_schema->definition_level; + + if (prev_is_null == is_null && (USHRT_MAX - null_map.back() >= loop_read)) { + null_map.back() += loop_read; + } else { + if (!(prev_is_null ^ is_null)) { + null_map.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + null_map.emplace_back(USHRT_MAX); + null_map.emplace_back(0); + remaining -= USHRT_MAX; + } + null_map.emplace_back((u_short)remaining); + prev_is_null = is_null; + } + } + return Status::OK(); + } + + Status gen_filter_map(FilterMap& filter_map, size_t filter_loc, size_t level_start_idx, + size_t level_end_idx, std::vector& nested_filter_map_data, + FilterMap* nested_filter_map) { + DORIS_CHECK(nested_filter_map != nullptr); + nested_filter_map_data.resize(level_end_idx - level_start_idx); + for (size_t idx = level_start_idx; idx < level_end_idx; idx++) { + if (idx != level_start_idx && _rep_levels[idx] == 0) { + filter_loc++; + } + nested_filter_map_data[idx - level_start_idx] = + filter_map.filter_map_data()[filter_loc]; + } + + return nested_filter_map->init(nested_filter_map_data.data(), nested_filter_map_data.size(), + false); + } + + DataTypeSerDeSPtr _serde; + const IDataType* _serde_type = nullptr; + ParquetDecodeContext _decode_context; + ParquetMaterializationState _materialization_state; + bool _dictionary_index_only = false; + // Normal-size batch scratch is retained by the persistent leaf reader. Oversized allocations + // are released only after the top-level parent has consumed this leaf's level plan. + std::vector _null_run_lengths; + std::unordered_set _ancestor_null_indices; + std::vector _nested_filter_map_data; + NullMap _plain_predicate_nulls; + IColumn::Filter _plain_predicate_matches; + FilterMap _nested_filter_map; + ColumnSelectVector _select_vector; + uint8_t _oversized_scratch_idle_batches = 0; + + size_t retained_batch_scratch_bytes() const; + size_t active_batch_scratch_bytes() const; + + Status _skip_values(size_t num_values); + Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, bool is_dict_filter); + Status _read_plain_filter_values(size_t num_values, const VExprSPtrs& conjuncts, int column_id, + FilterMap& filter_map, IColumn::Filter* row_filter); + Status _read_nested_column(ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter); + Status _try_load_dict_page(bool* loaded, bool* has_dict); +}; + +class ArrayColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ArrayColumnReader) +public: + ArrayColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~ArrayColumnReader() override { close(); } + Status init(std::unique_ptr element_reader, NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override { + return _element_reader->read_column_levels(filter_map, batch_size, read_rows, eof); + } + const std::vector& get_rep_level() const override { + return _element_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _element_reader->get_def_level(); + } + ColumnStatistics column_statistics() override { return _element_reader->column_statistics(); } + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + _element_reader->release_batch_scratch(max_retained_bytes); + } + + void reset_filter_map_index() override { _element_reader->reset_filter_map_index(); } + +private: + std::unique_ptr _element_reader; +}; + +class MapColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(MapColumnReader) +public: + MapColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~MapColumnReader() override { close(); } + + Status init(std::unique_ptr key_reader, + std::unique_ptr value_reader, NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + return _key_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _key_reader->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics kst = _key_reader->column_statistics(); + ColumnStatistics vst = _value_reader->column_statistics(); + kst.merge(vst); + return kst; + } + + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + _key_reader->release_batch_scratch(max_retained_bytes); + _value_reader->release_batch_scratch(max_retained_bytes); + } + + void reset_filter_map_index() override { + _key_reader->reset_filter_map_index(); + _value_reader->reset_filter_map_index(); + } + +private: + std::unique_ptr _key_reader; + std::unique_ptr _value_reader; +}; + +class StructColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(StructColumnReader) +public: + StructColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~StructColumnReader() override { close(); } + + Status init(std::unordered_map>&& child_readers, + NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + if (!_read_column_names.empty()) { + // can't use _child_readers[*_read_column_names.begin()] + // because the operator[] of std::unordered_map is not const :( + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column, + * that is `_read_column_names.front()`. + */ + return _child_readers.find(_read_column_names.front())->second->get_rep_level(); + } + return _child_readers.begin()->second->get_rep_level(); + } + + const std::vector& get_def_level() const override { + if (!_read_column_names.empty()) { + return _child_readers.find(_read_column_names.front())->second->get_def_level(); + } + return _child_readers.begin()->second->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics st; + for (const auto& column_name : _read_column_names) { + auto reader = _child_readers.find(column_name); + if (reader != _child_readers.end()) { + ColumnStatistics cst = reader->second->column_statistics(); + st.merge(cst); + } + } + return st; + } + + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + for (const auto& reader : _child_readers) { + reader.second->release_batch_scratch(max_retained_bytes); + } + } + + void reset_filter_map_index() override { + for (const auto& reader : _child_readers) { + reader.second->reset_filter_map_index(); + } + } + +private: + std::unordered_map> _child_readers; + std::vector _read_column_names; + //Need to use vector instead of set,see `get_rep_level()` for the reason. +}; + +// A special reader that skips actual reading but provides empty data with correct structure +// This is used when a column is not needed but its structure is required (e.g., for map keys) +class SkipReadingReader : public ColumnReader { +public: + SkipReadingReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx, NativeFieldSchema* field_schema) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) { + _field_schema = field_schema; // Use inherited member from base class + VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: " + << _field_schema->name; + } + + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override { + VLOG_DEBUG << "[ParquetReader] SkipReadingReader::read_column_data for field: " + << _field_schema->name << ", batch_size: " << batch_size; + DCHECK(real_column_size >= 0); // real_column_size for filtered column size. + + // Simulate reading without actually reading data + // Fill with default/null values based on column type + doris_column = IColumn::mutate(std::move(doris_column)); + MutableColumnPtr data_column = doris_column->assert_mutable(); + + if (real_column_size > 0) { + if (is_column_nullable(*doris_column)) { + auto* nullable_column = static_cast(data_column.get()); + nullable_column->insert_many_defaults(real_column_size); + } else { + // For non-nullable columns, insert appropriate default values + for (size_t i = 0; i < real_column_size; ++i) { + data_column->insert_default(); + } + } + } + + *read_rows = batch_size; // Indicate we "read" batch_size rows + *eof = false; // We can always provide more empty data + + VLOG_DEBUG << "[ParquetReader] SkipReadingReader generated " << batch_size + << " default values for field: " << _field_schema->name; + + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t*, bool*) override { + return Status::InternalError("Skip reader cannot provide Parquet levels for field {}", + _field_schema->name); + } + + static std::unique_ptr create_unique(const RowRanges& row_ranges, + size_t total_rows, cctz::time_zone* ctz, + io::IOContext* io_ctx, + NativeFieldSchema* field_schema) { + return std::make_unique(row_ranges, total_rows, ctz, io_ctx, + field_schema); + } + + // These methods should not be called for SkipReadingReader + // If they are called, it indicates a logic error in the code + const std::vector& get_rep_level() const override { + LOG(FATAL) << "get_rep_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + const std::vector& get_def_level() const override { + LOG(FATAL) << "get_def_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + // Implement required pure virtual methods from base class + ColumnStatistics column_statistics() override { + return ColumnStatistics(); // Return empty statistics + } + + void close() override { + // Nothing to close for skip reading + } + + void release_batch_scratch(size_t) override {} + + void reset_filter_map_index() override { _filter_map_index = 0; } +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/common.cpp b/be/src/format_v2/parquet/reader/native/common.cpp new file mode 100644 index 00000000000000..f048d2b878ebee --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.cpp @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/common.h" + +#include + +#include "core/types.h" +#include "util/simd/bits.h" + +namespace doris::format::parquet::native { + +Status FilterMap::init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all) { + _filter_all = filter_all; + _filter_map_data = filter_map_data; + _filter_map_size = filter_map_size; + if (filter_all) { + _has_filter = true; + _filter_ratio = 1; + } else if (filter_map_data == nullptr) { + _has_filter = false; + _filter_ratio = 0; + } else { + const size_t filter_count = simd::count_zero_num( + reinterpret_cast(filter_map_data), filter_map_size); + if (filter_count == filter_map_size) { + _has_filter = true; + _filter_all = true; + _filter_ratio = 1; + } else if (filter_count > 0 && filter_map_size > 0) { + _has_filter = true; + _filter_ratio = static_cast(filter_count) / filter_map_size; + } else { + _has_filter = false; + _filter_ratio = 0; + } + } + return Status::OK(); +} + +bool FilterMap::can_filter_all(size_t remaining_num_values, size_t filter_map_index) { + if (!_has_filter) { + return false; + } + if (_filter_all) { + DCHECK_LE(remaining_num_values + filter_map_index, _filter_map_size); + return true; + } + if (remaining_num_values + filter_map_index > _filter_map_size) { + return false; + } + return simd::count_zero_num( + reinterpret_cast(_filter_map_data + filter_map_index), + remaining_num_values) == remaining_num_values; +} + +Status FilterMap::generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index) const { + if (!has_filter() || filter_all()) { + return Status::InternalError( + "Native FilterMap requires a partial filter: has_filter={}, filter_all={}", + has_filter(), filter_all()); + } + if (rep_levels.empty()) { + return Status::OK(); + } + + nested_filter_map_data.resize(rep_levels.size()); + size_t current_row = current_row_ptr != nullptr ? *current_row_ptr : 0; + for (size_t i = start_index; i < rep_levels.size(); ++i) { + if (i != start_index && rep_levels[i] == 0) { + ++current_row; + if (current_row >= _filter_map_size) { + return Status::InvalidArgument("Nested filter row {} exceeds filter map size {}", + current_row, _filter_map_size); + } + } + nested_filter_map_data[i] = _filter_map_data[current_row]; + } + if (current_row_ptr != nullptr) { + *current_row_ptr = current_row; + } + + auto new_filter = std::make_unique(); + RETURN_IF_ERROR( + new_filter->init(nested_filter_map_data.data(), nested_filter_map_data.size(), false)); + *nested_filter_map = std::move(new_filter); + return Status::OK(); +} + +Status ColumnSelectVector::init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices) { + _num_values = num_values; + _num_nulls = 0; + _read_index = 0; + size_t map_index = 0; + bool is_null = false; + _has_filter = filter_map->has_filter(); + + if (_has_filter) { + _data_map.resize(num_values); + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_NULL; + } + } else { + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_CONTENT; + } + } + is_null = !is_null; + } + + size_t num_read = 0; + size_t source_index = 0; + size_t valid_count = 0; + while (valid_count < num_values) { + DCHECK_LT(filter_map_index + source_index, filter_map->filter_map_size()); + if (skipped_indices != nullptr && + skipped_indices->contains(filter_map_index + source_index)) { + ++source_index; + continue; + } + if (filter_map->filter_map_data()[filter_map_index + source_index] != 0) { + _data_map[valid_count] = + _data_map[valid_count] == FILTERED_NULL ? NULL_DATA : CONTENT; + ++num_read; + } + ++valid_count; + ++source_index; + } + _num_filtered = num_values - num_read; + + if (null_map != nullptr && num_read > 0) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_read); + if (_num_nulls == 0) { + memset(null_map->data() + null_map_index, 0, num_read); + } else if (_num_nulls == num_values) { + memset(null_map->data() + null_map_index, 1, num_read); + } else { + for (size_t i = 0; i < num_values; ++i) { + if (_data_map[i] == CONTENT) { + (*null_map)[null_map_index++] = static_cast(false); + } else if (_data_map[i] == NULL_DATA) { + (*null_map)[null_map_index++] = static_cast(true); + } + } + } + } + } else { + _num_filtered = 0; + _run_length_null_map = &run_length_null_map; + if (null_map != nullptr) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_values); + for (const auto run_length : run_length_null_map) { + memset(null_map->data() + null_map_index, is_null ? 1 : 0, run_length); + null_map_index += run_length; + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } else { + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/common.h b/be/src/format_v2/parquet/reader/native/common.h new file mode 100644 index 00000000000000..4ad204da2ff30d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_nullable.h" + +namespace doris::format::parquet::native { + +// These cursor types belong to the native v2 decoder so its execution path never reaches into +// the legacy Parquet reader for filtering or level materialization. +using level_t = int16_t; + +class FilterMap { +public: + Status init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all); + + Status generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index = 0) const; + + const uint8_t* filter_map_data() const { return _filter_map_data; } + size_t filter_map_size() const { return _filter_map_size; } + bool has_filter() const { return _has_filter; } + bool filter_all() const { return _filter_all; } + double filter_ratio() const { return _has_filter ? _filter_ratio : 0; } + + bool can_filter_all(size_t remaining_num_values, size_t filter_map_index); + +private: + bool _filter_all = false; + bool _has_filter = false; + const uint8_t* _filter_map_data = nullptr; + size_t _filter_map_size = 0; + double _filter_ratio = 0; +}; + +class ColumnSelectVector { +public: + enum DataReadType : uint8_t { CONTENT = 0, NULL_DATA, FILTERED_CONTENT, FILTERED_NULL }; + + Status init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices = nullptr); + + size_t num_values() const { return _num_values; } + size_t num_nulls() const { return _num_nulls; } + size_t num_filtered() const { return _num_filtered; } + bool has_filter() const { return _has_filter; } + + template + size_t get_next_run(DataReadType* data_read_type) { + DCHECK_EQ(_has_filter, has_filter); + if constexpr (has_filter) { + if (_read_index == _num_values) { + return 0; + } + const DataReadType type = _data_map[_read_index++]; + size_t run_length = 1; + while (_read_index < _num_values && _data_map[_read_index] == type) { + ++run_length; + ++_read_index; + } + *data_read_type = type; + return run_length; + } + + size_t run_length = 0; + while (run_length == 0) { + if (_read_index == _run_length_null_map->size()) { + return 0; + } + *data_read_type = _read_index % 2 == 0 ? CONTENT : NULL_DATA; + run_length = (*_run_length_null_map)[_read_index++]; + } + return run_length; + } + +private: + std::vector _data_map; + const std::vector* _run_length_null_map = nullptr; + bool _has_filter = false; + size_t _num_values = 0; + size_t _num_nulls = 0; + size_t _num_filtered = 0; + size_t _read_index = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/decoder.cpp b/be/src/format_v2/parquet/reader/native/decoder.cpp new file mode 100644 index 00000000000000..5f6a978317bde7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.cpp @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/decoder.h" + +#include +#include + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" +#include "format_v2/parquet/reader/native/fix_length_dict_decoder.hpp" +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +namespace doris::format::parquet::native { +namespace { +Status unsupported_type(tparquet::Type::type type, tparquet::Encoding::type encoding) { + return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", + tparquet::to_string(type), tparquet::to_string(encoding)); +} + +Status create_plain_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::PLAIN); + } +} + +Status create_dictionary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + return Status::InternalError("Boolean type cannot have a dictionary page"); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT96: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FLOAT: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::DOUBLE: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique>(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::RLE_DICTIONARY); + } +} + +Status create_delta_binary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + default: + return Status::InternalError("DELTA_BINARY_PACKED only supports INT32 and INT64"); + } +} + +Status create_byte_stream_split_decoder(tparquet::Type::type type, + std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::BYTE_STREAM_SPLIT); + } +} +} // namespace + +Status Decoder::get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder) { + switch (encoding) { + case tparquet::Encoding::PLAIN: + return create_plain_decoder(type, decoder); + case tparquet::Encoding::RLE_DICTIONARY: + return create_dictionary_decoder(type, decoder); + case tparquet::Encoding::RLE: + if (type != tparquet::Type::BOOLEAN) { + return unsupported_type(type, encoding); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::DELTA_BINARY_PACKED: + return create_delta_binary_decoder(type, decoder); + case tparquet::Encoding::DELTA_BYTE_ARRAY: + if (type != tparquet::Type::BYTE_ARRAY && type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) { + return Status::InternalError( + "DELTA_BYTE_ARRAY only supports BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY."); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + if (type != tparquet::Type::BYTE_ARRAY) { + return Status::InternalError("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY."); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::BYTE_STREAM_SPLIT: + return create_byte_stream_split_decoder(type, decoder); + default: + return Status::InternalError("Unsupported encoding {}(type={}) in parquet decoder", + tparquet::to_string(encoding), tparquet::to_string(type)); + } +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h new file mode 100644 index 00000000000000..379176d5e459e0 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __AVX2__ +#include +#endif + +#include "common/status.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/types.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { + +inline bool dictionary_indices_in_bounds(const uint32_t* indices, size_t count, + size_t dictionary_size) { + if (count == 0) { + return true; + } + if (dictionary_size == 0) { + return false; + } + uint32_t max_index = 0; + size_t row = 0; +#ifdef __AVX2__ + __m256i vector_max = _mm256_setzero_si256(); + for (; row + 8 <= count; row += 8) { + const auto values = _mm256_loadu_si256(reinterpret_cast(indices + row)); + vector_max = _mm256_max_epu32(vector_max, values); + } + alignas(32) uint32_t lanes[8]; + _mm256_store_si256(reinterpret_cast<__m256i*>(lanes), vector_max); + for (const auto lane : lanes) { + max_index = std::max(max_index, lane); + } +#endif + for (; row < count; ++row) { + max_index = std::max(max_index, indices[row]); + } + return static_cast(max_index) < dictionary_size; +} + +class Decoder : public ParquetDecodeSource { +public: + Decoder() = default; + virtual ~Decoder() = default; + + static Status get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder); + + // The type with fix length + void set_type_length(int32_t type_length) { _type_length = type_length; } + + // Set the data to be decoded + virtual Status set_data(Slice* data) { + _data = data; + _offset = 0; + return Status::OK(); + } + + // Page headers provide an upper bound before encoding-specific headers are trusted. + virtual void set_expected_values(size_t expected_values) { _expected_values = expected_values; } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + return Status::NotSupported("Fixed values are not supported by this Parquet decoder"); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + return Status::NotSupported("Binary values are not supported by this Parquet decoder"); + } + + Status skip_values(size_t num_values) override = 0; + + virtual void release_scratch(size_t max_retained_bytes) {} + virtual size_t retained_scratch_bytes() const { return 0; } + virtual size_t active_scratch_bytes() const { return 0; } + + virtual Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + return Status::NotSupported("set_dict is not supported"); + } + +protected: + template + static void release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + if (values->capacity() * sizeof(T) > max_retained_bytes) { + std::vector().swap(*values); + } + } + + int32_t _type_length = -1; + Slice* _data = nullptr; + // Page offsets are host-sized so checked advances cannot wrap at the 4 GiB boundary. + size_t _offset = 0; + size_t _expected_values = std::numeric_limits::max(); +}; + +class BaseDictDecoder : public Decoder { +public: + BaseDictDecoder() = default; + ~BaseDictDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + if (UNLIKELY(data == nullptr || data->size == 0)) { + return Status::Corruption("Parquet dictionary index stream is empty"); + } + _data = data; + _offset = 0; + uint8_t bit_width = *data->data; + // Dictionary indices are uint32_t; wider external widths make repeated runs overwrite the + // decoder's four-byte state before any dictionary-bound check can run. + if (UNLIKELY(bit_width > 32)) { + return Status::Corruption("Parquet dictionary index bit width {} exceeds 32", + bit_width); + } + _index_batch_decoder = std::make_unique>( + reinterpret_cast(data->data) + 1, static_cast(data->size) - 1, + bit_width); + return Status::OK(); + } + + bool has_dictionary() const override { return true; } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + indices->resize(num_values); + const auto decoded = + _index_batch_decoder->GetBatch(indices->data(), cast_set(num_values)); + if (UNLIKELY(decoded != num_values)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t num_dictionary_values = dictionary_size(); + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data(), num_values, + num_dictionary_values))) { + // The SIMD common path only computes a bound; recover the exact corrupt row for the + // diagnostic after the batch has already been proven invalid. + for (size_t row = 0; row < num_values; ++row) { + if ((*indices)[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[row], row, num_dictionary_values); + } + } + return Status::OK(); + } + + Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + const size_t num_dictionary_values = dictionary_size(); + indices->resize(selection.selected_values); + size_t cursor = 0; + size_t output = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + const auto decoded = _index_batch_decoder->GetBatch(indices->data() + output, + cast_set(range.count)); + if (UNLIKELY(decoded != range.count)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data() + output, range.count, + num_dictionary_values))) { + for (size_t row = 0; row < range.count; ++row) { + if ((*indices)[output + row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[output + row], range.first + row, num_dictionary_values); + } + } + output += range.count; + cursor = range.first + range.count; + } + DORIS_CHECK(cursor <= selection.total_values); + RETURN_IF_ERROR(_decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values)); + DORIS_CHECK_EQ(output, selection.selected_values); + return Status::OK(); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + return _decode_dictionary_values(num_values, 0, dictionary_size(), consumer); + } + + Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) override { + const size_t num_dictionary_values = dictionary_size(); + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + RETURN_IF_ERROR(_decode_dictionary_values(range.count, range.first, + num_dictionary_values, consumer)); + cursor = range.first + range.count; + } + DORIS_CHECK(cursor <= selection.total_values); + return _decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values); + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_skip_indices, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _skip_indices.capacity() * sizeof(uint32_t); + } + size_t active_scratch_bytes() const override { return _skip_indices.size() * sizeof(uint32_t); } + +protected: + Status skip_values(size_t num_values) override { + return _decode_and_validate_skipped(num_values, 0, dictionary_size()); + } + + Status _decode_and_validate_skipped(size_t num_values, size_t row_offset, + size_t num_dictionary_values) { + constexpr size_t kSkipBatchSize = 4096; + // Skipped dictionary ids are still external input and must be bounds-checked, but keeping + // only one bounded gap buffer avoids the page-sized scratch used by sparse selections. + _skip_indices.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped_values = 0; + while (skipped_values < num_values) { + const size_t batch_size = std::min(num_values - skipped_values, kSkipBatchSize); + const auto skipped = _index_batch_decoder->GetBatch(_skip_indices.data(), + static_cast(batch_size)); + if (UNLIKELY(skipped != batch_size)) { + return Status::IOError( + "Can't skip enough Parquet dictionary indices at row {}: {} of {}", + row_offset + skipped_values, skipped, batch_size); + } + // Filter gaps may be huge RLE runs; validate them in bounded SIMD-sized batches. + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch_size, + num_dictionary_values))) { + for (size_t row = 0; row < batch_size; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at skipped row {} exceeds dictionary " + "size {}", + _skip_indices[row], row_offset + skipped_values + row, + num_dictionary_values); + } + } + skipped_values += batch_size; + } + return Status::OK(); + } + + Status _decode_dictionary_values(size_t num_values, size_t row_offset, + size_t num_dictionary_values, + ParquetDictionaryValueConsumer& consumer) { + constexpr size_t kLiteralBatchSize = 1024; + size_t decoded_values = 0; + while (decoded_values < num_values) { + const int32_t repeats = _index_batch_decoder->NextNumRepeats(); + if (repeats > 0) { + const size_t run = std::min(repeats, num_values - decoded_values); + const uint32_t index = + _index_batch_decoder->GetRepeatedValue(cast_set(run)); + if (UNLIKELY(static_cast(index) >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + index, row_offset + decoded_values, num_dictionary_values); + } + RETURN_IF_ERROR(consumer.consume_repeated(index, run)); + decoded_values += run; + continue; + } + + const int32_t literals = _index_batch_decoder->NextNumLiterals(); + if (UNLIKELY(literals == 0)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t batch = std::min({static_cast(literals), + num_values - decoded_values, kLiteralBatchSize}); + _skip_indices.resize(batch); + if (UNLIKELY(!_index_batch_decoder->GetLiteralValues(cast_set(batch), + _skip_indices.data()))) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch, + num_dictionary_values))) { + for (size_t row = 0; row < batch; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + _skip_indices[row], row_offset + decoded_values + row, + num_dictionary_values); + } + } + RETURN_IF_ERROR(consumer.consume_indices(_skip_indices.data(), batch)); + decoded_values += batch; + } + return Status::OK(); + } + + // For dictionary encoding + DorisUniqueBufferPtr _dict; + std::unique_ptr> _index_batch_decoder; + std::vector _skip_indices; + uint64_t _dictionary_generation = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp new file mode 100644 index 00000000000000..c94aead8caa6b6 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" + +namespace doris::format::parquet::native { +Status DeltaLengthByteArrayDecoder::_init_lengths() { + auto length_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_len_decoder.set_bit_reader(length_reader)); + const uint32_t num_lengths = _len_decoder.valid_values_count(); + + DeltaBitPackDecoder length_locator; + length_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(length_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector lengths(std::min(num_lengths, kLocateBatchSize)); + size_t located = 0; + int64_t payload_size = 0; + while (located < num_lengths) { + const size_t batch_size = std::min(num_lengths - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR( + length_locator.decode(lengths.data(), static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + for (size_t i = 0; i < batch_size; ++i) { + if (UNLIKELY(lengths[i] < 0) || + common::add_overflow(payload_size, static_cast(lengths[i]), + payload_size)) { + return Status::Corruption("Invalid Parquet delta byte-array length"); + } + } + located += batch_size; + } + // The locator leaves the shared reader at the first payload byte without retaining every + // length; the independent decoder supplies bounded batches during materialization or skip. + if (UNLIKELY(payload_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + payload_size, _bit_reader->bytes_left()); + } + _num_valid_values = num_lengths; + return Status::OK(); +} + +Status DeltaLengthByteArrayDecoder::_get_internal(Slice* buffer, int max_values, + int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + + int64_t data_size = 0; + _buffered_length.resize(max_values); + uint32_t lengths_decoded = 0; + RETURN_IF_ERROR(_len_decoder.decode(_buffered_length.data(), static_cast(max_values), + &lengths_decoded)); + if (UNLIKELY(lengths_decoded != max_values)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + const int32_t* length_ptr = _buffered_length.data(); + for (int i = 0; i < max_values; ++i) { + int32_t len = length_ptr[i]; + if (len < 0) [[unlikely]] { + return Status::InvalidArgument("Negative string delta length"); + } + buffer[i].size = len; + if (common::add_overflow(data_size, static_cast(len), data_size)) { + return Status::InvalidArgument("Excess expansion in DELTA_(LENGTH_)BYTE_ARRAY"); + } + } + // Every declared byte must exist in this page. Check before resize so a tiny malformed stream + // cannot reserve memory based only on attacker-controlled decoded lengths. + if (UNLIKELY(data_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + data_size, _bit_reader->bytes_left()); + } + _buffered_data.resize(data_size); + char* data_ptr = _buffered_data.data(); + for (int64_t j = 0; j < data_size; j++) { + if (!_bit_reader->GetValue(8, data_ptr + j)) { + return Status::IOError("Get length bytes EOF"); + } + } + + for (int i = 0; i < max_values; ++i) { + buffer[i].data = data_ptr; + data_ptr += buffer[i].size; + } + // this->num_values_ -= max_values; + _num_valid_values -= max_values; + *out_num_values = max_values; + return Status::OK(); +} + +Status DeltaByteArrayDecoder::_get_internal(Slice* buffer, int max_values, int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = max_values; + return Status::OK(); + } + + int suffix_read; + RETURN_IF_ERROR(_suffix_decoder.decode(buffer, max_values, &suffix_read)); + if (suffix_read != max_values) [[unlikely]] { + return Status::IOError("Read {}, expecting {} from suffix decoder", + std::to_string(suffix_read), std::to_string(max_values)); + } + + int64_t data_size = 0; + _buffered_prefix_length.resize(max_values); + uint32_t prefixes_decoded = 0; + RETURN_IF_ERROR(_prefix_len_decoder.decode( + _buffered_prefix_length.data(), static_cast(max_values), &prefixes_decoded)); + if (UNLIKELY(prefixes_decoded != max_values)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + const int32_t* prefix_len_ptr = _buffered_prefix_length.data(); + size_t preceding_value_size = _last_value.size(); + for (int i = 0; i < max_values; ++i) { + if (prefix_len_ptr[i] < 0) [[unlikely]] { + return Status::InvalidArgument("negative prefix length in DELTA_BYTE_ARRAY"); + } + const size_t prefix_size = static_cast(prefix_len_ptr[i]); + if (prefix_size > preceding_value_size) [[unlikely]] { + // Prefixes form a dependency chain, so validate each one before aggregate allocation. + return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); + } + size_t reconstructed_size = 0; + if (common::add_overflow(prefix_size, buffer[i].size, reconstructed_size) || + reconstructed_size > static_cast(std::numeric_limits::max()) || + common::add_overflow(data_size, static_cast(reconstructed_size), data_size)) + [[unlikely]] { + return Status::InvalidArgument("excess expansion in DELTA_BYTE_ARRAY"); + } + preceding_value_size = reconstructed_size; + } + _buffered_data.resize(data_size); + + std::string_view prefix {_last_value}; + + char* data_ptr = _buffered_data.data(); + for (int i = 0; i < max_values; ++i) { + memcpy(data_ptr, prefix.data(), prefix_len_ptr[i]); + // buffer[i] currently points to the string suffix + memcpy(data_ptr + prefix_len_ptr[i], buffer[i].data, buffer[i].size); + buffer[i].data = data_ptr; + buffer[i].size += prefix_len_ptr[i]; + data_ptr += buffer[i].size; + prefix = std::string_view {buffer[i].data, buffer[i].size}; + } + _num_valid_values -= max_values; + _last_value = std::string {prefix}; + + if (_num_valid_values == 0) { + _last_value_in_previous_page = _last_value; + } + *out_num_values = max_values; + return Status::OK(); +} +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h new file mode 100644 index 00000000000000..26e13119b8cdfb --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -0,0 +1,698 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "exec/common/arithmetic_overflow.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +namespace detail { + +inline Status checked_delta_padding_bits(uint32_t bit_width, uint32_t remaining_values, + int64_t* padding_bits) { + DORIS_CHECK(padding_bits != nullptr); + const uint64_t widened_bits = static_cast(bit_width) * remaining_values; + if (UNLIKELY(widened_bits > static_cast(std::numeric_limits::max()))) { + return Status::Corruption("Parquet delta miniblock padding is too large"); + } + *padding_bits = static_cast(widened_bits); + return Status::OK(); +} + +} // namespace detail + +class DeltaDecoder : public Decoder { +public: + DeltaDecoder() = default; + ~DeltaDecoder() override = default; +}; + +/** + * Format + * [header] [block 1] [block 2] ... [block N] + * Header + * [block size] [_mini_blocks_per_block] [_total_value_count] [first value] + * Block + * [min delta] [list of bitwidths of the mini blocks] [miniblocks] + */ +template +class DeltaBitPackDecoder final : public DeltaDecoder { +public: + using UT = std::make_unsigned_t; + + DeltaBitPackDecoder() = default; + ~DeltaBitPackDecoder() override = default; + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + uint32_t num_valid_values = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), static_cast(batch_size), + &num_valid_values)); + // Skips retain exact validation without allocating in proportion to a sparse gap. + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError("Expected to skip {} Parquet delta values, skipped {}", + num_values, skipped + num_valid_values); + } + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + _values.resize(num_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", num_values, + decoded_count); + } + return consumer.consume(reinterpret_cast(_values.data()), num_values, + sizeof(T)); + } + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + _values.resize(selection.total_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", + selection.total_values, decoded_count); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, range.count * sizeof(T)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(reinterpret_cast(_values.data()), output, + sizeof(T)); + } + + Status decode(T* buffer, uint32_t num_values, uint32_t* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + uint32_t valid_values_count() { + // _total_value_count in header ignores of null values + return _total_values_remaining; + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + // The bit-width table drives later miniblock transitions, so it is reclaimable only after + // the page is exhausted; _values contains completed batch output and is always disposable. + if (_total_values_remaining == 0) { + release_vector_if_oversized(&_delta_bit_widths, max_retained_bytes); + } + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(T) + _delta_bit_widths.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(T) + _delta_bit_widths.size() * sizeof(uint8_t); + } + + Status set_data(Slice* slice) override { + _bit_reader.reset( + new BitReader((const uint8_t*)slice->data, cast_set(slice->size))); + RETURN_IF_ERROR(_init_header()); + _data = slice; + _offset = 0; + return Status::OK(); + } + + // Set BitReader which is already initialized by DeltaLengthByteArrayDecoder or + // DeltaByteArrayDecoder + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_init_header()); + return Status::OK(); + } + +private: + static constexpr int kMaxDeltaBitWidth = static_cast(sizeof(T) * 8); + Status _init_header(); + Status _init_block(); + Status _init_mini_block(int bit_width); + Status _get_internal(T* buffer, uint32_t max_values, uint32_t* out_num_values); + + std::vector _values; + + std::shared_ptr _bit_reader; + uint32_t _values_per_block; + uint32_t _mini_blocks_per_block; + uint32_t _values_per_mini_block; + uint32_t _total_value_count; + + T _min_delta; + T _last_value; + + uint32_t _mini_block_idx; + std::vector _delta_bit_widths; + int _delta_bit_width; + // If the page doesn't contain any block, `_block_initialized` will + // always be false. Otherwise, it will be true when first block initialized. + bool _block_initialized; + + uint32_t _total_values_remaining; + // Remaining values in current mini block. If the current block is the last mini block, + // _values_remaining_current_mini_block may greater than _total_values_remaining. + uint32_t _values_remaining_current_mini_block; +}; + +class DeltaLengthByteArrayDecoder final : public DeltaDecoder { +public: + explicit DeltaLengthByteArrayDecoder() + : _len_decoder(), _buffered_length(0), _buffered_data(0) {} + + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _len_decoder.set_expected_values(expected_values); + } + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + int num_valid_values = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), static_cast(batch_size), &num_valid_values)); + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError( + "Expected to skip {} Parquet delta-length values, skipped {}", num_values, + skipped + num_valid_values); + } + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + num_values, decoded_count); + } + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + _values.resize(selection.total_values); + int decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + selection.total_values, decoded_count); + } + _string_refs.resize(selection.selected_values); + size_t output = 0; + for (const auto& range : selection.ranges) { + for (size_t row = 0; row < range.count; ++row) { + const auto& value = _values[range.first + row]; + _string_refs[output++] = StringRef(value.data, value.size); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + int valid_values_count() const { return _num_valid_values; } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_buffered_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _len_decoder.release_scratch(max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _buffered_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _len_decoder.retained_scratch_bytes(); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _buffered_length.size() * sizeof(int32_t) + _buffered_data.size() * sizeof(char) + + _len_decoder.active_scratch_bytes(); + } + + Status set_data(Slice* slice) override { + if (slice == nullptr || slice->size == 0) { + // Reused decoders must never retain lengths or payload pointers from the prior page. + _bit_reader.reset(); + _data = nullptr; + _offset = 0; + _num_valid_values = 0; + _buffered_length.clear(); + _buffered_data.clear(); + return Status::Corruption("Parquet delta-length page is empty"); + } + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + _data = slice; + _offset = 0; + RETURN_IF_ERROR(_init_lengths()); + return Status::OK(); + } + + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_init_lengths()); + return Status::OK(); + } + +private: + Status _init_lengths(); + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _len_decoder; + + int _num_valid_values; + std::vector _buffered_length; + std::vector _buffered_data; +}; + +class DeltaByteArrayDecoder : public DeltaDecoder { +public: + explicit DeltaByteArrayDecoder() : _buffered_prefix_length(0), _buffered_data(0) {} + + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _prefix_len_decoder.set_expected_values(expected_values); + _suffix_decoder.set_expected_values(expected_values); + } + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + RETURN_IF_ERROR(_decode_slices(batch_size)); + RETURN_IF_ERROR(_validate_fixed_width_values()); + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(selection.total_values)); + _string_refs.resize(selection.selected_values); + size_t output = 0; + for (const auto& range : selection.ranges) { + for (size_t row = 0; row < range.count; ++row) { + const auto& value = _values[range.first + row]; + _string_refs[output++] = StringRef(value.data, value.size); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + RETURN_IF_ERROR(_validate_fixed_width_values()); + const size_t byte_size = num_values * static_cast(_type_length); + _fixed_values.resize(byte_size); + for (size_t row = 0; row < num_values; ++row) { + memcpy(_fixed_values.data() + row * _type_length, _values[row].data, _type_length); + } + return consumer.consume(_fixed_values.data(), num_values, + static_cast(_type_length)); + } + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(selection.total_values)); + // Validate every consumed value, including filtered ranges, so selection cannot hide a + // malformed FIXED_LEN_BYTE_ARRAY width that a later cursor advance has already committed. + RETURN_IF_ERROR(_validate_fixed_width_values()); + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.selected_values > + std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet delta-byte-array selection byte size overflows"); + } + _fixed_values.resize(selection.selected_values * value_width); + size_t output = 0; + for (const auto& range : selection.ranges) { + for (size_t row = 0; row < range.count; ++row) { + const auto& value = _values[range.first + row]; + memcpy(_fixed_values.data() + output * value_width, value.data, value_width); + ++output; + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_fixed_values.data(), output, value_width); + } + + Status set_data(Slice* slice) override { + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + auto prefix_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_prefix_len_decoder.set_bit_reader(prefix_reader)); + + // get the number of encoded prefix lengths + int num_prefix = _prefix_len_decoder.valid_values_count(); + DeltaBitPackDecoder prefix_locator; + prefix_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(prefix_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector ignored_prefixes( + std::min(static_cast(num_prefix), kLocateBatchSize)); + size_t located = 0; + while (located < static_cast(num_prefix)) { + const size_t batch_size = + std::min(static_cast(num_prefix) - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR(prefix_locator.decode(ignored_prefixes.data(), + static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + located += batch_size; + } + _num_valid_values = num_prefix; + + // at this time, the decoder_ will be at the start of the encoded suffix data. + RETURN_IF_ERROR(_suffix_decoder.set_bit_reader(_bit_reader)); + if (UNLIKELY(_suffix_decoder.valid_values_count() != num_prefix)) { + return Status::Corruption("Parquet delta prefix and suffix counts differ"); + } + + // TODO: read corrupted files written with bug(PARQUET-246). _last_value should be set + // to _last_value_in_previous_page when decoding a new page(except the first page) + _last_value = ""; + return Status::OK(); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_fixed_values, max_retained_bytes); + release_vector_if_oversized(&_buffered_prefix_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _prefix_len_decoder.release_scratch(max_retained_bytes); + _suffix_decoder.release_scratch(max_retained_bytes); + // Prefix reconstruction crosses batch boundaries, so the previous value remains semantic + // decoder state until every value in the current page has been consumed. + if (_num_valid_values == 0) { + if (_last_value.capacity() > max_retained_bytes) std::string().swap(_last_value); + if (_last_value_in_previous_page.capacity() > max_retained_bytes) { + std::string().swap(_last_value_in_previous_page); + } + } + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _fixed_values.capacity() * sizeof(uint8_t) + + _buffered_prefix_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _last_value.capacity() + + _last_value_in_previous_page.capacity() + + _prefix_len_decoder.retained_scratch_bytes() + + _suffix_decoder.retained_scratch_bytes(); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _fixed_values.size() * sizeof(uint8_t) + + _buffered_prefix_length.size() * sizeof(int32_t) + + _buffered_data.size() * sizeof(char) + _last_value.size() + + _last_value_in_previous_page.size() + _prefix_len_decoder.active_scratch_bytes() + + _suffix_decoder.active_scratch_bytes(); + } + +private: + Status _validate_fixed_width_values() const { + if (_type_length <= 0) { + return Status::OK(); + } + const size_t value_width = static_cast(_type_length); + for (const auto& value : _values) { + if (UNLIKELY(value.size != value_width)) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + value.size, value_width); + } + } + return Status::OK(); + } + + Status _decode_slices(size_t num_values) { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-byte-array values, decoded {}", + num_values, decoded_count); + } + return Status::OK(); + } + + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::vector _fixed_values; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _prefix_len_decoder; + DeltaLengthByteArrayDecoder _suffix_decoder; + std::string _last_value; + // string buffer for last value in previous page + std::string _last_value_in_previous_page; + int _num_valid_values; + std::vector _buffered_prefix_length; + std::vector _buffered_data; +}; +} // namespace doris::format::parquet::native + +namespace doris::format::parquet::native { + +template +Status DeltaBitPackDecoder::_init_header() { + if (!_bit_reader->GetVlqInt(&_values_per_block) || + !_bit_reader->GetVlqInt(&_mini_blocks_per_block) || + !_bit_reader->GetVlqInt(&_total_value_count) || + !_bit_reader->GetZigZagVlqInt(&_last_value)) { + return Status::IOError("Init header eof"); + } + if (_values_per_block == 0) { + return Status::InvalidArgument("Cannot have zero value per block"); + } + if (_values_per_block % 128 != 0) { + return Status::InvalidArgument( + "the number of values in a block must be multiple of 128, but it's " + + std::to_string(_values_per_block)); + } + if (_mini_blocks_per_block == 0) { + return Status::InvalidArgument("Cannot have zero miniblock per block"); + } + _values_per_mini_block = _values_per_block / _mini_blocks_per_block; + if (_values_per_mini_block == 0) { + return Status::InvalidArgument("Cannot have zero value per miniblock"); + } + if (_values_per_mini_block % 32 != 0) { + return Status::InvalidArgument( + "The number of values in a miniblock must be multiple of 32, but it's " + + std::to_string(_values_per_mini_block)); + } + // Encoded counts are external ULEB32 values. Bound them by the page's advertised logical + // values before any vector uses the count; optional levels may only reduce this upper bound. + if (UNLIKELY(_total_value_count > _expected_values)) { + return Status::Corruption("Parquet delta header advertises {} values, page allows {}", + _total_value_count, _expected_values); + } + _total_values_remaining = _total_value_count; + _delta_bit_widths.clear(); + // init as empty property + _block_initialized = false; + _delta_bit_width = 0; + _values_remaining_current_mini_block = 0; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_block() { + DCHECK_GT(_total_values_remaining, 0) << "InitBlock called at EOF"; + if (!_bit_reader->GetZigZagVlqInt(&_min_delta)) { + return Status::IOError("Init block eof"); + } + + // One byte follows for each miniblock. Defer allocation until a block is actually consumed so + // a one-value page cannot turn an irrelevant malicious miniblock count into a large resize. + if (UNLIKELY(_mini_blocks_per_block > static_cast(_bit_reader->bytes_left()))) { + return Status::Corruption("Parquet delta miniblock count {} exceeds remaining {} bytes", + _mini_blocks_per_block, _bit_reader->bytes_left()); + } + _delta_bit_widths.resize(_mini_blocks_per_block); + + // read the bitwidth of each miniblock + uint8_t* bit_width_data = _delta_bit_widths.data(); + for (uint32_t i = 0; i < _mini_blocks_per_block; ++i) { + if (!_bit_reader->GetAligned(1, bit_width_data + i)) { + return Status::IOError("Decode bit-width EOF"); + } + // Note that non-conformant bitwidth entries are allowed by the Parquet spec + // for extraneous miniblocks in the last block (GH-14923), so we check + // the bitwidths when actually using them (see InitMiniBlock()). + } + _mini_block_idx = 0; + _block_initialized = true; + RETURN_IF_ERROR(_init_mini_block(bit_width_data[0])); + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_mini_block(int bit_width) { + if (bit_width > kMaxDeltaBitWidth) [[unlikely]] { + return Status::InvalidArgument("delta bit width larger than integer bit width"); + } + _delta_bit_width = bit_width; + _values_remaining_current_mini_block = _values_per_mini_block; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_get_internal(T* buffer, uint32_t num_values, + uint32_t* out_num_values) { + num_values = std::min(num_values, _total_values_remaining); + if (num_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + uint32_t i = 0; + while (i < num_values) { + if (_values_remaining_current_mini_block == 0) [[unlikely]] { + if (!_block_initialized) [[unlikely]] { + buffer[i++] = _last_value; + DCHECK_EQ(i, 1); // we're at the beginning of the page + if (i == num_values) { + // When block is uninitialized and i reaches num_values we have two + // different possibilities: + // 1. _total_value_count == 1, which means that the page may have only + // one value (encoded in the header), and we should not initialize + // any block. + // 2. _total_value_count != 1, which means we should initialize the + // incoming block for subsequent reads. + if (_total_value_count != 1) { + RETURN_IF_ERROR(_init_block()); + } + break; + } + RETURN_IF_ERROR(_init_block()); + } else { + ++_mini_block_idx; + if (_mini_block_idx < _mini_blocks_per_block) { + RETURN_IF_ERROR(_init_mini_block(_delta_bit_widths.data()[_mini_block_idx])); + } else { + RETURN_IF_ERROR(_init_block()); + } + } + } + + uint32_t values_decode = std::min(_values_remaining_current_mini_block, num_values - i); + for (uint32_t j = 0; j < values_decode; ++j) { + if (!_bit_reader->GetValue(_delta_bit_width, buffer + i + j)) { + return Status::IOError("Get batch EOF"); + } + } + for (int j = 0; j < values_decode; ++j) { + // Addition between min_delta, packed int and last_value should be treated as + // unsigned addition. Overflow is as expected. + buffer[i + j] = static_cast(_min_delta) + static_cast(buffer[i + j]) + + static_cast(_last_value); + _last_value = buffer[i + j]; + } + _values_remaining_current_mini_block -= values_decode; + i += values_decode; + } + _total_values_remaining -= num_values; + + if (_total_values_remaining == 0) [[unlikely]] { + int64_t padding_bits = 0; + // Footer-controlled miniblock counts can exceed 32-bit products. Widen before multiplying; + // BitReader::Advance then verifies that the full declared padding exists in the stream. + RETURN_IF_ERROR(detail::checked_delta_padding_bits(cast_set(_delta_bit_width), + _values_remaining_current_mini_block, + &padding_bits)); + if (!_bit_reader->Advance(padding_bits)) { + return Status::IOError("Skip padding EOF"); + } + _values_remaining_current_mini_block = 0; + } + *out_num_values = num_values; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp new file mode 100644 index 00000000000000..1f1166421a4035 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { + +// Dictionary decoders retain only encoded physical values and the index-stream cursor. Logical +// interpretation is deliberately delegated to DataTypeSerDe through decode_dictionary(). +template +class FixLengthDictDecoder final : public BaseDictDecoder { +public: + FixLengthDictDecoder() = default; + ~FixLengthDictDecoder() override = default; + + size_t dictionary_size() const override { return _num_dictionary_values; } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return fixed_consumer.consume(_dict.get(), _num_dictionary_values, + static_cast(_type_length)); + } + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override { + if (UNLIKELY(_type_length <= 0 || length < 0 || + num_values > std::numeric_limits::max() / + static_cast(_type_length) || + num_values * static_cast(_type_length) != + static_cast(length))) { + return Status::Corruption("Wrong dictionary data for fixed length type"); + } + if (UNLIKELY(dict == nullptr)) { + return Status::Corruption("Fixed-length Parquet dictionary is null"); + } + _dict = std::move(dict); + _num_dictionary_values = num_values; + ++_dictionary_generation; + return Status::OK(); + } + +private: + size_t _num_dictionary_values = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp new file mode 100644 index 00000000000000..53fb11a09ee87f --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +#include + +namespace doris::format::parquet::native { + +Status FixLengthPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + // Truncated PLAIN pages retain the public error keyword used by corrupt-file regression + // checks, while the bounds check still prevents the native decoder from reading past data. + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); + } + RETURN_IF_ERROR(consumer.consume(reinterpret_cast(_data->data) + _offset, + num_values, static_cast(_type_length))); + _offset += byte_size; + return Status::OK(); +} + +Status FixLengthPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet plain selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError("Unexpected end of stream in Parquet plain selection decoder"); + } + const auto* values = reinterpret_cast(_data->data) + _offset; + _offset += input_bytes; + // PLAIN pages are random-access fixed-width spans. Keep those page bytes pinned while the + // consumer gathers directly into the final column, otherwise sparse scans pay for a second + // selected-width buffer and copy before materialization. + return consumer.consume_selected(values, value_width, selection.ranges); +} + +Status FixLengthPlainDecoder::skip_values(size_t num_values) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || num_values > (_data->size - _offset) / value_width)) { + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); + } + _offset += num_values * value_width; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h new file mode 100644 index 00000000000000..c6c6784cd35f76 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { + +class FixLengthPlainDecoder final : public Decoder { +public: + FixLengthPlainDecoder() = default; + ~FixLengthPlainDecoder() override = default; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + size_t retained_scratch_bytes() const override { return 0; } +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.cpp b/be/src/format_v2/parquet/reader/native/level_decoder.cpp new file mode 100644 index 00000000000000..618d6ebf9f6d45 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.cpp @@ -0,0 +1,271 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_decoder.h" + +#include + +#include +#include + +#include "common/cast_set.h" +#include "util/bit_stream_utils.inline.h" +#include "util/bit_util.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { + +static constexpr size_t V1_LEVEL_SIZE = 4; + +Status LevelDecoder::init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels) { + _encoding = encoding; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; + switch (encoding) { + case tparquet::Encoding::RLE: { + if (slice->size < V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + + uint8_t* data = (uint8_t*)slice->data; + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > slice->size - V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + _rle_decoder = RleBatchDecoder(data + V1_LEVEL_SIZE, num_bytes, _bit_width); + + slice->data += V1_LEVEL_SIZE + num_bytes; + slice->size -= V1_LEVEL_SIZE + num_bytes; + break; + } + case tparquet::Encoding::BIT_PACKED: { + // Header counts are uint32_t and can overflow before the byte bound check. Widen first so + // a forged level count cannot wrap to a small slice and desynchronize following values. + const uint64_t num_bits = static_cast(num_levels) * _bit_width; + const size_t num_bytes = static_cast((num_bits + 7) / 8); + if (num_bytes > slice->size) { + return Status::Corruption("Wrong parquet level format"); + } + if (num_bytes > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet BIT_PACKED level stream is too large"); + } + _bit_packed_decoder = BitReader((uint8_t*)slice->data, cast_set(num_bytes)); + + slice->data += num_bytes; + slice->size -= num_bytes; + break; + } + default: + return Status::IOError("Unsupported encoding for parquet level"); + } + return Status::OK(); +} + +Status LevelDecoder::init_v2(const Slice& levels, level_t max_level, uint32_t num_levels) { + _encoding = tparquet::Encoding::RLE; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; + size_t byte_length = levels.size; + _rle_decoder = RleBatchDecoder((uint8_t*)levels.data, cast_set(byte_length), + _bit_width); + return Status::OK(); +} + +size_t LevelDecoder::get_levels(level_t* levels, size_t n) { + _can_rewind = false; + // toto template. + if (_encoding == tparquet::Encoding::RLE) { + n = std::min((size_t)_num_levels, n); + size_t num_decoded = 0; + if (_has_buffered_level && n > 0) { + if (!accept_level(_buffered_level)) { + return 0; + } + levels[num_decoded++] = _buffered_level; + _has_buffered_level = false; + } + if (num_decoded < n) { + const size_t remaining = n - num_decoded; + _rle_scratch.resize(remaining); + const size_t batch_decoded = + _rle_decoder.GetBatch(_rle_scratch.data(), cast_set(remaining)); + for (size_t i = 0; i < batch_decoded; ++i) { + const level_t level = cast_set(_rle_scratch[i]); + if (!accept_level(level)) { + return 0; + } + levels[num_decoded + i] = level; + } + num_decoded += batch_decoded; + } + _num_levels -= num_decoded; + _can_rewind = false; + return num_decoded; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + n = std::min((size_t)_num_levels, n); + size_t decoded = 0; + for (; decoded < n; ++decoded) { + level_t level = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &level)) { + break; + } + if (!accept_level(level)) { + return 0; + } + levels[decoded] = level; + } + _num_levels -= decoded; + return decoded; + } + return 0; +} + +size_t LevelDecoder::get_next_run(level_t* val, size_t max_run) { + DORIS_CHECK(val != nullptr); + _can_rewind = false; + max_run = std::min(max_run, _num_levels); + if (max_run == 0) { + return 0; + } + if (_encoding == tparquet::Encoding::RLE) { + size_t decoded = 0; + if (_has_buffered_level) { + *val = _buffered_level; + _has_buffered_level = false; + if (!accept_level(*val)) { + return 0; + } + decoded = 1; + } else { + uint16_t first = 0; + if (_rle_decoder.GetBatch(&first, 1) != 1) { + return 0; + } + *val = cast_set(first); + if (!accept_level(*val)) { + return 0; + } + decoded = 1; + } + while (decoded < max_run) { + const int32_t repeats = _rle_decoder.NextNumRepeats(); + if (repeats > 0) { + const level_t repeated = cast_set(_rle_decoder.GetRepeatedValue(0)); + if (!accept_level(repeated)) return 0; + if (repeated != *val) break; + const int32_t consume = std::min(repeats, cast_set(max_run - decoded)); + _rle_decoder.GetRepeatedValue(consume); + decoded += consume; + continue; + } + if (_rle_decoder.NextNumLiterals() == 0) break; + uint16_t literal = 0; + if (!_rle_decoder.GetLiteralValues(1, &literal)) break; + const level_t next = cast_set(literal); + if (!accept_level(next)) return 0; + if (next != *val) { + // Batch RLE has no physical rewind; retain the one-value lookahead logically. + _buffered_level = next; + _has_buffered_level = true; + break; + } + ++decoded; + } + _num_levels -= decoded; + _can_rewind = false; + return decoded; + } + if (_encoding != tparquet::Encoding::BIT_PACKED || + !_bit_packed_decoder.GetValue(_bit_width, val)) { + return 0; + } + if (!accept_level(*val)) { + return 0; + } + size_t decoded = 1; + while (decoded < max_run) { + level_t next = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &next)) { + break; + } + if (!accept_level(next)) return 0; + if (next != *val) { + // The lookahead belongs to the following run, so cursor APIs must leave it unread. + _bit_packed_decoder.Rewind(_bit_width); + break; + } + ++decoded; + } + _num_levels -= decoded; + return decoded; +} + +level_t LevelDecoder::get_next() { + if (_num_levels == 0) { + return -1; + } + level_t next = -1; + bool decoded = false; + if (_encoding == tparquet::Encoding::RLE) { + if (_has_buffered_level) { + next = _buffered_level; + _has_buffered_level = false; + decoded = true; + } else { + uint16_t value = 0; + decoded = _rle_decoder.GetBatch(&value, 1) == 1; + next = cast_set(value); + } + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + decoded = _bit_packed_decoder.GetValue(_bit_width, &next); + } + if (!decoded) { + return -1; + } + if (!accept_level(next)) { + return -1; + } + --_num_levels; + _last_level = next; + _can_rewind = true; + return next; +} + +void LevelDecoder::rewind_one() { + if (_encoding == tparquet::Encoding::RLE) { + DORIS_CHECK(_can_rewind && !_has_buffered_level); + _buffered_level = _last_level; + _has_buffered_level = true; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + DORIS_CHECK(_can_rewind); + _bit_packed_decoder.Rewind(_bit_width); + } else { + return; + } + // Rewinding restores one advertised level as well as its encoded bits. + ++_num_levels; + _can_rewind = false; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.h b/be/src/format_v2/parquet/reader/native/level_decoder.h new file mode 100644 index 00000000000000..9dcff491eb0759 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/status.h" +#include "format_v2/parquet/reader/native/common.h" +#include "util/bit_stream_utils.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +class LevelDecoder { +public: + LevelDecoder() = default; + ~LevelDecoder() = default; + + Status init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels); + + Status init_v2(const Slice& levels, level_t max_level, uint32_t num_levels); + + inline bool has_levels() const { return _num_levels > 0; } + + size_t get_levels(level_t* levels, size_t n); + + size_t get_next_run(level_t* val, size_t max_run); + + level_t get_next(); + + void rewind_one(); + + void release_scratch(size_t max_retained_bytes) { + if (_rle_scratch.capacity() * sizeof(uint16_t) > max_retained_bytes) { + std::vector().swap(_rle_scratch); + } + } + size_t retained_scratch_bytes() const { return _rle_scratch.capacity() * sizeof(uint16_t); } + size_t active_scratch_bytes() const { return _rle_scratch.size() * sizeof(uint16_t); } + +private: + bool accept_level(level_t level) { + if (level >= 0 && level <= _max_level) { + return true; + } + // Bit width rounds up to a power of two, so encoded values can fit the bit stream while + // still exceeding the schema maximum. Poison the decoder before any caller can use them. + _num_levels = 0; + _has_buffered_level = false; + _can_rewind = false; + return false; + } + + tparquet::Encoding::type _encoding; + level_t _bit_width = 0; + level_t _max_level = 0; + uint32_t _num_levels = 0; + RleBatchDecoder _rle_decoder; + std::vector _rle_scratch; + BitReader _bit_packed_decoder; + bool _has_buffered_level = false; + bool _can_rewind = false; + level_t _buffered_level = -1; + level_t _last_level = -1; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.cpp b/be/src/format_v2/parquet/reader/native/level_reader.cpp new file mode 100644 index 00000000000000..01efbcae9b6233 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.cpp @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_reader.h" + +#include +#include + +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/tracing_file_reader.h" + +namespace doris::format::parquet::native { + +class LevelReader::Impl { +public: + virtual ~Impl() = default; + + virtual Status init() = 0; + virtual Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) = 0; + virtual ColumnChunkReaderStatistics statistics() = 0; +}; + +template +class LevelReaderImpl final : public LevelReader::Impl { +public: + LevelReaderImpl(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, std::string page_cache_file_key, + ParquetReaderCompat compat) + : _file(std::move(file)), + _column_chunk(std::move(column_chunk)), + _field(field), + _total_rows(total_rows), + _max_buffer_size(max_buffer_size), + _io_ctx(io_ctx), + _enable_page_cache(enable_page_cache), + _page_cache_file_key(std::move(page_cache_file_key)), + _compat(compat) {} + + Status init() override { + DORIS_CHECK(_file != nullptr); + DORIS_CHECK(_field != nullptr); + const auto& metadata = _column_chunk.meta_data; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(metadata, _file->size(), + _compat.parquet_816_padding, &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_size = chunk_range.length; + size_t prefetch_buffer_size = std::min(chunk_size, _max_buffer_size); + auto* tracing_reader = typeid_cast(_file.get()); + if ((tracing_reader != nullptr && + typeid_cast(tracing_reader->inner_reader().get()) != + nullptr) || + typeid_cast(_file.get()) != nullptr) { + prefetch_buffer_size = 0; + } + _stream = std::make_unique(_file, chunk_start, chunk_size, + prefetch_buffer_size); + _chunk_reader = std::make_unique>( + _stream.get(), &_column_chunk, _field, nullptr, _total_rows, _io_ctx, + ParquetPageReadContext(_enable_page_cache, _page_cache_file_key, + _compat.data_page_v2_always_compressed), + &chunk_range); + return _chunk_reader->init(); + } + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) override { + DORIS_CHECK(repetition_levels != nullptr); + DORIS_CHECK(definition_levels != nullptr); + DORIS_CHECK(rows_read != nullptr); + repetition_levels->clear(); + definition_levels->clear(); + *rows_read = 0; + if (_current_row > _total_rows || rows > _total_rows - _current_row) { + return Status::Corruption("Parquet level reader requested rows [{}, {}) of {}", + _current_row, _current_row + rows, _total_rows); + } + if constexpr (IN_COLLECTION) { + RETURN_IF_ERROR( + read_nested_rows(rows, repetition_levels, definition_levels, rows_read)); + } else { + RETURN_IF_ERROR(read_flat_rows(rows, repetition_levels, definition_levels, rows_read)); + } + _current_row += *rows_read; + return Status::OK(); + } + + ColumnChunkReaderStatistics statistics() override { return _chunk_reader->statistics(); } + +private: + Status read_flat_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + const size_t read_now = std::min( + rows - *rows_read, static_cast(_chunk_reader->remaining_num_values())); + if (read_now == 0) { + return Status::Corruption("Parquet flat level reader made no progress"); + } + RETURN_IF_ERROR( + _chunk_reader->read_levels(read_now, repetition_levels, definition_levels)); + *rows_read += read_now; + if (_chunk_reader->remaining_num_values() == 0 && _chunk_reader->has_next_page()) { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + return Status::OK(); + } + + Status read_nested_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(_current_row + *rows_read)); + const size_t start_level = definition_levels->size(); + size_t loaded_rows = 0; + bool crosses_page = false; + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows( + *repetition_levels, rows - *rows_read, &loaded_rows, &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(*definition_levels, start_level)); + while (crosses_page) { + const size_t continuation_start = definition_levels->size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(*repetition_levels, + &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR( + _chunk_reader->skip_nested_values(*definition_levels, continuation_start)); + } + if (loaded_rows == 0) { + return Status::Corruption("Parquet nested level reader made no progress"); + } + *rows_read += loaded_rows; + } + return Status::OK(); + } + + io::FileReaderSPtr _file; + tparquet::ColumnChunk _column_chunk; + NativeFieldSchema* _field = nullptr; + size_t _total_rows = 0; + size_t _max_buffer_size = 0; + io::IOContext* _io_ctx = nullptr; + bool _enable_page_cache = false; + std::string _page_cache_file_key; + ParquetReaderCompat _compat; + size_t _current_row = 0; + std::unique_ptr _stream; + std::unique_ptr> _chunk_reader; +}; + +Status LevelReader::create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, + std::unique_ptr* reader) { + DORIS_CHECK(reader != nullptr); + DORIS_CHECK(field != nullptr); + std::unique_ptr impl; + if (field->repetition_level > 0) { + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); + } else { + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); + } + RETURN_IF_ERROR(impl->init()); + reader->reset(new LevelReader(std::move(impl))); + return Status::OK(); +} + +LevelReader::LevelReader(std::unique_ptr impl) : _impl(std::move(impl)) {} + +LevelReader::~LevelReader() = default; + +Status LevelReader::read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + return _impl->read_rows(rows, repetition_levels, definition_levels, rows_read); +} + +Status LevelReader::skip_rows(size_t rows) { + size_t rows_read = 0; + RETURN_IF_ERROR( + read_rows(rows, &_skip_repetition_levels, &_skip_definition_levels, &rows_read)); + if (rows_read != rows) { + return Status::Corruption("Parquet level reader skipped {} of {} rows", rows_read, rows); + } + return Status::OK(); +} + +ColumnChunkReaderStatistics LevelReader::statistics() { + return _impl->statistics(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.h b/be/src/format_v2/parquet/reader/native/level_reader.h new file mode 100644 index 00000000000000..11117d25dcb252 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris::io { +struct IOContext; +} // namespace doris::io + +namespace doris::format::parquet::native { + +// A physical-leaf reader that advances all three Parquet streams (repetition levels, definition +// levels and encoded values) but retains only the two level streams. It is used for shape-only +// operations such as COUNT(nullable_col), where materializing a large BYTE_ARRAY value would be +// both unnecessary and potentially unbounded. +// +// This deliberately shares ColumnChunkReader with the value path. Page V1/V2 parsing, +// decompression, dictionary-page handling, page cache semantics and corruption checks therefore +// cannot drift between an aggregate shortcut and an ordinary scan. +class LevelReader { +public: + class Impl; + + static Status create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, + std::unique_ptr* reader); + + ~LevelReader(); + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read); + Status skip_rows(size_t rows); + ColumnChunkReaderStatistics statistics(); + +private: + explicit LevelReader(std::unique_ptr impl); + + std::unique_ptr _impl; + // COUNT range gaps can call skip repeatedly as the adaptive row cap changes. Keep these + // throw-away level buffers on the persistent reader so only their logical sizes are reset. + std::vector _skip_repetition_levels; + std::vector _skip_definition_levels; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp new file mode 100644 index 00000000000000..f9216a244a4e20 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -0,0 +1,306 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/page_reader.h" + +#include +#include +#include +#include + +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/slice.h" +#include "util/thrift_util.h" + +namespace doris { +namespace io { +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { +static constexpr size_t INIT_PAGE_HEADER_SIZE = 128; + +template +Status PageReader::_validate_page_header(uint32_t header_size) const { + if (UNLIKELY(_cur_page_header.compressed_page_size < 0 || + _cur_page_header.uncompressed_page_size < 0)) { + return Status::Corruption("Parquet page has a negative compressed or uncompressed size"); + } + if (UNLIKELY(header_size > _end_offset - _offset || + static_cast(_cur_page_header.compressed_page_size) > + _end_offset - _offset - header_size)) { + // Sizes are untrusted signed Thrift fields and must fit the column chunk before arithmetic. + return Status::Corruption("Parquet page payload exceeds its column chunk"); + } + + const bool has_v1 = _cur_page_header.__isset.data_page_header; + const bool has_v2 = _cur_page_header.__isset.data_page_header_v2; + const bool has_dictionary = _cur_page_header.__isset.dictionary_page_header; + const bool has_index = _cur_page_header.__isset.index_page_header; + bool matching_layout = false; + switch (_cur_page_header.type) { + case tparquet::PageType::DATA_PAGE: + matching_layout = has_v1 && !has_v2 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DATA_PAGE_V2: + matching_layout = has_v2 && !has_v1 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DICTIONARY_PAGE: + matching_layout = has_dictionary && !has_v1 && !has_v2 && !has_index; + break; + case tparquet::PageType::INDEX_PAGE: + matching_layout = has_index && !has_v1 && !has_v2 && !has_dictionary; + break; + default: + // Forward-compatible auxiliary pages have no known page-specific member. Their bounded + // payload can be skipped safely; accepting a known member here would let the enum and + // decoder layout disagree. + matching_layout = !has_v1 && !has_v2 && !has_dictionary && !has_index; + break; + } + if (UNLIKELY(!matching_layout)) { + // The type discriminant owns the only valid union member. Guessing from an optional header + // lets a malformed page use one layout for validation and another for decoding. + return Status::Corruption("Parquet page type does not match its page-header layout"); + } + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + const auto& v2 = _cur_page_header.data_page_header_v2; + if (UNLIKELY(v2.num_values < 0 || v2.num_rows < 0 || v2.num_nulls < 0 || + v2.repetition_levels_byte_length < 0 || + v2.definition_levels_byte_length < 0)) { + return Status::Corruption("Parquet data page v2 has negative counts or level sizes"); + } + if (UNLIKELY(v2.num_nulls > v2.num_values || v2.num_rows > v2.num_values)) { + return Status::Corruption( + "Parquet data page v2 null or row count exceeds its value count"); + } + const uint64_t level_bytes = static_cast(v2.repetition_levels_byte_length) + + static_cast(v2.definition_levels_byte_length); + if (UNLIKELY(level_bytes > static_cast(_cur_page_header.compressed_page_size) || + level_bytes > + static_cast(_cur_page_header.uncompressed_page_size))) { + return Status::Corruption("Parquet data page v2 level bytes exceed the page payload"); + } + } else if (_cur_page_header.type == tparquet::PageType::DATA_PAGE && + UNLIKELY(_cur_page_header.data_page_header.num_values < 0)) { + return Status::Corruption("Parquet data page has a negative value count"); + } else if (_cur_page_header.type == tparquet::PageType::DICTIONARY_PAGE && + UNLIKELY(_cur_page_header.dictionary_page_header.num_values < 0)) { + return Status::Corruption("Parquet dictionary page has a negative value count"); + } + return Status::OK(); +} + +template +PageReader::PageReader(io::BufferedStreamReader* reader, + io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, + const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index) + : _reader(reader), + _io_ctx(io_ctx), + _offset(offset), + _start_offset(offset), + _end_offset(offset + length), + _total_rows(total_rows), + _metadata(metadata), + _page_read_ctx(page_read_ctx), + _offset_index(offset_index) { + _next_header_offset = _offset; + _state = INITIALIZED; + _page_cache_key_builder.init(_page_read_ctx.page_cache_file_key); + + if constexpr (OFFSET_INDEX) { + _end_row = _offset_index != nullptr && _offset_index->page_locations.size() >= 2 + ? _offset_index->page_locations[1].first_row_index + : _total_rows; + } +} + +template +Status PageReader::parse_page_header() { + if (_state == HEADER_PARSED) { + return Status::OK(); + } + if (UNLIKELY(_offset < _start_offset || _offset >= _end_offset)) { + return Status::IOError("Out-of-bounds Access"); + } + if (UNLIKELY(_offset != _next_header_offset)) { + return Status::IOError("Wrong header position, should seek to a page header first"); + } + if (UNLIKELY(_state != INITIALIZED)) { + return Status::IOError("Should skip or load current page to get next page"); + } + + _page_statistics.page_read_counter += 1; + + // Parse page header from file; header bytes are saved for possible cache insertion + const uint8_t* page_header_buf = nullptr; + size_t max_size = _end_offset - _offset; + size_t header_size = std::min(INIT_PAGE_HEADER_SIZE, max_size); + const size_t MAX_PAGE_HEADER_SIZE = config::parquet_header_max_size_mb << 20; + uint32_t real_header_size = 0; + + // Try a header-only lookup in the page cache. Cached pages store + // header + optional v2 levels + uncompressed payload, so we can + // parse the page header directly from the cached bytes and avoid + // a file read for the header. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + PageCacheHandle handle; + StoragePageCache::CacheKey key = make_page_cache_key(static_cast(_offset)); + if (StoragePageCache::instance()->lookup(key, &handle, segment_v2::DATA_PAGE)) { + // Parse header directly from cached data + _page_cache_handle = std::move(handle); + Slice s = _page_cache_handle.data(); + real_header_size = cast_set(s.size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Thrift does not clear absent optional fields when reusing an output object. + _cur_page_header = tparquet::PageHeader {}; + auto st = deserialize_thrift_msg(reinterpret_cast(s.data), + &real_header_size, true, &_cur_page_header); + if (!st.ok()) return st; + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } + // Increment page cache counters for a true cache hit on header+payload + _page_statistics.page_cache_hit_counter += 1; + // Detect whether the cached payload is compressed or decompressed and record + bool is_cache_payload_decompressed = should_cache_decompressed( + &_cur_page_header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (is_cache_payload_decompressed) { + _page_statistics.page_cache_decompressed_hit_counter += 1; + } else { + _page_statistics.page_cache_compressed_hit_counter += 1; + } + + _is_cache_payload_decompressed = is_cache_payload_decompressed; + + if constexpr (OFFSET_INDEX == false) { + if (is_header_v2()) { + _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _end_row = _start_row + _cur_page_header.data_page_header.num_values; + } + } + + // Save header bytes for later use (e.g., to insert updated cache entries) + _header_buf.assign(s.data, s.data + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); + } else { + _page_statistics.page_cache_missing_counter += 1; + // Clear any existing cache handle on miss to avoid holding stale handle + _page_cache_handle = PageCacheHandle(); + } + } + // NOTE: page cache lookup for *decompressed* page data is handled in + // ColumnChunkReader::load_page_data(). PageReader should only be + // responsible for parsing the header bytes from the file and saving + // them in `_header_buf` for possible later insertion into the cache. + while (true) { + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + header_size = std::min(header_size, max_size); + { + SCOPED_RAW_TIMER(&_page_statistics.read_page_header_time); + RETURN_IF_ERROR(_reader->read_bytes(&page_header_buf, _offset, header_size, _io_ctx)); + } + real_header_size = cast_set(header_size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Reset on every retry as a partial deserialize can otherwise leak a stale union member. + _cur_page_header = tparquet::PageHeader {}; + auto st = + deserialize_thrift_msg(page_header_buf, &real_header_size, true, &_cur_page_header); + if (st.ok()) { + break; + } + if (_offset + header_size >= _end_offset || real_header_size > MAX_PAGE_HEADER_SIZE) { + return Status::IOError( + "Failed to deserialize parquet page header. offset: {}, " + "header size: {}, end offset: {}, real header size: {}", + _offset, header_size, _end_offset, real_header_size); + } + header_size <<= 2; + } + + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + if (is_header_v2()) { + _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _end_row = _start_row + _cur_page_header.data_page_header.num_values; + } + } + + if constexpr (OFFSET_INDEX == true) { + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + } + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } + + // Save header bytes for possible cache insertion later + _header_buf.assign(page_header_buf, page_header_buf + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status PageReader::get_page_data(Slice& slice) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::IOError("Should generate page header first to load current page data"); + } + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + slice.size = _cur_page_header.compressed_page_size; + RETURN_IF_ERROR(_reader->read_bytes(slice, _offset, _io_ctx)); + _offset += slice.size; + _state = DATA_LOADED; + return Status::OK(); +} + +template class PageReader; +template class PageReader; +template class PageReader; +template class PageReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.h b/be/src/format_v2/parquet/reader/native/page_reader.h new file mode 100644 index 00000000000000..56103a8994d059 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.h @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/status.h" +#include "storage/cache/page_cache.h" +#include "util/block_compression.h" +namespace doris { +class BlockCompressionCodec; + +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +/** + * Use to deserialize parquet page header, and get the page data in iterator interface. + */ + +// Session-level options for parquet page reading/caching. +struct ParquetPageReadContext { + // A default-constructed context has no stable file identity, so cache lookup must stay off. + bool enable_parquet_file_page_cache = false; + std::string page_cache_file_key; + bool data_page_v2_always_compressed = false; + ParquetPageReadContext() = default; + ParquetPageReadContext(bool enable_parquet_file_page_cache, std::string page_cache_file_key, + bool data_page_v2_always_compressed = false) + : enable_parquet_file_page_cache(enable_parquet_file_page_cache && + !page_cache_file_key.empty()), + page_cache_file_key(std::move(page_cache_file_key)), + data_page_v2_always_compressed(data_page_v2_always_compressed) {} +}; + +inline bool should_cache_decompressed(const tparquet::PageHeader* header, + const tparquet::ColumnMetaData& metadata, + bool data_page_v2_always_compressed = false) { + // Data Page V2 declares its payload representation independently of the column codec. A warm + // hit must never send an explicitly uncompressed cached payload through the codec again. + if (header->__isset.data_page_header_v2 && !header->data_page_header_v2.is_compressed && + !data_page_v2_always_compressed) { + return true; + } + if (header->compressed_page_size <= 0) return true; + if (metadata.codec == tparquet::CompressionCodec::UNCOMPRESSED) return true; + if (header->uncompressed_page_size == 0) return true; + + double ratio = static_cast(header->uncompressed_page_size) / + static_cast(header->compressed_page_size); + return ratio <= config::parquet_page_cache_decompress_threshold; +} + +class ParquetPageCacheKeyBuilder { +public: + void init(std::string file_key) { _file_key_prefix = std::move(file_key); } + StoragePageCache::CacheKey make_key(uint64_t end_offset, int64_t offset) const { + return StoragePageCache::CacheKey(_file_key_prefix, end_offset, offset); + } + +private: + std::string _file_key_prefix; +}; + +template +class PageReader { +public: + struct PageStatistics { + int64_t decode_header_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; + }; + + PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index = nullptr); + ~PageReader() = default; + + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + return _offset_index != nullptr && + _page_index + 1 < _offset_index->page_locations.size(); + } else { + // Deprecated + // Parquet file may not be standardized, + // _end_offset may exceed the actual data area. + // ColumnChunkReader::has_next_page() use the number of parsed values for judgment + // ref:https://github.com/duckdb/duckdb/issues/10829 + // [[deprecated]] + LOG(FATAL) << "has_next_page should not be called when no offset index"; + return _offset < _end_offset; + } + } + + Status parse_page_header(); + + Status next_page() { + _page_statistics.skip_page_header_num += _state == INITIALIZED; + if constexpr (OFFSET_INDEX) { + if (UNLIKELY(_offset_index == nullptr || + _page_index + 1 >= _offset_index->page_locations.size())) { + return Status::Corruption("Parquet OffsetIndex has no next page location"); + } + _page_index++; + _start_row = _offset_index->page_locations[_page_index].first_row_index; + if (_page_index + 1 < _offset_index->page_locations.size()) { + _end_row = _offset_index->page_locations[_page_index + 1].first_row_index; + } else { + _end_row = _total_rows; + } + int64_t next_page_offset = _offset_index->page_locations[_page_index].offset; + _offset = next_page_offset; + _next_header_offset = next_page_offset; + _state = INITIALIZED; + } else { + if (UNLIKELY(_offset == _start_offset)) { + return Status::Corruption("should parse first page."); + } + + if (is_header_v2()) { + _start_row += _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _start_row += _cur_page_header.data_page_header.num_values; + } + + _offset = _next_header_offset; + _state = INITIALIZED; + } + + return Status::OK(); + } + + Status skip_auxiliary_page() { + if constexpr (OFFSET_INDEX) { + // OffsetIndex enumerates data pages only, so an auxiliary physical page must not + // consume a logical page-location entry while advancing to its payload end. + skip_page_data(); + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + + Status dict_next_page() { + if constexpr (OFFSET_INDEX) { + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + + Status get_page_header(const tparquet::PageHeader** page_header) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::InternalError("Page header not parsed"); + } + *page_header = &_cur_page_header; + return Status::OK(); + } + + Status get_page_data(Slice& slice); + + // Skip page data and update offset (used when data is loaded from cache) + void skip_page_data() { + if (_state == HEADER_PARSED) { + _offset += _cur_page_header.compressed_page_size; + _state = DATA_LOADED; + } + } + + const std::vector& header_bytes() const { return _header_buf; } + // header start offset for current page + int64_t header_start_offset() const { + return static_cast(_next_header_offset) - static_cast(_last_header_size) - + static_cast(_cur_page_header.compressed_page_size); + } + uint64_t file_end_offset() const { return _end_offset; } + bool cached_decompressed() const { + return should_cache_decompressed(&_cur_page_header, _metadata, + _page_read_ctx.data_page_v2_always_compressed); + } + + PageStatistics& page_statistics() { return _page_statistics; } + + bool is_header_v2() { return _cur_page_header.__isset.data_page_header_v2; } + + // Returns whether the current page's cache payload is decompressed + bool is_cache_payload_decompressed() const { return _is_cache_payload_decompressed; } + + size_t start_row() const { return _start_row; } + + size_t end_row() const { return _end_row; } + + // Accessors for cache handle + bool has_page_cache_handle() const { return _page_cache_handle.cache() != nullptr; } + const doris::PageCacheHandle& page_cache_handle() const { return _page_cache_handle; } + StoragePageCache::CacheKey make_page_cache_key(int64_t offset) const { + return _page_cache_key_builder.make_key(_end_offset, offset); + } + +private: + Status _validate_page_header(uint32_t header_size) const; + + enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED }; + PageReaderState _state = INITIALIZED; + PageStatistics _page_statistics; + + io::BufferedStreamReader* _reader = nullptr; + io::IOContext* _io_ctx = nullptr; + // current reader offset in file location. + uint64_t _offset = 0; + // this page offset in file location. + uint64_t _start_offset = 0; + uint64_t _end_offset = 0; + uint64_t _next_header_offset = 0; + // current page row range + size_t _start_row = 0; + size_t _end_row = 0; + // total rows in this column chunk + size_t _total_rows = 0; + // Column metadata for this column chunk + const tparquet::ColumnMetaData& _metadata; + // Session-level parquet page cache options + ParquetPageReadContext _page_read_ctx; + // for page index + size_t _page_index = 0; + const tparquet::OffsetIndex* _offset_index; + + tparquet::PageHeader _cur_page_header; + bool _is_cache_payload_decompressed = true; + + // Page cache members + ParquetPageCacheKeyBuilder _page_cache_key_builder; + doris::PageCacheHandle _page_cache_handle; + // stored header bytes when cache miss so we can insert header+payload into cache + std::vector _header_buf; + // last parsed header size in bytes + uint32_t _last_header_size = 0; +}; + +template +std::unique_ptr> create_page_reader( + io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length, + size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& ctx, const tparquet::OffsetIndex* offset_index = nullptr) { + return std::make_unique>( + reader, io_ctx, offset, length, total_rows, metadata, ctx, offset_index); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp new file mode 100644 index 00000000000000..abed29d70253ce --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -0,0 +1,762 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native_column_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "runtime/runtime_state.h" + +namespace doris::format::parquet { +namespace { + +constexpr size_t MAX_RETAINED_BATCH_SCRATCH_BYTES = 4UL << 20; + +DataTypePtr projected_type(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection) { + if (!format::is_partial_projection(projection)) { + return schema.type; + } + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + return schema.type; + case ParquetColumnSchemaKind::STRUCT: { + DataTypes child_types; + Strings child_names; + child_types.reserve(projection->children.size()); + child_names.reserve(projection->children.size()); + for (const auto& child_projection : projection->children) { + const auto child_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(child_it != schema.children.end()); + child_types.push_back(make_nullable(projected_type(**child_it, &child_projection))); + child_names.push_back((*child_it)->name); + } + DataTypePtr type = std::make_shared(child_types, child_names); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto* child_projection = + format::find_child_projection(projection, schema.children[0]->local_id); + DORIS_CHECK(child_projection != nullptr); + DataTypePtr type = std::make_shared( + projected_type(*schema.children[0], child_projection)); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::MAP: { + DORIS_CHECK(schema.children.size() == 2); + const auto* value_projection = + format::find_child_projection(projection, schema.children[1]->local_id); + DORIS_CHECK(value_projection != nullptr); + DataTypePtr type = std::make_shared( + make_nullable(schema.children[0]->type), + make_nullable(projected_type(*schema.children[1], value_projection))); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + } + DORIS_CHECK(false); + return nullptr; +} + +const NativeFieldSchema* find_child_field(const NativeFieldSchema& parent, + const ParquetColumnSchema& child) { + auto field_it = std::ranges::find_if(parent.children, [&](const NativeFieldSchema& field) { + return (child.parquet_field_id >= 0 && field.field_id == child.parquet_field_id) || + field.name == child.name; + }); + return field_it == parent.children.end() ? nullptr : &*field_it; +} + +void collect_physical_subtree_ids(const NativeFieldSchema& field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + ids->insert(field.get_column_id()); + for (const auto& child : field.children) { + collect_physical_subtree_ids(child, ids); + } +} + +void collect_projected_ids(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const NativeFieldSchema& native_field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + if (!format::is_partial_projection(projection)) { + return; + } + for (const auto& child_projection : projection->children) { + const auto schema_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(schema_it != schema.children.end()); + const NativeFieldSchema* child_field = find_child_field(native_field, **schema_it); + DORIS_CHECK(child_field != nullptr); + if (format::is_full_projection(&child_projection)) { + // A full child path is a request for its complete physical subtree. Keeping only the + // child group id makes its grandchildren SkipReadingReaders and silently defaults + // STRUCT fields (or breaks ARRAY/MAP shape invariants). + collect_physical_subtree_ids(*child_field, ids); + } else { + ids->insert(child_field->get_column_id()); + collect_projected_ids(**schema_it, &child_projection, *child_field, ids); + } + } + if (schema.kind == ParquetColumnSchemaKind::MAP) { + DORIS_CHECK(!native_field.children.empty()); + // MAP entry existence and offsets are owned by the key stream even for value-only + // projections. Keep the key reader live so the native complex reader can validate + // key/value entry alignment while constructing offsets. + ids->insert(native_field.children[0].get_column_id()); + } +} + +Status append_non_null_dictionary_values(MutableColumnPtr& target, MutableColumnPtr values) { + DORIS_CHECK(target); + DORIS_CHECK(values); + const size_t value_count = values->size(); + if (auto* nullable = check_and_get_column(*target); nullable != nullptr) { + nullable->get_nested_column().insert_range_from(*values, 0, value_count); + auto& null_map = nullable->get_null_map_data(); + null_map.resize_fill(null_map.size() + value_count, 0); + return Status::OK(); + } + target->insert_range_from(*values, 0, value_count); + return Status::OK(); +} + +} // namespace + +NativeColumnReader::NativeColumnReader(const ParquetColumnSchema& schema, + DataTypePtr projected_type, + ParquetColumnReaderProfile profile) + : ParquetColumnReader(schema, std::move(projected_type), profile), + _nested(schema.kind != ParquetColumnSchemaKind::PRIMITIVE) {} + +NativeColumnReader::~NativeColumnReader() { + (void)sync_native_profile(); +} + +Status NativeColumnReader::create( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter, ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + if (reader == nullptr) { + return Status::InvalidArgument("Native parquet reader result is null"); + } + if (file == nullptr || metadata == nullptr) { + return Status::InvalidArgument("Native parquet file context is not initialized"); + } + if (row_group_id < 0 || + row_group_id >= static_cast(metadata->to_thrift().row_groups.size())) { + return Status::InvalidArgument("Invalid native parquet row group {}", row_group_id); + } + const auto& native_schema = metadata->schema(); + if (column_schema.local_id < 0 || column_schema.local_id >= native_schema.size()) { + return Status::InvalidArgument("Invalid native parquet top-level column id {} for {}", + column_schema.local_id, column_schema.name); + } + auto* field = const_cast(native_schema.get_column(column_schema.local_id)); + DORIS_CHECK(field != nullptr); + if (field->name != column_schema.name && + !(field->field_id >= 0 && field->field_id == column_schema.parquet_field_id)) { + return Status::Corruption( + "Native/metadata parquet schema mismatch at column {}: native={}, arrow={}", + column_schema.local_id, field->name, column_schema.name); + } + + auto type = projected_type(column_schema, projection); + std::shared_ptr schema_node; + RETURN_IF_ERROR(build_native_schema_node(type, column_schema, &schema_node)); + std::set projected_ids; + collect_projected_ids(column_schema, projection, *field, &projected_ids); + + auto native_reader = std::unique_ptr( + new NativeColumnReader(column_schema, std::move(type), profile)); + RETURN_IF_ERROR(native_reader->init( + std::move(file), metadata, row_group_id, field, std::move(schema_node), + std::move(projected_ids), selected_ranges, offset_indexes, timezone, io_ctx, + runtime_state, enable_page_cache, page_cache_file_key, enable_dictionary_filter)); + *reader = std::move(native_reader); + return Status::OK(); +} + +Status NativeColumnReader::init( + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + NativeFieldSchema* field, std::shared_ptr schema_node, + std::set projected_column_ids, const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(field != nullptr); + DORIS_CHECK(schema_node != nullptr); + const auto& row_group = metadata->to_thrift().row_groups[row_group_id]; + DORIS_CHECK(row_group.num_rows > 0); + _row_group_rows = row_group.num_rows; + _selected_ranges = selected_ranges; + DORIS_CHECK(!_selected_ranges.empty()); + for (const auto& range : _selected_ranges) { + DORIS_CHECK(range.start >= 0); + DORIS_CHECK(range.length > 0); + DORIS_CHECK(range.start + range.length <= _row_group_rows); + _row_ranges.add(segment_v2::RowRange(range.start, range.start + range.length)); + } + // Offset indexes are immutable row-group metadata owned by ParquetScanScheduler. Sharing them + // avoids retaining the full N-column page-location map once per projected reader. + _offset_indexes = &offset_indexes; + _schema_node = std::move(schema_node); + _projected_column_ids = std::move(projected_column_ids); + _dictionary_filter_enabled = enable_dictionary_filter; + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + const size_t max_buffer_size = std::min(max_group_buffer, max_column_buffer); + RuntimeState* native_runtime_state = runtime_state; + const bool runtime_page_cache_enabled = + runtime_state == nullptr || + runtime_state->query_options().enable_parquet_file_page_cache; + if (runtime_page_cache_enabled != enable_page_cache) { + TQueryOptions query_options = + runtime_state == nullptr ? TQueryOptions() : runtime_state->query_options(); + query_options.__set_enable_parquet_file_page_cache(enable_page_cache); + _page_cache_runtime_state = RuntimeState::create_unique(query_options, TQueryGlobals()); + native_runtime_state = _page_cache_runtime_state.get(); + } + const auto& thrift_metadata = metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + RETURN_IF_ERROR(native::ColumnReader::create( + std::move(file), field, row_group, _row_ranges, timezone, io_ctx, _native_reader, + max_buffer_size, *_offset_indexes, native_runtime_state, false, _projected_column_ids, + _filter_column_ids, page_cache_file_key, compat, + runtime_state != nullptr && runtime_state->enable_strict_mode())); + DORIS_CHECK(_native_reader != nullptr); + _skip_column = _type->create_column(); + return Status::OK(); +} + +Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, MutableColumnPtr& column, + const DataTypePtr& output_type, bool dictionary_ids, + int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(column); + DORIS_CHECK(output_type != nullptr); + DORIS_CHECK(rows_read != nullptr); + *rows_read = 0; + if (rows == 0) { + return Status::OK(); + } + + native::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + ColumnPtr native_column(std::move(column)); + bool eof = false; + int64_t native_calls = 0; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + ++native_calls; + size_t loop_rows = 0; + RETURN_IF_ERROR(_native_reader->read_column_data( + native_column, output_type, _schema_node, filter, + static_cast(rows - *rows_read), &loop_rows, &eof, dictionary_ids)); + if (loop_rows == 0 && !eof) { + // A selected RowRanges plan may reject the current data page completely. V1 advances + // the page cursor and deliberately returns zero rows so the caller can request the + // next page. Bound consecutive empty transitions by the Row Group row count to retain + // a deterministic corruption exit if a decoder ever stops advancing. + if (++consecutive_empty_calls > _row_group_rows + 1) { + column = IColumn::mutate(std::move(native_column)); + return Status::Corruption("Native parquet reader made no progress for column {}", + _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + column = IColumn::mutate(std::move(native_column)); + if (_profile.native_read_calls != nullptr) { + COUNTER_UPDATE(_profile.native_read_calls, native_calls); + } + if (_nested && _profile.nested_batches != nullptr) { + COUNTER_UPDATE(_profile.nested_batches, 1); + } + // Retained-capacity inspection walks the native reader tree. Check it periodically instead of + // on every small batch; row-group destruction is still the hard lifetime bound for scratch. + constexpr size_t SCRATCH_CHECK_BATCH_INTERVAL = 16; + if (++_batches_since_scratch_check >= SCRATCH_CHECK_BATCH_INTERVAL) { + _native_reader->release_batch_scratch(MAX_RETAINED_BATCH_SCRATCH_BYTES); + _batches_since_scratch_check = 0; + } + if (*rows_read != rows) { + return Status::Corruption("Native parquet reader returned {} rows, expected {} for {}", + *rows_read, rows, _name); + } + return Status::OK(); +} + +Status NativeColumnReader::read_with_plain_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, const VExprSPtrs& conjuncts, + int column_id, IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(rows_read != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *rows_read = 0; + *used_filter = false; + if (rows == 0) { + return Status::OK(); + } + + native::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + bool eof = false; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + size_t loop_rows = 0; + IColumn::Filter loop_filter; + bool loop_used = false; + RETURN_IF_ERROR(_native_reader->read_plain_filter( + conjuncts, column_id, filter, static_cast(rows - *rows_read), &loop_filter, + &loop_rows, &eof, &loop_used)); + if (!loop_used) { + if (UNLIKELY(*rows_read != 0)) { + // Footer encoding lists are untrusted. Once a prior page advanced the cursor, a + // typed fallback would restart the request at the wrong row, so reject the file + // instead of terminating the BE or returning shifted results. + return Status::Corruption( + "Parquet PLAIN predicate encoding changed after {} rows for column {}", + *rows_read, _name); + } + row_filter->clear(); + return Status::OK(); + } + row_filter->insert(row_filter->end(), loop_filter.begin(), loop_filter.end()); + if (loop_rows == 0 && !eof) { + if (++consecutive_empty_calls > _row_group_rows + 1) { + return Status::Corruption( + "Native parquet PLAIN predicate made no progress for column {}", _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + if (*rows_read != rows) { + return Status::Corruption( + "Native parquet PLAIN predicate returned {} rows, expected {} for {}", *rows_read, + rows, _name); + } + *used_filter = true; + return Status::OK(); +} + +Status NativeColumnReader::validate_selected_span(int64_t rows) { + DORIS_CHECK(rows >= 0); + while (_selected_range_idx < _selected_ranges.size()) { + const auto& range = _selected_ranges[_selected_range_idx]; + const int64_t range_end = range.start + range.length; + if (_logical_row_position < range_end) { + break; + } + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + return Status::Corruption("Native parquet read past selected ranges for column {}", _name); + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start || + rows > range.start + range.length - _logical_row_position) { + return Status::Corruption( + "Native parquet read [{}, {}) crosses selected range [{}, {}) for column {}", + _logical_row_position, _logical_row_position + rows, range.start, + range.start + range.length, _name); + } + return Status::OK(); +} + +void NativeColumnReader::advance_selected_span(int64_t rows) { + _logical_row_position += rows; + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } +} + +Status NativeColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + RETURN_IF_ERROR(validate_selected_span(rows)); + RETURN_IF_ERROR(read_with_filter(rows, nullptr, false, column, _type, false, rows_read)); + advance_selected_span(*rows_read); + update_reader_read_rows(*rows_read); + return Status::OK(); +} + +Status NativeColumnReader::skip(int64_t rows) { + if (rows <= 0) { + return Status::OK(); + } + DORIS_CHECK(_logical_row_position <= _row_group_rows - rows); + int64_t remaining = rows; + int64_t native_skipped_rows = 0; + while (remaining > 0) { + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + _logical_row_position += remaining; + break; + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start) { + const int64_t gap = std::min(remaining, range.start - _logical_row_position); + _logical_row_position += gap; + remaining -= gap; + continue; + } + const int64_t selected_rows = detail::bounded_native_lazy_skip_rows( + std::min(remaining, range.start + range.length - _logical_row_position)); + _skip_column->clear(); + // Pending skips can span many filtered batches and are replayed for every lazy column. + // Chunking here bounds each dense bitmap while preserving one logical scheduler skip. + _filter_scratch.assign(static_cast(selected_rows), 0); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(selected_rows, _filter_scratch.data(), true, _skip_column, + _type, false, &rows_read)); + DORIS_CHECK(_skip_column->empty()); + DORIS_CHECK(rows_read == selected_rows); + _logical_row_position += rows_read; + native_skipped_rows += rows_read; + remaining -= rows_read; + } + update_reader_skip_rows(native_skipped_rows); + return Status::OK(); +} + +Status NativeColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column) { + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + const size_t old_size = column->size(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, column, _type, + false, &rows_read)); + advance_selected_span(rows_read); + if (column->size() != old_size + selected_rows) { + return Status::Corruption( + "Native parquet selection appended {} rows, expected {} for column {}", + column->size() - old_size, selected_rows, _name); + } + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + return Status::OK(); +} + +Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, + IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + *used_filter = false; + row_filter->clear(); + if (!_dictionary_filter_enabled) { + return Status::OK(); + } + *used_filter = true; + + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + const bool nullable = _type->is_nullable(); + DataTypePtr id_type = std::make_shared(); + if (nullable) { + id_type = make_nullable(id_type); + } + if (!_dictionary_id_column) { + _dictionary_id_column = id_type->create_column(); + } + _dictionary_id_column->clear(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, + _dictionary_id_column, id_type, true, &rows_read)); + advance_selected_span(rows_read); + if (_dictionary_id_column->size() != selected_rows) { + return Status::Corruption( + "Native parquet dictionary reader appended {} rows, expected {} for {}", + _dictionary_id_column->size(), selected_rows, _name); + } + + const ColumnInt32* ids = nullptr; + const NullMap* null_map = nullptr; + if (const auto* nullable_ids = check_and_get_column(*_dictionary_id_column); + nullable_ids != nullptr) { + ids = check_and_get_column(nullable_ids->get_nested_column()); + null_map = &nullable_ids->get_null_map_data(); + } else { + ids = check_and_get_column(*_dictionary_id_column); + } + DORIS_CHECK(ids != nullptr); + + if (!_matched_dictionary_ids) { + _matched_dictionary_ids = ColumnInt32::create(); + } + _matched_dictionary_ids->clear(); + auto& matched_ids = assert_cast(*_matched_dictionary_ids).get_data(); + row_filter->reserve(selected_rows); + const auto& id_data = ids->get_data(); + for (size_t row = 0; row < selected_rows; ++row) { + bool keep = false; + if (null_map == nullptr || (*null_map)[row] == 0) { + const int32_t dictionary_id = id_data[row]; + if (dictionary_id < 0 || + static_cast(dictionary_id) >= dictionary_filter.size()) { + return Status::Corruption( + "Invalid parquet dictionary id {} for column {} with {} entries", + dictionary_id, _name, dictionary_filter.size()); + } + keep = dictionary_filter[static_cast(dictionary_id)] != 0; + if (keep) { + matched_ids.push_back(dictionary_id); + } + } + row_filter->push_back(keep ? 1 : 0); + } + + const auto* matched_id_column = check_and_get_column(*_matched_dictionary_ids); + DORIS_CHECK(matched_id_column != nullptr); + auto string_values = DORIS_TRY( + _native_reader->convert_dict_column_to_string_column(matched_id_column, _type)); + RETURN_IF_ERROR(append_non_null_dictionary_values(column, std::move(string_values))); + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(cast_set(matched_ids.size())); + update_reader_skip_rows(batch_rows - cast_set(matched_ids.size())); + return Status::OK(); +} + +Status NativeColumnReader::select_with_plain_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const VExprSPtrs& conjuncts, int column_id, + IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_plain_filter(batch_rows, filter_data, selected_rows == 0, conjuncts, + column_id, row_filter, &rows_read, used_filter)); + if (!*used_filter) { + return Status::OK(); + } + DORIS_CHECK_EQ(rows_read, batch_rows); + if (row_filter->size() != selected_rows) { + return Status::Corruption( + "Native parquet PLAIN predicate returned {} selected rows, expected {} for {}", + row_filter->size(), selected_rows, _name); + } + advance_selected_span(rows_read); + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + return Status::OK(); +} + +void NativeColumnReader::flush_profile() { + record_page_fragments(sync_native_profile()); +} + +bool NativeColumnReader::crossed_page_since_last_batch() { + if (_native_reader == nullptr) { + return false; + } + const auto stats = _native_reader->column_statistics(); + bool crossed_page = false; + if (stats.leaf_page_read_counters.size() == _batch_leaf_page_read_counters.size()) { + for (size_t leaf = 0; leaf < stats.leaf_page_read_counters.size(); ++leaf) { + crossed_page |= + stats.leaf_page_read_counters[leaf] - _batch_leaf_page_read_counters[leaf] > 1; + } + } else { + // The first snapshot covers the first batch; later snapshots must keep the stable tree shape. + for (const int64_t page_reads : stats.leaf_page_read_counters) { + crossed_page |= page_reads > 1; + } + } + _batch_leaf_page_read_counters = stats.leaf_page_read_counters; + return crossed_page; +} + +Result NativeColumnReader::dictionary_values() { + DORIS_CHECK(_native_reader != nullptr); + return _native_reader->dictionary_values(_type); +} + +void NativeColumnReader::record_page_fragments(int64_t page_fragments) { + if (_profile.native_page_fragments != nullptr) { + COUNTER_UPDATE(_profile.native_page_fragments, page_fragments); + } +} + +int64_t NativeColumnReader::sync_native_profile() { + if (_native_reader == nullptr) { + return 0; + } + const auto stats = _native_reader->column_statistics(); + const auto& reported = _reported_native_stats; + if (_profile.decompress_time != nullptr) { + COUNTER_UPDATE(_profile.decompress_time, stats.decompress_time - reported.decompress_time); + } + if (_profile.decompress_count != nullptr) { + COUNTER_UPDATE(_profile.decompress_count, stats.decompress_cnt - reported.decompress_cnt); + } + if (_profile.decode_header_time != nullptr) { + COUNTER_UPDATE(_profile.decode_header_time, + stats.decode_header_time - reported.decode_header_time); + } + if (_profile.decode_value_time != nullptr) { + COUNTER_UPDATE(_profile.decode_value_time, + stats.decode_value_time - reported.decode_value_time); + } + if (_profile.decode_dictionary_time != nullptr) { + COUNTER_UPDATE(_profile.decode_dictionary_time, + stats.decode_dict_time - reported.decode_dict_time); + } + if (_profile.decode_level_time != nullptr) { + COUNTER_UPDATE(_profile.decode_level_time, + stats.decode_level_time - reported.decode_level_time); + } + if (_profile.decode_null_map_time != nullptr) { + COUNTER_UPDATE(_profile.decode_null_map_time, + stats.decode_null_map_time - reported.decode_null_map_time); + } + if (_profile.materialization_time != nullptr) { + COUNTER_UPDATE(_profile.materialization_time, + stats.materialization_time - reported.materialization_time); + } + if (_profile.hybrid_selection_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_batches, + stats.hybrid_selection_batches - reported.hybrid_selection_batches); + } + if (_profile.hybrid_selection_ranges != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_ranges, + stats.hybrid_selection_ranges - reported.hybrid_selection_ranges); + } + if (_profile.hybrid_selection_null_fallback_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_null_fallback_batches, + stats.hybrid_selection_null_fallback_batches - + reported.hybrid_selection_null_fallback_batches); + } + if (_profile.page_index_read_calls != nullptr) { + COUNTER_UPDATE(_profile.page_index_read_calls, + stats.page_index_read_calls - reported.page_index_read_calls); + } + if (_profile.skip_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.skip_page_header_count, + stats.skip_page_header_num - reported.skip_page_header_num); + } + if (_profile.parse_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.parse_page_header_count, + stats.parse_page_header_num - reported.parse_page_header_num); + } + if (_profile.read_page_header_time != nullptr) { + COUNTER_UPDATE(_profile.read_page_header_time, + stats.read_page_header_time - reported.read_page_header_time); + } + const int64_t page_read_delta = stats.page_read_counter - reported.page_read_counter; + if (_profile.page_read_count != nullptr) { + COUNTER_UPDATE(_profile.page_read_count, page_read_delta); + } + if (_profile.page_cache_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_write_count, + stats.page_cache_write_counter - reported.page_cache_write_counter); + } + if (_profile.page_cache_compressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_write_count, + stats.page_cache_compressed_write_counter - + reported.page_cache_compressed_write_counter); + } + if (_profile.page_cache_decompressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_write_count, + stats.page_cache_decompressed_write_counter - + reported.page_cache_decompressed_write_counter); + } + if (_profile.page_cache_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_hit_count, + stats.page_cache_hit_counter - reported.page_cache_hit_counter); + } + if (_profile.page_cache_miss_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_miss_count, + stats.page_cache_missing_counter - reported.page_cache_missing_counter); + } + if (_profile.page_cache_compressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_hit_count, + stats.page_cache_compressed_hit_counter - + reported.page_cache_compressed_hit_counter); + } + if (_profile.page_cache_decompressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_hit_count, + stats.page_cache_decompressed_hit_counter - + reported.page_cache_decompressed_hit_counter); + } + _reported_native_stats = stats; + return page_read_delta; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h new file mode 100644 index 00000000000000..4e504d3b2d7492 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format_v2/column_data.h" +#include "format_v2/parquet/native_schema_node.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/native/column_reader.h" + +namespace doris { +class RuntimeState; +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet { + +class NativeParquetMetadata; + +namespace detail { +inline constexpr int64_t MAX_NATIVE_LAZY_SKIP_ROWS = std::numeric_limits::max(); + +inline int64_t bounded_native_lazy_skip_rows(int64_t rows) { + return std::min(rows, MAX_NATIVE_LAZY_SKIP_ROWS); +} +} // namespace detail + +// Production adapter from FileScannerV2's selection-oriented reader contract to Doris' native +// Parquet page/encoding reader. The owned native reader decodes page bytes directly into the final +// Doris column. It never creates an Arrow Array/Builder, DecodedColumnView, or intermediate nested +// values_column. +// +// Cursor contract: +// - read(rows) consumes and appends exactly `rows` logical top-level rows; +// - select(selection, batch_rows) consumes `batch_rows` and appends only selected rows; +// - skip(rows) consumes `rows` with an all-false FilterMap and appends no payload; +// - one adapter lives for one top-level column in one Row Group, so decoder dictionaries, +// decompression buffers, level buffers, converters, and destination capacity survive adaptive +// batch-size changes. +class NativeColumnReader final : public ParquetColumnReader { +public: + static Status create(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, io::FileReaderSPtr file, + const NativeParquetMetadata* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, + RuntimeState* runtime_state, bool enable_page_cache, + const std::string& page_cache_file_key, bool enable_dictionary_filter, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + ~NativeColumnReader() override; + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + Status select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, + MutableColumnPtr& column) override; + Status select_with_dictionary_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, IColumn::Filter* row_filter, + bool* used_filter) override; + Status select_with_plain_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, const VExprSPtrs& conjuncts, int column_id, + IColumn::Filter* row_filter, bool* used_filter) override; + void flush_profile() override; + bool crossed_page_since_last_batch() override; + Result dictionary_values() override; + +private: + NativeColumnReader(const ParquetColumnSchema& schema, DataTypePtr projected_type, + ParquetColumnReaderProfile profile); + + Status init(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + NativeFieldSchema* field, std::shared_ptr schema_node, + std::set projected_column_ids, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter); + + Status read_with_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + MutableColumnPtr& column, const DataTypePtr& output_type, + bool dictionary_ids, int64_t* rows_read); + Status read_with_plain_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + const VExprSPtrs& conjuncts, int column_id, + IColumn::Filter* row_filter, int64_t* rows_read, + bool* used_filter); + int64_t sync_native_profile(); + void record_page_fragments(int64_t page_fragments); + Status validate_selected_span(int64_t rows); + void advance_selected_span(int64_t rows); + + // Native ParquetColumnReader keeps a reference to RowRanges; declare it before the reader. + segment_v2::RowRanges _row_ranges; + std::set _projected_column_ids; + std::set _filter_column_ids; + const std::unordered_map* _offset_indexes = nullptr; + std::shared_ptr _schema_node; + std::unique_ptr _native_reader; + std::unique_ptr _page_cache_runtime_state; + std::vector _selected_ranges; + size_t _selected_range_idx = 0; + int64_t _logical_row_position = 0; + int64_t _row_group_rows = 0; + + bool _dictionary_filter_enabled = false; + bool _nested = false; + // The native tree exposes cumulative statistics. Keep the last reported snapshot so each + // FileScannerV2 batch contributes only its delta to RuntimeProfile. + native::ColumnReader::ColumnStatistics _reported_native_stats; + // Page-crossing is sampled at every scheduler batch, independently of amortized profile flushes. + std::vector _batch_leaf_page_read_counters; + std::vector _filter_scratch; + size_t _batches_since_scratch_check = 0; + MutableColumnPtr _skip_column; + MutableColumnPtr _dictionary_id_column; + MutableColumnPtr _matched_dictionary_ids; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp b/be/src/format_v2/parquet/reader/nested_column_materializer.cpp deleted file mode 100644 index e06b7eaaf317e7..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/nested_column_materializer.h" - -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" - -namespace doris::format::parquet { - -ColumnArray* array_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnMap* map_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return &nullable_column->get_null_map_data(); - } - return nullptr; -} - -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts) { - offsets.reserve(offsets.size() + entry_counts.size()); - uint64_t current_offset = offsets.empty() ? 0 : offsets.back(); - for (const auto entry_count : entry_counts) { - current_offset += entry_count; - offsets.push_back(current_offset); - } -} - -void append_parent_nulls(NullMap* dst, const NullMap& src) { - if (dst == nullptr) { - return; // target column is not nullable; no null marker is needed - } - dst->insert(src.begin(), src.end()); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.h b/be/src/format_v2/parquet/reader/nested_column_materializer.h deleted file mode 100644 index 90fac01eb2f5e5..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.h +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "core/column/column.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_struct.h" - -namespace doris::format::parquet { - -// ============================================================================ -// ============================================================================ - -ColumnArray* array_column_from_output(MutableColumnPtr& column); - -ColumnMap* map_column_from_output(MutableColumnPtr& column); - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column); - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column); - -// offsets[i] = offsets[i-1] + entry_counts[i]. -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts); - -void append_parent_nulls(NullMap* dst, const NullMap& src); - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp deleted file mode 100644 index fd261ef5219d27..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp +++ /dev/null @@ -1,803 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "core/string_ref.h" -#include "runtime/runtime_profile.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -DecodedTimeUnit decoded_time_unit(ParquetTimeUnit time_unit) { - switch (time_unit) { - case ParquetTimeUnit::MILLIS: - return DecodedTimeUnit::MILLIS; - case ParquetTimeUnit::MICROS: - return DecodedTimeUnit::MICROS; - case ParquetTimeUnit::NANOS: - return DecodedTimeUnit::NANOS; - case ParquetTimeUnit::UNKNOWN: - default: - return DecodedTimeUnit::UNKNOWN; - } -} - -Status decoded_fixed_value_size(const std::string& column_name, DecodedValueKind value_kind, - size_t* value_size) { - switch (value_kind) { - case DecodedValueKind::BOOL: - *value_size = sizeof(bool); - return Status::OK(); - case DecodedValueKind::INT32: - *value_size = sizeof(int32_t); - return Status::OK(); - case DecodedValueKind::UINT32: - *value_size = sizeof(uint32_t); - return Status::OK(); - case DecodedValueKind::INT64: - *value_size = sizeof(int64_t); - return Status::OK(); - case DecodedValueKind::UINT64: - *value_size = sizeof(uint64_t); - return Status::OK(); - case DecodedValueKind::INT96: - *value_size = 12; - return Status::OK(); - case DecodedValueKind::FLOAT: - *value_size = sizeof(float); - return Status::OK(); - case DecodedValueKind::DOUBLE: - *value_size = sizeof(double); - return Status::OK(); - case DecodedValueKind::BINARY: - case DecodedValueKind::FIXED_BINARY: - return Status::InvalidArgument("Parquet binary value kind has no fixed value size for {}", - column_name); - } - return Status::InternalError("Unknown decoded value kind for column {}", column_name); -} - -Status get_binary_chunks(const std::string& column_name, - ::parquet::internal::RecordReader& record_reader, - std::vector>* chunks) { - if (auto* dictionary_reader = - dynamic_cast<::parquet::internal::DictionaryRecordReader*>(&record_reader); - dictionary_reader != nullptr) { - auto chunked = dictionary_reader->GetResult(); - if (chunked == nullptr) { - return Status::Corruption( - "Parquet dictionary record reader returned null result for column {}", - column_name); - } - *chunks = chunked->chunks(); - return Status::OK(); - } - auto* binary_reader = dynamic_cast<::parquet::internal::BinaryRecordReader*>(&record_reader); - if (binary_reader == nullptr) { - return Status::InternalError("Parquet binary record reader is not available for column {}", - column_name); - } - *chunks = binary_reader->GetBuilderChunks(); - return Status::OK(); -} - -Status append_dictionary_binary_values(const std::string& column_name, - const ::arrow::DictionaryArray& dictionary_array, - std::vector* values) { - DORIS_CHECK(values != nullptr); - const auto& dictionary = dictionary_array.dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - column_name); - } - auto append_value = [&](int64_t dictionary_index) -> Status { - if (dictionary_index < 0 || dictionary_index >= dictionary->length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(dictionary.get())) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(dictionary.get())) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back( - reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); - }; - for (int64_t row_idx = 0; row_idx < dictionary_array.length(); ++row_idx) { - if (dictionary_array.IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - RETURN_IF_ERROR(append_value(dictionary_array.GetValueIndex(row_idx))); - } - return Status::OK(); -} - -Status build_binary_values(const std::string& column_name, - const std::vector>& chunks, - int64_t records_read, const NullMap* null_map, - bool read_dense_for_nullable, std::vector* binary_values) { - std::vector compact_values; - auto* values = read_dense_for_nullable ? &compact_values : binary_values; - values->reserve(records_read); - for (const auto& chunk : chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < binary_array->length(); ++row_idx) { - if (binary_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(row_idx, &length); - values->emplace_back(reinterpret_cast(value), length); - } - } else if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < fixed_array->length(); ++row_idx) { - if (fixed_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(row_idx)), - fixed_array->byte_width()); - } - } else if (auto* dictionary_array = dynamic_cast<::arrow::DictionaryArray*>(chunk.get())) { - RETURN_IF_ERROR( - append_dictionary_binary_values(column_name, *dictionary_array, values)); - } else { - return Status::InternalError("Unexpected Arrow binary array type for column {}", - column_name); - } - } - if (read_dense_for_nullable) { - if (null_map == nullptr || null_map->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid dense nullable parquet null map for column {}: rows={}, null_map={}", - column_name, records_read, null_map == nullptr ? 0 : null_map->size()); - } - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (compact_values.size() != static_cast(non_null_count)) { - return Status::Corruption( - "Invalid dense nullable parquet binary values for column {}: values={}, " - "records={}, nulls={}", - column_name, compact_values.size(), records_read, - records_read - non_null_count); - } - binary_values->reserve(records_read); - size_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - binary_values->emplace_back(static_cast(nullptr), 0); - continue; - } - binary_values->emplace_back(compact_values[value_idx++]); - } - return Status::OK(); - } - if (binary_values->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid parquet binary record read result for column {}: rows={}, records={}", - column_name, binary_values->size(), records_read); - } - return Status::OK(); -} - -float half_to_float(uint16_t value) { - const uint32_t sign = (value & 0x8000U) << 16; - const uint32_t exponent = (value & 0x7C00U) >> 10; - const uint32_t mantissa = value & 0x03FFU; - - if (exponent == 0) { - if (mantissa == 0) { - return std::bit_cast(sign); - } - const float subnormal = std::ldexp(static_cast(mantissa), -24); - return sign == 0 ? subnormal : -subnormal; - } - if (exponent == 0x1FU) { - return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); - } - return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); -} - -Status build_float16_values(const std::string& column_name, - const ParquetTypeDescriptor& type_descriptor, - const std::vector& binary_values, int64_t row_count, - std::vector* float_values) { - if (type_descriptor.fixed_length != 2) { - return Status::Corruption("Invalid parquet Float16 length for column {}: {}", column_name, - type_descriptor.fixed_length); - } - if (binary_values.size() != static_cast(row_count)) { - return Status::Corruption( - "Invalid parquet Float16 value count for column {}: values={}, rows={}", - column_name, binary_values.size(), row_count); - } - float_values->resize(static_cast(row_count)); - for (int64_t row = 0; row < row_count; ++row) { - const auto& binary_value = binary_values[static_cast(row)]; - if (binary_value.data == nullptr && binary_value.size == 0) { - (*float_values)[static_cast(row)] = 0; - continue; - } - if (binary_value.data == nullptr || binary_value.size != 2) { - return Status::Corruption( - "Invalid parquet Float16 value for column {} at row {}: data={}, size={}", - column_name, row, binary_value.data == nullptr ? "null" : "non-null", - binary_value.size); - } - uint16_t raw_value = 0; - std::memcpy(&raw_value, binary_value.data, sizeof(raw_value)); - (*float_values)[static_cast(row)] = half_to_float(raw_value); - } - return Status::OK(); -} - -} // namespace - -Status ParquetLeafReader::collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - batch->_values_written = record_reader.values_written(); - - if (!batch->is_binary_value()) { - batch->_fixed_values = record_reader.values(); - return Status::OK(); - } - - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &batch->_binary_chunks)); - batch->_values_written = 0; - for (const auto& chunk : batch->_binary_chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", _name); - } - batch->_values_written += chunk->length(); - } - return Status::OK(); -} - -Status ParquetLeafReader::collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - - // Arrow's RecordReader::Reset() does not reset ByteArray/FLBA builders. GetBuilderChunks() - // (or DictionaryRecordReader::GetResult()) is the documented reset operation and must be - // called before the next ReadRecords(). Otherwise a levels-only skip followed by a normal read - // observes values from both batches; for example, skipping ARRAY ["a", "b"] and then - // reading ["c"] would report one current level but three values. Release the chunks here and - // let the temporary vector destroy them immediately. We deliberately do not inspect or copy - // their payload into a Doris Column, so the levels-only contract still avoids Doris-side value - // materialization. - if (batch->is_binary_value()) { - std::vector> discarded_chunks; - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &discarded_chunks)); - } - - // COUNT(col) and nested skip only need top-level shape. Fixed-width values remain owned by the - // RecordReader and are cleared by Reset(); binary values were released above solely to reset - // the Arrow builder. - batch->_values_written = 0; - return Status::OK(); -} - -// - FLOAT16: binary -> half_to_float -> float_values -Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, MutableColumnPtr& column) const { - std::vector binary_values; - std::vector spaced_values; - std::vector float_values; - DecodedColumnView view; - view.value_kind = batch._value_kind; - view.time_unit = decoded_time_unit(_type_descriptor.time_unit); - view.row_count = row_count; - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.decimal_precision = _type_descriptor.decimal_precision; - view.decimal_scale = _type_descriptor.decimal_scale; - view.fixed_length = _type_descriptor.fixed_length; - view.timestamp_is_adjusted_to_utc = _type_descriptor.timestamp_is_adjusted_to_utc; - view.timezone = _timezone; - view.enable_strict_mode = _enable_strict_mode; - view.null_map = null_map == nullptr || null_map->empty() ? nullptr : null_map->data(); - const bool read_dense_for_nullable = batch._read_dense_for_nullable && view.null_map != nullptr; - - if (_type_descriptor.extra_type_info == ParquetExtraTypeInfo::FLOAT16) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - RETURN_IF_ERROR(build_float16_values(_name, _type_descriptor, binary_values, row_count, - &float_values)); - view.value_kind = DecodedValueKind::FLOAT; - view.values = reinterpret_cast(float_values.data()); - } else if (batch.is_binary_value()) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - view.binary_values = &binary_values; - } else if (read_dense_for_nullable) { - RETURN_IF_ERROR(build_spaced_fixed_values(batch, row_count, null_map, &spaced_values)); - view.values = spaced_values.data(); - } else { - view.values = batch._fixed_values; - } - - if (_decoded_value_appender != nullptr) { - return _decoded_value_appender(column, view); - } - - { - SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& tmp_null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = tmp_null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - tmp_null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(tmp_null_map.data() + old_null_map_size, 0, - tmp_null_map.size() - old_null_map_size); - } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); - } - } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); - } - } - return Status::OK(); -} - -bool ParquetLeafBatch::is_binary_value() const { - return _value_kind == DecodedValueKind::BINARY || _value_kind == DecodedValueKind::FIXED_BINARY; -} - -Status ParquetLeafReader::build_spaced_fixed_values(const ParquetLeafBatch& batch, - int64_t row_count, const NullMap* null_map, - std::vector* spaced_values) const { - DORIS_CHECK(null_map != nullptr); - DORIS_CHECK(spaced_values != nullptr); - size_t value_size = 0; - RETURN_IF_ERROR(decoded_fixed_value_size(_name, batch._value_kind, &value_size)); - spaced_values->resize(static_cast(row_count) * value_size); - const auto non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (batch._values_written != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet values for column {}: values={}, records={}, " - "nulls={}", - _name, batch._values_written, row_count, row_count - non_null_count); - } - auto* dst = spaced_values->data(); - int64_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < row_count; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - continue; // NULL row: skip it and keep the target slot zeroed - } - std::memcpy(dst + static_cast(record_idx) * value_size, - batch._fixed_values + static_cast(value_idx) * value_size, value_size); - ++value_idx; - } - return Status::OK(); -} - -ParquetLeafReader::ParquetLeafReader( - const ::parquet::ColumnDescriptor* descriptor, ParquetTypeDescriptor type_descriptor, - DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile, const cctz::time_zone* timezone, - bool enable_strict_mode, - std::function decoded_value_appender) - : _descriptor(descriptor), - _type_descriptor(type_descriptor), - _type(std::move(type)), - _name(std::move(name)), - _record_reader(std::move(record_reader)), - _profile(profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _decoded_value_appender(std::move(decoded_value_appender)) {} - -Status ParquetLeafReader::read_batch(int64_t batch_rows, ParquetLeafBatch* batch, - int64_t* rows_read) const { - if (batch == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet leaf batch result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - *rows_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet records for column {}: {}", _name, - e.what()); - } - if (*rows_read < 0 || *rows_read > batch_rows) { - return Status::Corruption("Invalid parquet record read result for column {}: {}", _name, - *rows_read); - } - return collect_batch(*_record_reader, batch); -} - -Status ParquetLeafReader::build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const { - if (_descriptor->max_definition_level() == 0) { - return Status::OK(); - } - auto* def_levels = batch.def_levels(); - if (def_levels == nullptr && records_read > 0) { - return Status::Corruption( - "Parquet record reader returned null definition levels for nullable column {}", - _name); - } - const int16_t max_definition_level = _descriptor->max_definition_level(); - null_map->resize(records_read); - auto* __restrict dst = null_map->data(); - const auto* __restrict src = def_levels; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - dst[record_idx] = src[record_idx] != max_definition_level; - } - return Status::OK(); -} - -Status ParquetLeafReader::read_nested_batch(int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - ParquetLeafBatch leaf_batch; - int64_t records_read = 0; - RETURN_IF_ERROR(read_batch(batch_rows, &leaf_batch, &records_read)); - return build_nested_batch_from_leaf_batch(leaf_batch, records_read, value_slot_definition_level, - batch, value_slot_repetition_level); -} - -Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - int64_t records_read = 0; - ParquetLeafBatch leaf_batch; - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - records_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } - if (records_read < 0 || records_read > batch_rows) { - return Status::Corruption("Invalid parquet level read result for column {}: {}", _name, - records_read); - } - RETURN_IF_ERROR(collect_levels_batch(*_record_reader, &leaf_batch)); - return build_nested_levels_batch_from_leaf_batch(leaf_batch, records_read, batch); -} - -Status ParquetLeafReader::build_nested_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar batch is null for column {}", _name); - } - *batch = ParquetNestedScalarBatch(); - batch->value_slot_definition_level = value_slot_definition_level; - batch->value_slot_repetition_level = value_slot_repetition_level; - - batch->records_read = records_read; - if (_type->is_nullable() && leaf_batch.read_dense_for_nullable()) { - return Status::NotSupported( - "Dense nullable parquet nested reader is not supported for column {}", _name); - } - batch->levels_written = leaf_batch.consumed_level_count(); - const int64_t values_written = leaf_batch.values_written(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - if (batch->levels_written == 0 && batch->records_read > 0 && - values_written == batch->records_read && _descriptor->max_definition_level() == 0 && - _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read || values_written < 0 || - values_written > batch->levels_written) { - return Status::Corruption( - "Invalid nested parquet read result for column {}: rows={}, levels={}, values={}", - _name, batch->records_read, batch->levels_written, values_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - - const int16_t leaf_definition_level = _descriptor->max_definition_level(); - // Arrow's RecordReader may emit value placeholders for null ancestors that are below the - // Doris materialization threshold. Those slots must still advance the payload value index; - // otherwise the next defined child level points at the placeholder instead of its real value. - auto count_value_slots = [&](int16_t slot_definition_level) { - int64_t slot_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] >= slot_definition_level && - batch->rep_levels[level_idx] <= value_slot_repetition_level) { - ++slot_count; - } - } - return slot_count; - }; - - const int64_t value_slot_count = count_value_slots(value_slot_definition_level); - int16_t payload_slot_definition_level = value_slot_definition_level; - int64_t payload_value_slot_count = value_slot_count; - while (payload_slot_definition_level > 0 && payload_value_slot_count < values_written) { - --payload_slot_definition_level; - payload_value_slot_count = count_value_slots(payload_slot_definition_level); - } - - int64_t leaf_value_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < value_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - if (batch->def_levels[level_idx] == leaf_definition_level) { - ++leaf_value_count; - } - } - - enum class ValueLayout { LEVELS, VALUE_SLOTS, LEAF_VALUES, PAYLOAD_VALUE_SLOTS }; - ValueLayout value_layout = ValueLayout::LEAF_VALUES; - if (values_written == batch->levels_written) { - value_layout = ValueLayout::LEVELS; - } else if (values_written == value_slot_count) { - value_layout = ValueLayout::VALUE_SLOTS; - } else if (values_written == leaf_value_count) { - value_layout = ValueLayout::LEAF_VALUES; - } else if (values_written == payload_value_slot_count) { - value_layout = ValueLayout::PAYLOAD_VALUE_SLOTS; - } else { - return Status::Corruption( - "Nested parquet reader returned inconsistent value count for column {}: values={}, " - "levels={}, slots={}, leaf_values={}, payload_slots={}, " - "payload_slot_definition_level={}", - _name, values_written, batch->levels_written, value_slot_count, leaf_value_count, - payload_value_slot_count, payload_slot_definition_level); - } - - batch->value_indices.resize(static_cast(batch->levels_written), -1); - NullMap value_nulls(static_cast(values_written), 1); - int64_t value_idx = 0; - const int16_t decoded_slot_definition_level = value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS - ? payload_slot_definition_level - : value_slot_definition_level; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < decoded_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - const bool has_leaf_value = batch->def_levels[level_idx] == leaf_definition_level; - int64_t decoded_value_idx = -1; - if (value_layout == ValueLayout::LEVELS) { - decoded_value_idx = level_idx; - } else if (value_layout == ValueLayout::VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else if (value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else { - if (!has_leaf_value) { - continue; - } - decoded_value_idx = value_idx++; - } - DORIS_CHECK(decoded_value_idx >= 0); - DORIS_CHECK(decoded_value_idx < values_written); - if (has_leaf_value) { - batch->value_indices[static_cast(level_idx)] = decoded_value_idx; - value_nulls[static_cast(decoded_value_idx)] = 0; - } - } - if (value_layout != ValueLayout::LEVELS && value_idx != values_written) { - return Status::Corruption( - "Nested parquet reader value cursor stopped early for column {}: values={}, " - "visited={}", - _name, values_written, value_idx); - } - - const auto value_type = remove_nullable(_type); - batch->values_column = value_type->create_column(); - if (values_written > 0) { - ParquetLeafReader value_reader(_descriptor, _type_descriptor, value_type, _name, - _record_reader, _profile, _timezone, _enable_strict_mode); - RETURN_IF_ERROR(value_reader.append_values(leaf_batch, values_written, &value_nulls, - batch->values_column)); - } - return Status::OK(); -} - -Status ParquetLeafReader::build_nested_levels_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - *batch = ParquetNestedScalarBatch(); - batch->records_read = records_read; - batch->levels_written = leaf_batch.consumed_level_count(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - - // Required flat leaves do not have physical def/rep level buffers. Synthesize one level slot - // per top-level row so the COUNT(col) aggregation code can use the same shape loop. - if (batch->levels_written == 0 && batch->records_read > 0 && - _descriptor->max_definition_level() == 0 && _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read) { - return Status::Corruption( - "Invalid nested parquet levels result for column {}: rows={}, levels={}", _name, - batch->records_read, batch->levels_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - return Status::OK(); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h deleted file mode 100644 index b396b35fd1f32c..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h +++ /dev/null @@ -1,173 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/status.h" -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace arrow { -class Array; -} // namespace arrow - -namespace doris::format::parquet { - -struct ParquetLeafReaderTestAccess; - -// Read result for a nested scalar leaf, separating Dremel-encoded shape from actual values. -// The COUNT(col) aggregation fast path consumes only records_read, levels_written, def_levels, and rep_levels. -// That path does not populate value_indices or values_column, so callers must not call build_nested_column() afterwards. -struct ParquetNestedScalarBatch { - int64_t records_read = 0; - int64_t levels_written = 0; - int16_t value_slot_definition_level = 0; - int16_t value_slot_repetition_level = std::numeric_limits::max(); - std::vector def_levels; - std::vector rep_levels; - std::vector value_indices; - MutableColumnPtr values_column; - - bool empty() const { return levels_written == 0; } -}; - -class ParquetLeafBatch { -public: - int64_t consumed_level_count() const { return _consumed_level_count; } - int64_t decoded_level_count() const { return _decoded_level_count; } - int64_t values_written() const { return _values_written; } - bool read_dense_for_nullable() const { return _read_dense_for_nullable; } - const int16_t* def_levels() const { return _def_levels; } - const int16_t* rep_levels() const { return _rep_levels; } - const std::vector>& binary_chunks() const { - return _binary_chunks; - } - -private: - friend class ParquetLeafReader; - - bool is_binary_value() const; - - DecodedValueKind _value_kind = DecodedValueKind::INT32; - int64_t _consumed_level_count = 0; - int64_t _decoded_level_count = 0; - int64_t _values_written = 0; - const int16_t* _def_levels = nullptr; - const int16_t* _rep_levels = nullptr; - const uint8_t* _fixed_values = nullptr; - bool _read_dense_for_nullable = false; - std::vector> _binary_chunks; -}; - -// read_batch() -> build_null_map() + append_values() -// read_nested_batch() -class ParquetLeafReader { -public: - ParquetLeafReader(const ::parquet::ColumnDescriptor* descriptor, - ParquetTypeDescriptor type_descriptor, DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile = {}, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - std::function - decoded_value_appender = nullptr); - - Status read_batch(int64_t batch_rows, ParquetLeafBatch* batch, int64_t* rows_read) const; - - Status build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const; - - Status append_values(const ParquetLeafBatch& batch, int64_t row_count, const NullMap* null_map, - MutableColumnPtr& column) const; - - // LEVELS / VALUE_SLOTS / LEAF_VALUES / PAYLOAD_VALUE_SLOTS. - Status read_nested_batch( - int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level = std::numeric_limits::max()) const; - - // COUNT(col) and nested-skip shape-only read path. It still calls Arrow - // RecordReader::ReadRecords() to advance the Parquet cursor and obtain def/rep levels, but - // Doris only copies levels: - // - it does not build value_indices or values_column - // - it does not enter DataTypeSerde::read_column_from_decoded_values() - // - for Binary/FLBA, it releases and immediately discards Arrow builder chunks because that is - // the RecordReader's required reset operation; it never copies them into a Doris Column - // This lets COUNT(col) on MAP/ARRAY/STRUCT evaluate top-level NULL state and lets skip advance - // nested shape without Doris-side STRING/BINARY materialization. Arrow RecordReader does not - // expose a public levels-only API, so ReadRecords may still perform required page decoding. - Status read_nested_levels_batch(int64_t batch_rows, ParquetNestedScalarBatch* batch) const; - -private: - friend struct ParquetLeafReaderTestAccess; - - Status collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - // Levels-only variant of collect_batch(). It snapshots only def/rep level state and does not - // expose value buffers. Binary chunks are released only to reset Arrow's builder and are - // immediately discarded. Used by COUNT(col) and nested skip. - Status collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - Status build_spaced_fixed_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, - std::vector* spaced_values) const; - - Status build_nested_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const; - Status build_nested_levels_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - ParquetNestedScalarBatch* batch) const; - - const ::parquet::ColumnDescriptor* _descriptor = - nullptr; // Arrow column descriptor (physical_type, max_dl, max_rl) - ParquetTypeDescriptor - _type_descriptor; // type encoding information (decimal precision, timestamp unit, etc.) - DataTypePtr _type; // Doris target type - std::string _name; // column name for error messages - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader (shared ownership) - ParquetColumnReaderProfile _profile; // profile counters - const cctz::time_zone* _timezone = nullptr; // timezone for timestamp conversion - bool _enable_strict_mode = false; // strict mode for type mismatch errors - std::function _decoded_value_appender; -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp index 4e9a363b13c7cb..dcd601b003ae3c 100644 --- a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp @@ -20,6 +20,7 @@ #include "core/assert_cast.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" +#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" namespace doris::format::parquet { diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp deleted file mode 100644 index 520a3cb1e62e3b..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp +++ /dev/null @@ -1,584 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/scalar_column_reader.h" - -#include -#include -#include - -#include -#include -#include - -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -class ParquetNestedScalarValueCursor { -public: - explicit ParquetNestedScalarValueCursor(const ParquetNestedScalarBatch* batch) { reset(batch); } - - void reset(const ParquetNestedScalarBatch* batch) { - DORIS_CHECK(batch != nullptr); - _batch = batch; - } - - Status value_index(const std::string& column_name, int64_t level_idx, int64_t* value_idx) { - DORIS_CHECK(_batch != nullptr); - DORIS_CHECK(value_idx != nullptr); - DORIS_CHECK(level_idx < _batch->levels_written); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(static_cast(level_idx) < _batch->value_indices.size()); - const int64_t computed_value_idx = _batch->value_indices[static_cast(level_idx)]; - if (computed_value_idx < 0) { - return Status::Corruption("Nested parquet value is absent for column {}", column_name); - } - DORIS_CHECK(_batch->values_column.get() != nullptr); - if (computed_value_idx >= _batch->values_column->size()) { - return Status::Corruption("Nested parquet value index is out of range for column {}", - column_name); - } - *value_idx = computed_value_idx; - return Status::OK(); - } - -private: - const ParquetNestedScalarBatch* _batch = nullptr; -}; - -Status append_scalar_batch_value(const ScalarColumnReader& column_reader, - const ParquetNestedScalarBatch& batch, int64_t level_idx, - ParquetNestedScalarValueCursor* value_cursor, - MutableColumnPtr& column) { - DORIS_CHECK(value_cursor != nullptr); - int64_t value_idx = -1; - RETURN_IF_ERROR(value_cursor->value_index(column_reader.name(), level_idx, &value_idx)); - auto* nullable_column = check_and_get_column(*column); - if (nullable_column != nullptr) { - nullable_column->get_nested_column().insert_from(*batch.values_column, - static_cast(value_idx)); - nullable_column->get_null_map_data().push_back(0); - return Status::OK(); - } - column->insert_from(*batch.values_column, static_cast(value_idx)); - return Status::OK(); -} - -Status append_arrow_binary_dictionary_value(const std::string& column_name, - const ::arrow::Array& dictionary, - int64_t dictionary_index, - std::vector* values) { - DORIS_CHECK(values != nullptr); - if (dictionary_index < 0 || dictionary_index >= dictionary.length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast(&dictionary)) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast(&dictionary)) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); -} - -} // namespace - -ScalarColumnReader::ScalarColumnReader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile profile) - : ParquetColumnReader(column_schema, column_schema.type, profile), - _descriptor(column_schema.descriptor), - _type_descriptor(column_schema.type_descriptor), - _record_reader(std::move(record_reader)), - _page_skip_plan(page_skip_plan), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _nested_batch(std::make_unique()) {} - -ScalarColumnReader::~ScalarColumnReader() = default; - -Status ScalarColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet column read result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - auto reader = leaf_reader(); - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(reader.read_batch(rows, &leaf_batch, rows_read)); - - NullMap null_map; - RETURN_IF_ERROR(reader.build_null_map(leaf_batch, *rows_read, &null_map)); - const auto value_kind = decoded_value_kind(_type_descriptor); - const bool is_binary_value = - value_kind == DecodedValueKind::BINARY || value_kind == DecodedValueKind::FIXED_BINARY; - if (!is_binary_value && leaf_batch.read_dense_for_nullable() && !null_map.empty()) { - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map.data()), null_map.size())); - const int64_t null_count = *rows_read - non_null_count; - if (leaf_batch.values_written() != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet record read result for column {}: values={}, " - "records={}, nulls={}", - _name, leaf_batch.values_written(), *rows_read, null_count); - } - } else if (!is_binary_value && !leaf_batch.read_dense_for_nullable() && - leaf_batch.values_written() != *rows_read) { - return Status::Corruption( - "Invalid parquet record read result for column {}: values={}, records={}", _name, - leaf_batch.values_written(), *rows_read); - } - - RETURN_IF_ERROR(reader.append_values(leaf_batch, *rows_read, &null_map, column)); - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::skip_records(int64_t rows) { - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - if (rows <= 0) { - return Status::OK(); - } - int64_t skipped_rows = 0; - SCOPED_TIMER(_profile.arrow_skip_records_time); - try { - _record_reader->Reset(); - while (skipped_rows < rows) { - const int64_t skipped = _record_reader->SkipRecords(rows - skipped_rows); - if (skipped <= 0) { - return Status::Corruption( - "Failed to skip parquet records for column {}: skipped {} of {} rows", - _name, skipped_rows, rows); - } - skipped_rows += skipped; - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -int64_t ScalarColumnReader::page_filtered_rows_to_skip(int64_t rows) const { - if (_page_skip_plan == nullptr || rows <= 0) { - return 0; - } - const int64_t skip_end = _row_group_rows_read + rows; - int64_t filtered_rows = 0; - for (const auto& range : _page_skip_plan->skipped_ranges) { - const int64_t range_end = range.start + range.length; - if (range_end <= _row_group_rows_read) { - continue; - } - if (range.start >= skip_end) { - break; - } - const int64_t start = std::max(range.start, _row_group_rows_read); - const int64_t end = std::min(range_end, skip_end); - if (start < end) { - // Scheduler gap skips are derived from page-index selected_ranges. A page-filtered - // range can only overlap such a gap when the whole data page is outside every selected - // range, so partial overlap would mean the planner and scheduler are out of sync. - DORIS_CHECK(start == range.start); - DORIS_CHECK(end == range_end); - filtered_rows += end - start; - } - } - return filtered_rows; -} - -void ScalarColumnReader::advance_rows_read(int64_t rows) { - DORIS_CHECK(rows >= 0); - _row_group_rows_read += rows; -} - -Status ScalarColumnReader::skip(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - const int64_t page_filtered_rows = page_filtered_rows_to_skip(rows); - DORIS_CHECK(page_filtered_rows <= rows); - const int64_t record_reader_skip_rows = rows - page_filtered_rows; - RETURN_IF_ERROR(skip_records(record_reader_skip_rows)); - advance_rows_read(rows); - return Status::OK(); -} - -Status ScalarColumnReader::select_with_dictionary_filter(const SelectionVector& sel, - uint16_t selected_rows, int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, - IColumn::Filter* row_filter, - bool* used_filter) { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(used_filter != nullptr); - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); - *used_filter = false; - row_filter->clear(); - row_filter->reserve(selected_rows); - DORIS_CHECK(_record_reader != nullptr); - // A clean fallback is possible only before any range skip or read advances the record reader. - // Once dictionary selection starts, losing dictionary output is corruption rather than a - // reason to retry through the ordinary selected-read path with an already advanced stream. - if (!_record_reader->read_dictionary()) { - return Status::OK(); - } - *used_filter = true; - - const auto ranges = selection_to_ranges(sel, selected_rows); - int64_t cursor = 0; - for (const auto& range : ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument( - "Invalid parquet dictionary selection range [{}, {}) for column {}", - range.start, range.start + range.length, _name); - } - RETURN_IF_ERROR(skip(range.start - cursor)); - - int64_t range_rows_read = 0; - RETURN_IF_ERROR(read_range_with_dictionary_filter(range.length, dictionary_filter, column, - row_filter, &range_rows_read, - used_filter)); - if (!*used_filter) { - return Status::OK(); - } - if (range_rows_read != range.length) { - return Status::Corruption( - "Parquet dictionary selected read returned {} rows, expected {} rows for " - "column {}", - range_rows_read, range.length, _name); - } - cursor = range.start + range.length; - } - RETURN_IF_ERROR(skip(batch_rows - cursor)); - if (_profile.reader_select_rows != nullptr) { - COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); - } - return Status::OK(); -} - -Status ScalarColumnReader::read_range_with_dictionary_filter( - int64_t rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* rows_read, bool* used_filter) { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(rows_read != nullptr); - DORIS_CHECK(used_filter != nullptr); - DORIS_CHECK(_record_reader != nullptr); - if (!_record_reader->read_dictionary()) { - return Status::Corruption( - "Parquet dictionary reader became unavailable after selected reading started for " - "column {}", - _name); - } - - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(leaf_reader().read_batch(rows, &leaf_batch, rows_read)); - int64_t matched_rows = 0; - RETURN_IF_ERROR(append_dictionary_filtered_values(leaf_batch.binary_chunks(), dictionary_filter, - column, row_filter, &matched_rows, - used_filter)); - if (!*used_filter) { - return Status::Corruption( - "Parquet dictionary reader did not return dictionary batches for column {}", _name); - } - if (row_filter->size() < static_cast(*rows_read)) { - return Status::Corruption( - "Parquet dictionary filter produced too few row decisions for column {}: " - "filter={}, rows={}", - _name, row_filter->size(), *rows_read); - } - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(matched_rows != nullptr); - DORIS_CHECK(used_filter != nullptr); - *matched_rows = 0; - *used_filter = false; - - std::vector selected_values; - for (const auto& chunk : chunks) { - DORIS_CHECK(chunk != nullptr); - const auto* dict_array = dynamic_cast(chunk.get()); - if (dict_array == nullptr) { - // The caller has already consumed rows from a DictionaryRecordReader. Falling back to a - // normal selected read would desynchronize the Parquet stream, so absence of a - // DictionaryArray is reported as corruption by read_range_with_dictionary_filter(). - return Status::OK(); - } - *used_filter = true; - const auto& dictionary = dict_array->dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - _name); - } - - // Dictionary predicates are evaluated once against the dictionary page and produce a - // dictionary-entry bitmap. DATA_PAGE rows then only need an integer-index lookup. NULL rows - // do not have a dictionary entry and cannot satisfy the supported equality/IN predicates. - for (int64_t row = 0; row < dict_array->length(); ++row) { - bool keep = false; - if (!dict_array->IsNull(row)) { - const int64_t dictionary_index = dict_array->GetValueIndex(row); - if (dictionary_index < 0 || dictionary_index >= dictionary->length() || - dictionary_index >= static_cast(dictionary_filter.size())) { - return Status::Corruption( - "Invalid parquet dictionary index {} for column {}: dictionary={}, " - "filter={}", - dictionary_index, _name, dictionary->length(), - dictionary_filter.size()); - } - keep = dictionary_filter[static_cast(dictionary_index)] != 0; - if (keep) { - RETURN_IF_ERROR(append_arrow_binary_dictionary_value( - _name, *dictionary, dictionary_index, &selected_values)); - ++*matched_rows; - } - } - row_filter->push_back(keep ? 1 : 0); - } - } - - if (!*used_filter) { - return Status::OK(); - } - return append_decoded_binary_values(selected_values, column); -} - -Status ScalarColumnReader::append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const { - DecodedColumnView view; - view.value_kind = decoded_value_kind(_type_descriptor); - view.row_count = static_cast(values.size()); - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.fixed_length = _type_descriptor.fixed_length; - view.binary_values = &values; - - SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(null_map.data() + old_null_map_size, 0, null_map.size() - old_null_map_size); - return Status::OK(); - } - return _type->get_serde()->read_column_from_decoded_values(*column, view); - } - - NullMap null_map(values.size(), 0); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - return _type->get_serde()->read_column_from_decoded_values(*column, view); -} - -// The value index stream must advance on those null slots, otherwise later payload values shift. -Status ScalarColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - RETURN_IF_ERROR(leaf_reader().read_nested_batch(rows, materialized_slot_definition_level, - _nested_batch.get(), _repetition_level)); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(leaf_reader().read_nested_levels_batch(rows, _nested_batch.get())); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar build result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - // The levels-only loader intentionally does not populate value-slot metadata or payload - // buffers. Derive the logical slot threshold from the schema, exactly as load_nested_batch() - // does, so this consumer works for both loaded batch forms. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_read = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_read < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - if (def_level == _definition_level) { - RETURN_IF_ERROR(append_scalar_batch_value(*this, *_nested_batch, current_level_idx, - &value_cursor, column)); - } else { - if (!_type->is_nullable() && def_level >= _nullable_definition_level) { - return Status::Corruption( - "Parquet scalar column {} contains null for non-nullable field", _name); - } - column->insert_default(); - } - ++*values_read; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - if (values_consumed == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar consume result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - // A levels-only batch intentionally has no value-slot metadata. Reconstruct the same logical - // slot threshold used by load_nested_batch(): a nullable leaf owns a slot at one definition - // level below a non-null value, while a required leaf owns a slot only at its full definition - // level. For example, an empty ARRAY boundary must not be consumed as a STRING value. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_consumed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_consumed < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - RETURN_IF_ERROR(validate_nested_value(current_level_idx, false)); - ++*values_consumed; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::append_nested_value(int64_t level_idx, MutableColumnPtr& column) const { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar append result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return append_scalar_batch_value(*this, *_nested_batch, level_idx, &value_cursor, column); - } - if (!_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable value", - _name); - } - column->insert_default(); - return Status::OK(); -} - -Status ScalarColumnReader::validate_nested_value(int64_t level_idx, bool require_non_null) const { - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return Status::OK(); - } - if (require_non_null || !_type->is_nullable()) { - return Status::Corruption("Parquet scalar column {} contains null for non-nullable field", - _name); - } - return Status::OK(); -} - -const std::vector& ScalarColumnReader::nested_definition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->def_levels; -} - -const std::vector& ScalarColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->rep_levels; -} - -int64_t ScalarColumnReader::nested_levels_written() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->levels_written; -} - -bool ScalarColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.h b/be/src/format_v2/parquet/reader/scalar_column_reader.h deleted file mode 100644 index 5342baa803eca0..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.h +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "core/string_ref.h" -#include "format_v2/parquet/parquet_type.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris::format::parquet { - -struct ScalarColumnReaderTestAccess; - -// load_nested_batch() / build_nested_column() -class ScalarColumnReader final : public ParquetColumnReader { - friend class MapColumnReader; - friend struct ScalarColumnReaderTestAccess; - -public: - ScalarColumnReader(const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan = nullptr, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - ParquetColumnReaderProfile profile = {}); - ~ScalarColumnReader() override; - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - bool* used_filter) override; - - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - -private: - Status append_nested_value(int64_t level_idx, MutableColumnPtr& column) const; - Status validate_nested_value(int64_t level_idx, bool require_non_null) const; - Status read_range_with_dictionary_filter(int64_t rows, const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - int64_t* rows_read, bool* used_filter); - Status append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const; - Status append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const; - - const ::parquet::ColumnDescriptor* descriptor() const { return _descriptor; } - - ParquetLeafReader leaf_reader() const { - return ParquetLeafReader(_descriptor, _type_descriptor, _type, _name, _record_reader, - _profile, _timezone, _enable_strict_mode); - } - - void advance_rows_read(int64_t rows); - Status skip_records(int64_t rows); - int64_t page_filtered_rows_to_skip(int64_t rows) const; - - const ::parquet::ColumnDescriptor* _descriptor = nullptr; // Arrow column descriptor - ParquetTypeDescriptor _type_descriptor; // type encoding information - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader - const ParquetPageSkipPlan* _page_skip_plan = - nullptr; // page-index pruning result (may be nullptr) - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - int64_t _row_group_rows_read = 0; // rows read in the current row group (cursor) - std::unique_ptr _nested_batch; // intermediate result for nested reads -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.cpp b/be/src/format_v2/parquet/reader/struct_column_reader.cpp deleted file mode 100644 index 5abe7abe75e9a2..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.cpp +++ /dev/null @@ -1,287 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/struct_column_reader.h" - -#include -#include -#include -#include - -#include "core/column/column_struct.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { - -ParquetColumnReader* StructColumnReader::shape_source_reader() const { - for (const auto& child : _children) { - auto* child_reader = child.get(); - DORIS_CHECK(child_reader != nullptr); - if (!child_reader->is_or_has_repeated_child()) { - return child_reader; - } - } - if (_children.empty()) { - return nullptr; - } - return _children[0].get(); -} - -Status StructColumnReader::advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const { - DORIS_CHECK(child_reader != nullptr); - const int64_t next_child_cursor = parent_level_idx + 1; - if (auto* scalar_child = dynamic_cast(child_reader)) { - if (next_child_cursor > scalar_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - scalar_child->name(), _name); - } - scalar_child->set_nested_build_level_cursor( - std::max(scalar_child->nested_build_level_cursor(), next_child_cursor)); - return Status::OK(); - } - if (auto* struct_child = dynamic_cast(child_reader); - struct_child != nullptr && !struct_child->is_or_has_repeated_child()) { - if (next_child_cursor > struct_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - struct_child->name(), _name); - } - struct_child->set_nested_build_level_cursor( - std::max(struct_child->nested_build_level_cursor(), next_child_cursor)); - for (auto& grandchild : struct_child->_children) { - RETURN_IF_ERROR(struct_child->advance_child_past_null_parent(grandchild.get(), - parent_level_idx)); - } - return Status::OK(); - } - - int64_t child_cursor = child_reader->nested_build_level_cursor(); - const auto& child_rep_levels = child_reader->nested_repetition_levels(); - const int64_t child_levels_written = child_reader->nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!child_reader->is_or_has_repeated_child() || child_rep_level <= _repetition_level) { - break; - } - } - child_reader->set_nested_build_level_cursor(child_cursor); - return Status::OK(); -} - -Status StructColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status StructColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status StructColumnReader::load_nested_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::load_nested_levels_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_levels_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet struct build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status StructColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument( - "Invalid parquet struct process result pointer for column {}", _name); - } - if (_children.empty()) { - if (column != nullptr) { - (*column)->resize((*column)->size() + static_cast(length_upper_bound)); - } - *values_processed = length_upper_bound; - return Status::OK(); - } - ColumnStruct* struct_column = nullptr; - NullMap* parent_null_map = nullptr; - if (column != nullptr) { - struct_column = struct_column_from_output(*column); - DORIS_CHECK(struct_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - } - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - const auto& def_levels = shape_reader->nested_definition_levels(); - const auto& rep_levels = shape_reader->nested_repetition_levels(); - const int64_t levels_written = shape_reader->nested_levels_written(); - - NullMap parent_nulls; - std::vector parent_level_indices; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < levels_written) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = - !shape_reader->is_or_has_repeated_child() || rep_level <= _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (def_level < _repeated_ancestor_definition_level) { - continue; - } - if (shape_reader->is_or_has_repeated_child() && rep_level > _repetition_level) { - continue; - } - const bool parent_is_null = def_level < _nullable_definition_level; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption( - "Parquet STRUCT column {} contains null for non-nullable struct", _name); - } - parent_nulls.push_back(parent_is_null); - parent_level_indices.push_back(current_level_idx); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - std::vector child_columns; - if (column != nullptr) { - child_columns.reserve(struct_column->get_columns().size()); - for (size_t child_idx = 0; child_idx < struct_column->get_columns().size(); ++child_idx) { - child_columns.push_back(struct_column->get_column_ptr(child_idx)->assert_mutable()); - } - } - for (size_t child_idx = 0; child_idx < _children.size(); ++child_idx) { - const int output_idx = _child_output_indices[child_idx]; - if (column != nullptr && output_idx < 0) { - continue; - } - // STRUCT owns row alignment. Child readers consume only present parent rows from their - // level streams; null STRUCT parents become default placeholders in every child column. - // This mirrors Arrow's separation between struct validity and child array materialization, - // and avoids asking scalar/list/map children to invent values for an absent parent. - int64_t pending_present_rows = 0; - int64_t total_child_rows = 0; - auto flush_present_rows = [&]() -> Status { - if (pending_present_rows == 0) { - return Status::OK(); - } - int64_t child_rows = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_children[child_idx]->build_nested_column( - pending_present_rows, child_columns[output_idx], &child_rows)); - } else { - RETURN_IF_ERROR(_children[child_idx]->consume_nested_column(pending_present_rows, - &child_rows)); - } - if (child_rows != pending_present_rows) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), child_rows, pending_present_rows, _name); - } - total_child_rows += child_rows; - pending_present_rows = 0; - return Status::OK(); - }; - for (size_t parent_idx = 0; parent_idx < parent_nulls.size(); ++parent_idx) { - const auto parent_is_null = parent_nulls[parent_idx]; - if (!parent_is_null) { - ++pending_present_rows; - continue; - } - RETURN_IF_ERROR(flush_present_rows()); - if (column != nullptr) { - child_columns[output_idx]->insert_default(); - } - RETURN_IF_ERROR(advance_child_past_null_parent(_children[child_idx].get(), - parent_level_indices[parent_idx])); - ++total_child_rows; - } - RETURN_IF_ERROR(flush_present_rows()); - if (total_child_rows != *values_processed) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), total_child_rows, *values_processed, _name); - } - } - if (column != nullptr) { - for (size_t child_idx = 0; child_idx < child_columns.size(); ++child_idx) { - struct_column->get_column_ptr(child_idx) = std::move(child_columns[child_idx]); - } - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& StructColumnReader::nested_definition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_definition_levels(); -} - -const std::vector& StructColumnReader::nested_repetition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_repetition_levels(); -} - -int64_t StructColumnReader::nested_levels_written() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_levels_written(); -} - -bool StructColumnReader::is_or_has_repeated_child() const { - auto* shape_reader = shape_source_reader(); - return shape_reader != nullptr && shape_reader->is_or_has_repeated_child(); -} - -void StructColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - for (auto& child : _children) { - DORIS_CHECK(child != nullptr); - child->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - } -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.h b/be/src/format_v2/parquet/reader/struct_column_reader.h deleted file mode 100644 index 3c2d6904cb36f4..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.h +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class StructColumnReader final : public ParquetColumnReader { -public: - StructColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::vector> children, - std::vector child_output_indices, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _children(std::move(children)), - _child_output_indices(std::move(child_output_indices)) { - DCHECK_EQ(_children.size(), _child_output_indices.size()); - } - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - ParquetColumnReader* shape_source_reader() const; - Status advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const; - - std::vector> _children; // projected child readers - std::vector _child_output_indices; // child reader -> struct output position mapping -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h index 589154d4acc0e4..ab2bf93c785edc 100644 --- a/be/src/format_v2/parquet/selection_vector.h +++ b/be/src/format_v2/parquet/selection_vector.h @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -33,11 +34,11 @@ struct RowRange { struct ParquetPageSkipPlan { int leaf_column_id = -1; // Page ordinal is the data-page ordinal in the column chunk. It intentionally excludes - // dictionary pages, matching Arrow PageReader::set_data_page_filter(). + // dictionary pages, matching the native page reader's ordinal domain. std::vector skipped_pages; std::vector skipped_page_compressed_sizes; - // Row ranges covered by skipped data pages. ScalarColumnReader uses these ranges to avoid - // calling RecordReader::SkipRecords() again for pages already skipped by Arrow. + // Row ranges covered by skipped data pages. NativeColumnReader uses these ranges to avoid + // consuming logical rows twice after the page reader has already skipped their payload. std::vector skipped_ranges; bool empty() const { return skipped_ranges.empty(); } @@ -66,6 +67,7 @@ class SelectionVector { _owned.clear(); _data = data; _size = count; + ++_generation; } void resize(size_t count) { @@ -75,12 +77,14 @@ class SelectionVector { for (size_t idx = 0; idx < count; ++idx) { _data[idx] = static_cast(idx); } + ++_generation; } void clear() { _owned.clear(); _data = nullptr; _size = 0; + ++_generation; } size_t size() const { return _size; } @@ -98,7 +102,29 @@ class SelectionVector { return _data[idx]; } - void set_index(size_t idx, Index value) { _data[idx] = value; } + void set_index(size_t idx, Index value) { + _data[idx] = value; + ++_generation; + } + + Status materialize_filter(size_t count, int64_t batch_rows, const uint8_t** filter) const { + DORIS_CHECK(filter != nullptr); + RETURN_IF_ERROR(verify(count, batch_rows)); + if (_filter_generation != _generation || _filter_count != count || + _filter_batch_rows != batch_rows) { + // Selection is shared by all readers in one scheduler batch. Cache its dense bitmap so + // a wide lazy projection does not rebuild the same O(batch_rows) filter per column. + _filter.assign(static_cast(batch_rows), 0); + for (size_t idx = 0; idx < count; ++idx) { + _filter[get_index(idx)] = 1; + } + _filter_generation = _generation; + _filter_count = count; + _filter_batch_rows = batch_rows; + } + *filter = _filter.data(); + return Status::OK(); + } Status verify(size_t count, int64_t batch_rows) const { if (batch_rows < 0) { @@ -135,13 +161,19 @@ class SelectionVector { std::vector _owned; Index* _data = nullptr; size_t _size = 0; + uint64_t _generation = 0; + mutable std::vector _filter; + mutable uint64_t _filter_generation = std::numeric_limits::max(); + mutable size_t _filter_count = 0; + mutable int64_t _filter_batch_rows = -1; }; -inline std::vector selection_to_ranges(const SelectionVector& selection, - uint16_t selected_rows) { - std::vector ranges; +inline void selection_to_ranges(const SelectionVector& selection, uint16_t selected_rows, + std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); if (selected_rows == 0) { - return ranges; + return; } int64_t range_start = selection.get_index(0); @@ -152,11 +184,17 @@ inline std::vector selection_to_ranges(const SelectionVector& selectio previous = current; continue; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); range_start = current; previous = current; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); +} + +inline std::vector selection_to_ranges(const SelectionVector& selection, + uint16_t selected_rows) { + std::vector ranges; + selection_to_ranges(selection, selected_rows, &ranges); return ranges; } diff --git a/be/src/format_v2/table/hive_reader.cpp b/be/src/format_v2/table/hive_reader.cpp index 74a95fd43965cf..619e9959a1bfa4 100644 --- a/be/src/format_v2/table/hive_reader.cpp +++ b/be/src/format_v2/table/hive_reader.cpp @@ -102,11 +102,16 @@ void add_name_mapping(std::vector* name_mapping, const std::string& } // namespace Status HiveReader::prepare_split(const format::SplitReadOptions& options) { - if (options.current_split_format != _format) { - return Status::InternalError( - "Hive scan expects all splits to use the same file format, " - "initialized_format={}, current_split_format={}", - static_cast(_format), static_cast(options.current_split_format)); + { + // Keep derived validation visible without overlapping the base scopes on the same timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + if (options.current_split_format != _format) { + return Status::InternalError( + "Hive scan expects all splits to use the same file format, " + "initialized_format={}, current_split_format={}", + static_cast(_format), static_cast(options.current_split_format)); + } } return format::TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 6c8917d867074c..d05cfe4b03e357 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -28,13 +28,20 @@ namespace doris::format::hudi { Status HudiReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.__isset.hudi_params && - options.current_range.table_format_params.hudi_params.__isset.schema_id) { - _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.hudi_params && + options.current_range.table_format_params.hudi_params.__isset.schema_id) { + _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -66,6 +73,8 @@ Status HudiHybridReader::init(format::TableReadOptions&& options) { } Status HudiHybridReader::prepare_split(const format::SplitReadOptions& options) { + // A newly selected child initializes against the same scanner profile. Keep hybrid dispatch + // outside those shared counters so first-split initialization is counted exactly once. RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); @@ -120,7 +129,15 @@ Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOpt DORIS_CHECK(_scan_params != nullptr); if (_is_jni_split(*_scan_params, options.current_range)) { if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -128,7 +145,15 @@ Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOpt format::FileFormat file_format; RETURN_IF_ERROR(_to_file_format(*_scan_params, options.current_range, &file_format)); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::hudi::HudiReader::create_unique(); + } +#else _native_reader = format::hudi::HudiReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index 77a71cd79ba166..d81e8c80e93ed3 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -73,6 +74,12 @@ class HudiHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -88,6 +95,10 @@ class HudiHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle native parquet/orc splits std::unique_ptr _jni_reader; // handle MOR JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::hudi diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index ef13f585fb6a6d..fc1d797173b0ab 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -161,12 +161,17 @@ Status IcebergPositionDeleteSysTableV2Reader::prepare_split( const format::SplitReadOptions& options) { RETURN_IF_ERROR(close()); RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + // The inner delete-file reader has distinct counters, so the outer preparation can safely + // contain its cache miss/open work without re-entering the same RuntimeProfile timer. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); _current_range = options.current_range; _has_split = true; return _init_split(); } Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); DORIS_CHECK(block != nullptr); DORIS_CHECK(eos != nullptr); @@ -307,6 +312,15 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { std::vector projected_columns; RETURN_IF_ERROR(_build_delete_file_projected_columns(&projected_columns)); + static constexpr const char* kPositionReaderProfile = "IcebergPositionDeleteFileReader"; + if (_position_reader_profile == nullptr) { + _position_reader_profile = _scanner_profile->get_child(kPositionReaderProfile); + if (_position_reader_profile == nullptr) { + // The outer system-table reader calls the inner reader synchronously. Giving both the + // same profile would nest identical counter pointers and double-count every timer. + _position_reader_profile = _scanner_profile->create_child(kPositionReaderProfile); + } + } _position_reader = std::make_unique(); RETURN_IF_ERROR(_position_reader->init({ .projected_columns = std::move(projected_columns), @@ -315,7 +329,7 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { .scan_params = _scan_params, .io_ctx = _io_ctx, .runtime_state = _runtime_state, - .scanner_profile = _scanner_profile, + .scanner_profile = _position_reader_profile, .file_slot_descs = nullptr, .push_down_agg_type = TPushAggOp::type::NONE, .condition_cache_digest = 0, diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h index 9c802e726053cb..09756daaabe4da 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h @@ -76,6 +76,7 @@ class IcebergPositionDeleteSysTableV2Reader final : public format::TableReader { const TIcebergDeleteFileDesc* _delete_file_desc = nullptr; DeleteFileKind _delete_file_kind = DeleteFileKind::POSITION_DELETE; std::unique_ptr _position_reader; + RuntimeProfile* _position_reader_profile = nullptr; std::vector _read_columns; ColumnPtr _partition_value; roaring::Roaring64Map _dv_positions; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index b6580a12dfcd0f..1c6f1935e015a8 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -44,6 +44,7 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "io/file_factory.h" +#include "util/debug_points.h" #include "util/url_coding.h" namespace doris::format::iceberg { @@ -225,25 +226,31 @@ Status IcebergTableReader::PositionDeleteRowsCollector::collect(const Block& blo } Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options) { - _row_lineage_columns = {}; - _iceberg_params.reset(); - _delete_predicates_initialized = false; - _position_delete_rows_storage.clear(); - _equality_delete_filters.clear(); - _split_cache = options.cache; - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.__isset.iceberg_params) { - const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; - _iceberg_params = iceberg_params; - if (iceberg_params.__isset.first_row_id) { - _row_lineage_columns.first_row_id = iceberg_params.first_row_id; - } - if (iceberg_params.__isset.last_updated_sequence_number) { - _row_lineage_columns.last_updated_sequence_number = - iceberg_params.last_updated_sequence_number; + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _row_lineage_columns = {}; + _iceberg_params.reset(); + _delete_predicates_initialized = false; + _position_delete_rows_storage.clear(); + _equality_delete_filters.clear(); + _split_cache = options.cache; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.iceberg_params) { + const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; + _iceberg_params = iceberg_params; + if (iceberg_params.__isset.first_row_id) { + _row_lineage_columns.first_row_id = iceberg_params.first_row_id; + } + if (iceberg_params.__isset.last_updated_sequence_number) { + _row_lineage_columns.last_updated_sequence_number = + iceberg_params.last_updated_sequence_number; + } } } RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -255,6 +262,8 @@ Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options if (_is_table_level_count_active()) { return Status::OK(); } + DBUG_EXECUTE_IF("IcebergTableReader.prepare_split.before_delete_file_scan", + DBUG_RUN_CALLBACK()); RETURN_IF_ERROR(_init_delete_predicates(options.current_range.table_format_params)); return Status::OK(); } @@ -642,10 +651,13 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && scan_params.enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = + scan_params.__isset.enable_mapping_varbinary && scan_params.enable_mapping_varbinary; if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { + // Delete and data files must parse raw binary fields with the same scan-level mapping. *reader = std::make_unique( system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, - enable_mapping_timestamp_tz); + enable_mapping_timestamp_tz, enable_mapping_varbinary); } else { *reader = std::make_unique(system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 0b3e410b87c0eb..942fbb234269fc 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -33,12 +33,19 @@ namespace doris::format::paimon { Status PaimonReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - const auto& paimon_params = options.current_range.table_format_params.paimon_params; - if (paimon_params.__isset.schema_id) { - _split_schema_id = paimon_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + const auto& paimon_params = options.current_range.table_format_params.paimon_params; + if (paimon_params.__isset.schema_id) { + _split_schema_id = paimon_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -93,6 +100,8 @@ Status PaimonHybridReader::init(format::TableReadOptions&& options) { } Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options) { + // Child initialization uses the scanner profile too; hybrid dispatch must not nest the same + // timer around the first native or JNI child and double-count that initialization. RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); @@ -147,7 +156,15 @@ Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadO if (_is_jni_split(options.current_range)) { DCHECK(options.current_split_format == format::FileFormat::JNI); if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -158,7 +175,15 @@ Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadO DCHECK(file_format == format::FileFormat::PARQUET || file_format == format::FileFormat::ORC); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::paimon::PaimonReader::create_unique(); + } +#else _native_reader = format::paimon::PaimonReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index 634f2bd7a6400c..ad29075fed1f08 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include "format_v2/table_reader.h" @@ -84,6 +85,12 @@ class PaimonHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -96,6 +103,10 @@ class PaimonHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle parquet/orc native splits std::unique_ptr _jni_reader; // handle serialized JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::paimon diff --git a/be/src/format_v2/table/remote_doris_reader.cpp b/be/src/format_v2/table/remote_doris_reader.cpp index c67cece6e05b8c..db278d9b0cedda 100644 --- a/be/src/format_v2/table/remote_doris_reader.cpp +++ b/be/src/format_v2/table/remote_doris_reader.cpp @@ -37,6 +37,7 @@ #include "format/arrow/arrow_utils.h" #include "format_v2/materialized_reader_util.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/timezone_utils.h" @@ -177,7 +178,27 @@ RemoteDorisFileReader::~RemoteDorisFileReader() { static_cast(close()); } +void RemoteDorisFileReader::_init_profile() { + if (_profile == nullptr) { + return; + } + const auto hierarchy = file_scan_profile::ensure_hierarchy(_profile); + _io_time = hierarchy.io; + static const char* remote_profile = "RemoteDorisFileReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, remote_profile, file_scan_profile::FILE_READER, 1); + _open_stream_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisOpenStreamTime", remote_profile, 1); + _next_batch_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisNextBatchTime", remote_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisMaterializeTime", remote_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisFilterTime", remote_profile, 1); +} + Status RemoteDorisFileReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); (void)state; RETURN_IF_ERROR(validate_remote_doris_range(_range)); RETURN_IF_ERROR(_build_col_name_to_file_id()); @@ -186,6 +207,7 @@ Status RemoteDorisFileReader::init(RuntimeState* state) { } Status RemoteDorisFileReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_schema != nullptr); file_schema->clear(); file_schema->reserve(_file_slot_descs.size()); @@ -206,6 +228,8 @@ Status RemoteDorisFileReader::get_schema(std::vector* file_sch } Status RemoteDorisFileReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_stream_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); RETURN_IF_ERROR(_open_stream()); _eof = false; @@ -213,6 +237,7 @@ Status RemoteDorisFileReader::open(std::shared_ptr request) { } Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -223,21 +248,32 @@ Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* e *rows = 0; *eof = false; std::shared_ptr batch; - RETURN_IF_ERROR(_stream->next(&batch)); + { + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_next_batch_time); + RETURN_IF_ERROR(_stream->next(&batch)); + } if (batch == nullptr) { *eof = true; _eof = true; return Status::OK(); } - RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + } _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR( - apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR( + apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + } return Status::OK(); } Status RemoteDorisFileReader::close() { + SCOPED_TIMER(_total_time); if (_stream != nullptr) { RETURN_IF_ERROR(_stream->close()); _stream.reset(); @@ -349,7 +385,12 @@ Status RemoteDorisReader::init(TableReadOptions&& options) { } Status RemoteDorisReader::prepare_split(const SplitReadOptions& options) { - RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + { + // Keep protocol validation visible while avoiding overlap with TableReader's own scopes. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + } return TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table/remote_doris_reader.h b/be/src/format_v2/table/remote_doris_reader.h index b4dd2a505a95ad..79c37a29be4a06 100644 --- a/be/src/format_v2/table/remote_doris_reader.h +++ b/be/src/format_v2/table/remote_doris_reader.h @@ -71,6 +71,7 @@ class RemoteDorisFileReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _open_stream(); Status _materialize_record_batch(const arrow::RecordBatch& batch, Block* file_block, size_t* rows) const; @@ -83,6 +84,12 @@ class RemoteDorisFileReader final : public FileReader { const std::vector _file_slot_descs; RemoteDorisStreamFactory _stream_factory; cctz::time_zone _ctz; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_stream_time = nullptr; + RuntimeProfile::Counter* _next_batch_time = nullptr; + RuntimeProfile::Counter* _io_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; std::unique_ptr _stream; std::unordered_map _col_name_to_file_id; }; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index e8725ed65fa49b..2e0000c5fa2ace 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -49,6 +49,7 @@ #include "format_v2/native/native_reader.h" #include "format_v2/orc/orc_reader.h" #include "format_v2/parquet/parquet_reader.h" +#include "runtime/file_scan_profile.h" #include "storage/segment/condition_cache.h" #include "util/debug_points.h" #include "util/string_util.h" @@ -520,24 +521,15 @@ std::optional TableReader::_find_current_table_column_by_field } Status TableReader::init(TableReadOptions&& options) { - _scan_params = options.scan_params; - _format = options.format; - _io_ctx = options.io_ctx; - _runtime_state = options.runtime_state; _scanner_profile = options.scanner_profile; - _file_slot_descs = options.file_slot_descs; - _push_down_agg_type = options.push_down_agg_type; - _push_down_count_columns = options.push_down_count_columns; - _initial_condition_cache_digest = options.condition_cache_digest; - _condition_cache_digest = _initial_condition_cache_digest; - _projected_columns = std::move(options.projected_columns); - _system_properties = create_system_properties(_scan_params); - _mapper_options.mode = TableColumnMappingMode::BY_NAME; - _conjuncts = std::move(options.conjuncts); - if (_scanner_profile != nullptr) { - static const char* table_profile = "TableReader"; - ADD_TIMER_WITH_LEVEL(_scanner_profile, table_profile, 1); + const auto hierarchy = file_scan_profile::ensure_hierarchy(_scanner_profile); + static const char* table_profile = file_scan_profile::TABLE_READER; + static const char* file_reader_profile = file_scan_profile::FILE_READER; + _profile.total_timer = hierarchy.table_reader; + _profile.file_reader_total_timer = hierarchy.file_reader; + _profile.init_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "InitTime", table_profile, 1); _profile.num_delete_files = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteFiles", TUnit::UNIT, table_profile, 1); _profile.num_delete_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteRows", @@ -570,11 +562,47 @@ Status TableReader::init(TableReadOptions&& options) { ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PushDownAggTime", table_profile, 1); _profile.open_reader_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "OpenReaderTime", table_profile, 1); - _profile.runtime_filter_partition_prune_timer = ADD_TIMER_WITH_LEVEL( - _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", 1); - _profile.runtime_filter_partition_pruned_range_counter = ADD_COUNTER_WITH_LEVEL( - _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, 1); - } + _profile.runtime_filter_partition_prune_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", table_profile, 1); + _profile.runtime_filter_partition_pruned_range_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, + table_profile, 1); + _profile.close_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CloseTime", table_profile, 1); + // Lifecycle timer names remain globally unique because RuntimeProfile's visual hierarchy + // does not namespace counters that share the same display parent. + _profile.file_reader_init_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderInitTime", file_reader_profile, 1); + _profile.file_reader_schema_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetSchemaTime", file_reader_profile, 1); + _profile.file_reader_mapper_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCreateColumnMapperTime", file_reader_profile, 1); + _profile.file_reader_open_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderOpenTime", file_reader_profile, 1); + _profile.file_reader_get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetBlockTime", file_reader_profile, 1); + _profile.file_reader_aggregate_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderAggregatePushDownTime", file_reader_profile, 1); + _profile.file_reader_close_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCloseTime", file_reader_profile, 1); + } + // Establish lifecycle timers before consuming options or constructing filesystem properties; + // placing these scopes at the tail records only scope teardown and hides expensive init work. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + _scan_params = options.scan_params; + _format = options.format; + _io_ctx = options.io_ctx; + _runtime_state = options.runtime_state; + _file_slot_descs = options.file_slot_descs; + _push_down_agg_type = options.push_down_agg_type; + _push_down_count_columns = options.push_down_count_columns; + _initial_condition_cache_digest = options.condition_cache_digest; + _condition_cache_digest = _initial_condition_cache_digest; + _projected_columns = std::move(options.projected_columns); + _system_properties = create_system_properties(_scan_params); + _mapper_options.mode = TableColumnMappingMode::BY_NAME; + _conjuncts = std::move(options.conjuncts); return Status::OK(); } @@ -723,7 +751,12 @@ Status TableReader::create_next_reader(bool* eos) { if (_batch_size > 0) { _data_reader.reader->set_batch_size(_batch_size); } - Status st = _data_reader.reader->init(_runtime_state); + Status st; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_init_timer); + st = _data_reader.reader->init(_runtime_state); + } if (!st.ok()) { if (_io_ctx != nullptr && _io_ctx->should_stop && st.is()) { *eos = true; @@ -754,10 +787,16 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { const bool enable_mapping_timestamp_tz = _scan_params != nullptr && _scan_params->__isset.enable_mapping_timestamp_tz && _scan_params->enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = _scan_params != nullptr && + _scan_params->__isset.enable_mapping_varbinary && + _scan_params->enable_mapping_varbinary; if (_format == FileFormat::PARQUET) { + // V2 must honor the scan contract directly; otherwise Hive STRING columns backed by an + // unannotated BYTE_ARRAY are silently exposed as VARBINARY and predicate bytes no longer + // match the table type. *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); return Status::OK(); } if (_format == FileFormat::ORC) { @@ -828,6 +867,7 @@ std::unique_ptr create_file_description(const TFileRangeDes } Status TableReader::prepare_split(const SplitReadOptions& options) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.prepare_split_timer); _current_split_pruned = false; _all_runtime_filters_applied_for_split = options.all_runtime_filters_applied; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index cddddccd6bd3e1..6c61acd6c58294 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -99,6 +99,8 @@ struct ProjectedColumnBuildContext { }; struct ReadProfile { + RuntimeProfile::Counter* total_timer = nullptr; + RuntimeProfile::Counter* init_timer = nullptr; RuntimeProfile::Counter* num_delete_files = nullptr; RuntimeProfile::Counter* num_delete_rows = nullptr; RuntimeProfile::Counter* parse_delete_file_time = nullptr; @@ -115,6 +117,15 @@ struct ReadProfile { RuntimeProfile::Counter* open_reader_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_prune_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_pruned_range_counter = nullptr; + RuntimeProfile::Counter* close_timer = nullptr; + RuntimeProfile::Counter* file_reader_total_timer = nullptr; + RuntimeProfile::Counter* file_reader_init_timer = nullptr; + RuntimeProfile::Counter* file_reader_schema_timer = nullptr; + RuntimeProfile::Counter* file_reader_mapper_timer = nullptr; + RuntimeProfile::Counter* file_reader_open_timer = nullptr; + RuntimeProfile::Counter* file_reader_get_block_timer = nullptr; + RuntimeProfile::Counter* file_reader_aggregate_timer = nullptr; + RuntimeProfile::Counter* file_reader_close_timer = nullptr; }; struct TableReadOptions { @@ -219,6 +230,10 @@ class TableReader { // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start // with no concrete reader or split-local state left from the failed split. virtual Status abort_split() { + // Ignored open failures still spend time closing partially initialized readers. Include + // that recovery path in the common lifecycle profile so NOT_FOUND cannot become invisible. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader != nullptr) { RETURN_IF_ERROR(close_current_reader()); } else { @@ -236,6 +251,7 @@ class TableReader { // advances across EOF, and closes exhausted readers. Subclasses provide protected hooks for // table-format-specific behavior. virtual Status get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); DORIS_CHECK(block->columns() == _projected_columns.size()); block->clear_column_data(_projected_columns.size()); @@ -285,8 +301,12 @@ class TableReader { _data_reader.block_template.clear_column_data( cast_set(_data_reader.file_block_layout.size())); size_t current_rows = 0; - RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, - ¤t_rows, ¤t_eof)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_get_block_timer); + RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, + ¤t_rows, ¤t_eof)); + } const bool stopped_during_read = _io_ctx != nullptr && _io_ctx->should_stop; if (current_rows == 0) { if (current_eof) { @@ -317,6 +337,8 @@ class TableReader { // Close the table reader and the currently active file reader. Subclasses that hold additional // table-format resources should override this and call TableReader::close() first. virtual Status close() { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader) { RETURN_IF_ERROR(close_current_reader()); } @@ -379,14 +401,22 @@ class TableReader { SCOPED_TIMER(_profile.open_reader_timer); // 1. Get file schema and create column mapping. std::vector file_schema; - RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_schema_timer); + RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + } // For Paimon/Hudi, FE can provide field ids through `history_schema_info`. Annotate the // file schema before column mapping when the table format maps columns by field id. RETURN_IF_ERROR(annotate_file_schema(&file_schema)); _data_reader.file_schema = file_schema; _mapper_options.mode = mapping_mode(); - _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_mapper_timer); + _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + } DORIS_CHECK(_data_reader.column_mapper != nullptr); RETURN_IF_ERROR(_data_reader.column_mapper->create_mapping(_projected_columns, _partition_values, file_schema)); @@ -472,7 +502,11 @@ class TableReader { VLOG_DEBUG << "TableReader debug: " << debug_string(); } RETURN_IF_ERROR(_open_mapping_exprs()); - RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_open_timer); + RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + } RETURN_IF_ERROR(_init_reader_condition_cache(*file_request)); return Status::OK(); } @@ -679,7 +713,11 @@ class TableReader { // close(), so it should remain idempotent. virtual Status close_current_reader() { _finalize_reader_condition_cache(); - RETURN_IF_ERROR(_data_reader.reader->close()); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_close_timer); + RETURN_IF_ERROR(_data_reader.reader->close()); + } _data_reader.reader.reset(); if (_data_reader.column_mapper != nullptr) { _data_reader.column_mapper->clear(); @@ -706,10 +744,11 @@ class TableReader { Status finalize_chunk(Block* block, const size_t rows) { SCOPED_TIMER(_profile.finalize_timer); size_t idx = 0; - for (const auto& mapping : _data_reader.column_mapper->mappings()) { + const auto& mappings = _data_reader.column_mapper->mappings(); + for (const auto& mapping : mappings) { ColumnPtr column; RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, rows, - &column)); + &column, idx + 1 == mappings.size())); block->replace_by_position(idx, IColumn::mutate(std::move(column))); idx++; } @@ -960,7 +999,12 @@ class TableReader { FileAggregateRequest file_request; RETURN_IF_ERROR(_build_file_aggregate_request(_push_down_agg_type, &file_request)); FileAggregateResult file_result; - const auto status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + Status status; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_aggregate_timer); + status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + } if (status.is()) { return Status::OK(); } @@ -1053,6 +1097,17 @@ class TableReader { return IColumn::mutate(std::move(column)); } + static ColumnPtr _take_and_detach_block_column(Block* block, int position) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(position >= 0 && position < static_cast(block->columns())); + auto& source = block->get_by_position(position); + ColumnPtr column = source.column; + // The final mapping no longer needs the file block. Release its COW owner before mutate(), + // otherwise nested MAP/STRING columns are deep-copied and a multi-GB payload can OOM. + block->replace_by_position(position, source.type->create_column()); + return _detach_column(std::move(column)); + } + static Status _align_column_nullability(ColumnPtr* column, const DataTypePtr& table_type) { DORIS_CHECK(column != nullptr); DORIS_CHECK(column->get() != nullptr); @@ -1196,7 +1251,8 @@ class TableReader { } Status _materialize_mapping_column(const ColumnMapping& mapping, Block* current_block, - const size_t rows, ColumnPtr* column) { + const size_t rows, ColumnPtr* column, + bool take_projection_result = false) { if (!mapping.is_trivial && mapping.file_local_id.has_value() && !mapping.child_mappings.empty()) { DCHECK(mapping.projection != nullptr); @@ -1209,7 +1265,9 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), *mapping.file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; + ColumnPtr result_column = take_projection_result + ? _take_and_detach_block_column(current_block, res_id) + : current_block->get_by_position(res_id).column; RETURN_IF_ERROR( _materialize_complex_mapping_column(mapping, result_column, rows, column)); return Status::OK(); @@ -1228,8 +1286,12 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; - *column = _detach_column(std::move(result_column)); + if (take_projection_result) { + *column = _take_and_detach_block_column(current_block, res_id); + } else { + ColumnPtr result_column = current_block->get_by_position(res_id).column; + *column = _detach_column(std::move(result_column)); + } return Status::OK(); } if (mapping.default_expr != nullptr) { @@ -1602,9 +1664,10 @@ class TableReader { for (size_t column_idx = 0; column_idx < _data_reader.column_mapper->mappings().size(); ++column_idx) { ColumnPtr table_column; - RETURN_IF_ERROR( - _materialize_mapping_column(_data_reader.column_mapper->mappings()[column_idx], - &file_block, 2, &table_column)); + RETURN_IF_ERROR(_materialize_mapping_column( + _data_reader.column_mapper->mappings()[column_idx], &file_block, 2, + &table_column, + column_idx + 1 == _data_reader.column_mapper->mappings().size())); block->replace_by_position(column_idx, std::move(table_column)); } return Status::OK(); diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 751d422f044c1d..7aed33ccaa6e07 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -126,13 +126,26 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(segment_footer_index_bytes_write_into_cache); SUBTRACT_FIELD(remote_only_on_miss_triggered); SUBTRACT_FIELD(remote_only_on_miss_threshold_bytes); + + SUBTRACT_FIELD(num_cross_cg_peer_io_total); + SUBTRACT_FIELD(bytes_read_from_cross_cg_peer); + SUBTRACT_FIELD(cross_cg_peer_io_timer); + SUBTRACT_FIELD(num_same_cg_peer_io_total); + SUBTRACT_FIELD(bytes_read_from_same_cg_peer); + SUBTRACT_FIELD(same_cg_peer_io_timer); + SUBTRACT_FIELD(num_peer_race_peer_win); + SUBTRACT_FIELD(num_peer_race_s3_win); + SUBTRACT_FIELD(num_peer_lazy_fetch); + SUBTRACT_FIELD(peer_lazy_fetch_timer); #undef SUBTRACT_FIELD return diff; } -FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _profile(profile) { +FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, + const std::string& parent_counter) + : _profile(profile) { static const char* cache_profile = "FileCache"; - ADD_TIMER_WITH_LEVEL(profile, cache_profile, 2); + total_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, cache_profile, parent_counter.c_str(), 2); num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumLocalIOTotal", TUnit::UNIT, cache_profile, 1); num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumRemoteIOTotal", TUnit::UNIT, @@ -239,6 +252,12 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _p } void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) const { + // These are the outer cache-path phases. Their sum keeps the group timer actionable instead of + // displaying zero while individual cache IO and coordination timers are non-zero. + COUNTER_UPDATE(total_time, statistics->local_io_timer + statistics->remote_io_timer + + statistics->peer_io_timer + statistics->remote_wait_timer + + statistics->write_cache_io_timer + + statistics->cache_get_or_set_timer); COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total); COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total); COUNTER_UPDATE(num_peer_io_total, statistics->num_peer_io_total); diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index 2a6e7b33980bb2..d3fd31033649c8 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -71,6 +71,7 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren struct FileCacheProfileReporter { RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* num_local_io_total = nullptr; RuntimeProfile::Counter* num_remote_io_total = nullptr; RuntimeProfile::Counter* num_peer_io_total = nullptr; @@ -129,7 +130,9 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* num_peer_lazy_fetch = nullptr; RuntimeProfile::Counter* peer_lazy_fetch_timer = nullptr; - FileCacheProfileReporter(RuntimeProfile* profile); + explicit FileCacheProfileReporter( + RuntimeProfile* profile, + const std::string& parent_counter = RuntimeProfile::ROOT_COUNTER); void update(const FileCacheStatistics* statistics) const; }; diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index 246ee5e480db5d..386c3e4192c3d6 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -32,6 +32,7 @@ #include "common/status.h" #include "core/custom_allocator.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -655,7 +656,9 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File std::function sync_buffer = nullptr; if (profile != nullptr) { const char* prefetch_buffered_reader = "PrefetchBufferedReader"; - ADD_TIMER(profile, prefetch_buffered_reader); + auto* total_time = + ADD_CHILD_TIMER(profile, prefetch_buffered_reader, + file_scan_profile::parent_or_root(profile, file_scan_profile::IO)); auto copy_time = ADD_CHILD_TIMER(profile, "CopyTime", prefetch_buffered_reader); auto read_time = ADD_CHILD_TIMER(profile, "ReadTime", prefetch_buffered_reader); auto prefetch_request_io = @@ -667,6 +670,7 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File auto request_bytes = ADD_CHILD_COUNTER(profile, "RequestBytes", TUnit::BYTES, prefetch_buffered_reader); sync_buffer = [=](PrefetchBuffer& buf) { + COUNTER_UPDATE(total_time, buf._statis.copy_time + buf._statis.read_time); COUNTER_UPDATE(copy_time, buf._statis.copy_time); COUNTER_UPDATE(read_time, buf._statis.read_time); COUNTER_UPDATE(prefetch_request_io, buf._statis.prefetch_request_io); @@ -924,7 +928,9 @@ RangeCacheFileReader::RangeCacheFileReader(RuntimeProfile* profile, io::FileRead if (_profile != nullptr) { const char* random_profile = "RangeCacheFileReader"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, random_profile, 1); _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, @@ -985,6 +991,7 @@ Status RangeCacheFileReader::read_at_impl(size_t offset, Slice result, size_t* b void RangeCacheFileReader::_collect_profile_before_close() { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _cache_statistics.request_time); COUNTER_UPDATE(_request_io, _cache_statistics.request_io); COUNTER_UPDATE(_request_bytes, _cache_statistics.request_bytes); COUNTER_UPDATE(_request_time, _cache_statistics.request_time); diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h index beaf3e87a8d3ce..9b0e66ebbd078b 100644 --- a/be/src/io/fs/buffered_reader.h +++ b/be/src/io/fs/buffered_reader.h @@ -38,6 +38,7 @@ #include "io/fs/path.h" #include "io/fs/s3_file_reader.h" #include "io/io_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/olap_define.h" #include "util/slice.h" @@ -179,6 +180,7 @@ class RangeCacheFileReader : public io::FileReader { bool _closed = false; RuntimeProfile::Counter* _request_io = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _request_bytes = nullptr; RuntimeProfile::Counter* _request_time = nullptr; RuntimeProfile::Counter* _read_to_cache_time = nullptr; @@ -301,7 +303,9 @@ class MergeRangeFileReader : public io::FileReader { if (_profile != nullptr) { const char* random_profile = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", random_profile, 1); _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", random_profile, 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, @@ -350,6 +354,7 @@ class MergeRangeFileReader : public io::FileReader { void _collect_profile_before_close() override { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -363,6 +368,7 @@ class MergeRangeFileReader : public io::FileReader { } private: + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _copy_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index d99db30f1c00fc..6e363636980d19 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -32,6 +32,7 @@ #include "cpp/sync_point.h" #include "io/fs/err_utils.h" #include "io/hdfs_util.h" +#include "runtime/file_scan_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" #include "service/backend_options.h" @@ -83,7 +84,9 @@ HdfsFileReader::HdfsFileReader(Path path, std::string fs_name, FileHandleCache:: if (_profile != nullptr && is_hdfs(_fs_name)) { #ifdef USE_HADOOP_HDFS const char* hdfs_profile_name = "HdfsIO"; - ADD_TIMER(_profile, hdfs_profile_name); + _total_read_time = + ADD_CHILD_TIMER(_profile, hdfs_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); _hdfs_profile.total_bytes_read = ADD_CHILD_COUNTER(_profile, "TotalBytesRead", TUnit::BYTES, hdfs_profile_name); _hdfs_profile.total_local_bytes_read = @@ -117,6 +120,7 @@ Status HdfsFileReader::close() { Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, const IOContext* io_ctx) { + SCOPED_TIMER(_total_read_time); auto st = do_read_at_impl(offset, result, bytes_read, io_ctx); if (!st.ok()) { _handle = nullptr; diff --git a/be/src/io/fs/hdfs_file_reader.h b/be/src/io/fs/hdfs_file_reader.h index 08f98bca29af0c..7bb73f30909452 100644 --- a/be/src/io/fs/hdfs_file_reader.h +++ b/be/src/io/fs/hdfs_file_reader.h @@ -88,6 +88,7 @@ class HdfsFileReader final : public FileReader { CachedHdfsFileHandle* _handle = nullptr; // owned by _cached_file_handle std::atomic _closed = false; RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* _total_read_time = nullptr; int64_t _mtime; #ifdef USE_HADOOP_HDFS HDFSProfile _hdfs_profile; diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 4eaa10f3311e06..af8dde36d2df50 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -37,6 +37,7 @@ #include "io/fs/err_utils.h" #include "io/fs/obj_storage_client.h" #include "io/fs/s3_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -214,7 +215,9 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea void S3FileReader::_collect_profile_before_close() { if (_profile != nullptr) { const char* s3_profile_name = "S3Profile"; - ADD_TIMER(_profile, s3_profile_name); + auto* total_time = + ADD_CHILD_TIMER(_profile, s3_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); RuntimeProfile::Counter* total_get_request_counter = ADD_CHILD_COUNTER(_profile, "TotalGetRequest", TUnit::UNIT, s3_profile_name); RuntimeProfile::Counter* too_many_request_err_counter = @@ -231,6 +234,7 @@ void S3FileReader::_collect_profile_before_close() { COUNTER_UPDATE(too_many_request_sleep_time, _s3_stats.too_many_request_sleep_time_ms); COUNTER_UPDATE(total_bytes_read, _s3_stats.total_bytes_read); COUNTER_UPDATE(total_get_request_time_ns, _s3_stats.total_get_request_time_ns); + COUNTER_UPDATE(total_time, _s3_stats.total_get_request_time_ns); } } diff --git a/be/src/runtime/file_scan_profile.h b/be/src/runtime/file_scan_profile.h new file mode 100644 index 00000000000000..4ec71c4f4820d4 --- /dev/null +++ b/be/src/runtime/file_scan_profile.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "runtime/runtime_profile.h" + +namespace doris::file_scan_profile { + +inline constexpr const char* SCANNER = "FileScannerV2"; +inline constexpr const char* TABLE_READER = "TableReader"; +inline constexpr const char* FILE_READER = "FileReader"; +inline constexpr const char* IO = "IO"; + +struct Hierarchy { + RuntimeProfile::Counter* scanner = nullptr; + RuntimeProfile::Counter* table_reader = nullptr; + RuntimeProfile::Counter* file_reader = nullptr; + RuntimeProfile::Counter* io = nullptr; +}; + +inline Hierarchy ensure_hierarchy(RuntimeProfile* profile) { + if (profile == nullptr) { + return {}; + } + // RuntimeProfile stores one flat counter namespace and a separate parent map. Create every + // layer in order so later format and IO profiles cannot accidentally become scanner siblings. + auto* scanner = ADD_TIMER_WITH_LEVEL(profile, SCANNER, 1); + auto* table_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, TABLE_READER, SCANNER, 1); + auto* file_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, FILE_READER, TABLE_READER, 1); + auto* io = ADD_CHILD_TIMER_WITH_LEVEL(profile, IO, FILE_READER, 1); + return {scanner, table_reader, file_reader, io}; +} + +inline const char* parent_or_root(RuntimeProfile* profile, const char* preferred_parent) { + return profile != nullptr && profile->get_counter(preferred_parent) != nullptr + ? preferred_parent + : RuntimeProfile::ROOT_COUNTER.c_str(); +} + +} // namespace doris::file_scan_profile diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp index abb64cff9e6a71..47c67a011a1701 100644 --- a/be/src/util/block_compression.cpp +++ b/be/src/util/block_compression.cpp @@ -769,11 +769,19 @@ class SnappyBlockCompression : public BlockCompressionCodec { } Status decompress(const Slice& input, Slice* output) override { + size_t uncompressed_size = 0; + if (!snappy::GetUncompressedLength(input.data, input.size, &uncompressed_size)) { + return Status::InvalidArgument("Fail to get Snappy uncompressed length"); + } + // RawUncompress has no capacity argument, so reject an undersized destination first. + if (uncompressed_size > output->size) { + return Status::InvalidArgument("Snappy output size {} exceeds buffer capacity {}", + uncompressed_size, output->size); + } if (!snappy::RawUncompress(input.data, input.size, output->data)) { return Status::InvalidArgument("Fail to do Snappy decompress"); } - // NOTE: GetUncompressedLength only takes O(1) time - snappy::GetUncompressedLength(input.data, input.size, &output->size); + output->size = uncompressed_size; return Status::OK(); } diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp index 400c8f9381b77c..8c24a886acb2b4 100644 --- a/be/src/util/cpu_info.cpp +++ b/be/src/util/cpu_info.cpp @@ -377,6 +377,17 @@ void CpuInfo::_get_cache_info(long cache_sizes[NUM_CACHE_LEVELS], #endif } +long CpuInfo::get_l2_cache_size() { + static const long l2_cache_size = [] { + long cache_sizes[NUM_CACHE_LEVELS] = {}; + long cache_line_sizes[NUM_CACHE_LEVELS] = {}; + _get_cache_info(cache_sizes, cache_line_sizes); + constexpr long DEFAULT_L2_CACHE_SIZE = 256 * 1024; + return cache_sizes[L2_CACHE] > 0 ? cache_sizes[L2_CACHE] : DEFAULT_L2_CACHE_SIZE; + }(); + return l2_cache_size; +} + std::string CpuInfo::debug_string() { DCHECK(initialized_); std::stringstream stream; diff --git a/be/src/util/cpu_info.h b/be/src/util/cpu_info.h index 19548ecc964659..c75b814721cc57 100644 --- a/be/src/util/cpu_info.h +++ b/be/src/util/cpu_info.h @@ -146,6 +146,10 @@ class CpuInfo { return model_name_; } + // Some virtualized hosts do not expose cache topology. Always return a positive value so hot + // decode paths can use this threshold without repeating platform-specific guards. + static long get_l2_cache_size(); + static std::string debug_string(); /// A utility class for temporarily disabling CPU features. Usage: diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp new file mode 100644 index 00000000000000..971eb067684c34 --- /dev/null +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -0,0 +1,827 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_decimal.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { +namespace { + +#pragma pack(1) +struct TestParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(TestParquetInt96Timestamp) == 12); + +constexpr int32_t JULIAN_UNIX_EPOCH = 2440588; +constexpr int64_t NANOS_PER_DAY = 86400000000000LL; + +class TestParquetDecodeSource final : public ParquetDecodeSource { +public: + template + void set_fixed_values(const std::vector& values) { + _value_width = sizeof(T); + _fixed_values.resize(values.size() * sizeof(T)); + memcpy(_fixed_values.data(), values.data(), _fixed_values.size()); + } + + void set_fixed_bytes(std::vector values, size_t value_width) { + _fixed_values = std::move(values); + _value_width = value_width; + } + + void set_dictionary(std::vector values, size_t value_width, + std::vector indices) { + _dictionary = std::move(values); + _dictionary_width = value_width; + _indices = std::move(indices); + _index_offset = 0; + ++_dictionary_generation; + } + + template + void set_fixed_dictionary(const std::vector& values, std::vector indices) { + std::vector encoded(values.size() * sizeof(T)); + memcpy(encoded.data(), values.data(), encoded.size()); + set_dictionary(std::move(encoded), sizeof(T), std::move(indices)); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_LE((_fixed_offset + num_values) * _value_width, _fixed_values.size()); + const uint8_t* begin = _fixed_values.data() + _fixed_offset * _value_width; + _fixed_offset += num_values; + return consumer.consume(begin, num_values, _value_width); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_LE(_binary_offset + num_values, _binary_refs.size()); + const StringRef* begin = _binary_refs.data() + _binary_offset; + _binary_offset += num_values; + return consumer.consume(begin, num_values); + } + + Status skip_values(size_t num_values) override { + _fixed_offset += num_values; + _binary_offset += num_values; + _index_offset += num_values; + return Status::OK(); + } + + bool has_dictionary() const override { return !_dictionary.empty(); } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + size_t dictionary_size() const override { + return _dictionary_width == 0 ? 0 : _dictionary.size() / _dictionary_width; + } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + ++_dictionary_decode_calls; + return fixed_consumer.consume(_dictionary.data(), dictionary_size(), _dictionary_width); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + DORIS_CHECK_LE(_index_offset + num_values, _indices.size()); + indices->assign(_indices.begin() + _index_offset, + _indices.begin() + _index_offset + num_values); + _index_offset += num_values; + return Status::OK(); + } + + size_t dictionary_decode_calls() const { return _dictionary_decode_calls; } + +private: + std::vector _fixed_values; + std::vector _binary_refs; + std::vector _dictionary; + std::vector _indices; + size_t _value_width = 0; + size_t _dictionary_width = 0; + size_t _fixed_offset = 0; + size_t _binary_offset = 0; + size_t _index_offset = 0; + uint64_t _dictionary_generation = 0; + size_t _dictionary_decode_calls = 0; +}; + +TEST(DataTypeSerDeParquetTest, MaterializesLogicalUnsignedIntegersDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 0, 7}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 32, + .logical_integer_is_signed = false}; + ParquetMaterializationState state; + DataTypeInt64 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 4294967295LL); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 7); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFloat16Directly) { + TestParquetDecodeSource source; + source.set_fixed_values({0x3C00, 0xC000, 0x7C00}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::FLOAT16, + .type_length = 2, + .logical_float16 = true}; + ParquetMaterializationState state; + DataTypeFloat32 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_FLOAT_EQ(data[0], 1.0F); + EXPECT_FLOAT_EQ(data[1], -2.0F); + EXPECT_TRUE(std::isinf(data[2])); +} + +TEST(DataTypeSerDeParquetTest, RescalesFixedBinaryDecimalDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x04, 0xD2, 0xFB, 0x2E}, 2); // 12.34 and -12.34 at scale 2. + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 2, + .decimal_precision = 4, + .decimal_scale = 2}; + ParquetMaterializationState state; + DataTypeDecimal64 type(18, 3); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 12340); + EXPECT_EQ(data[1].value, -12340); +} + +TEST(DataTypeSerDeParquetTest, RescalesExactIntegerDecimalsWithoutRowFailures) { + TestParquetDecodeSource source; + source.set_fixed_values({12300, -98700, 2147483600}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 10, + .decimal_scale = 4}; + ParquetMaterializationState state; + DataTypeDecimal64 type(9, 2); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + // Exact scale reduction can stay on the fast integer path without per-row failure bookkeeping. + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, -987); + EXPECT_EQ(data[2].value, 21474836); +} + +TEST(DataTypeSerDeParquetTest, DecimalScaleDownRejectsLossyPlainAndDictionaryValues) { + ParquetDecodeContext plain_context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 6, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + { + TestParquetDecodeSource source; + source.set_fixed_values({1230, 1234}); + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, 0); + EXPECT_EQ(null_map, IColumn::Filter({0, 1})); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({1234}); + ParquetMaterializationState state; + state.enable_strict_mode = true; + auto column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_dictionary({1230, 1234}, {1, 0, 1}); + auto dictionary_context = plain_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde() + ->read_column_from_parquet(*column, source, dictionary_context, 3, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[1].value, 123); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalUsesWideIntermediateBeforeNarrowing) { + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 19, + .decimal_scale = 0}; + DataTypeDecimal32 type(9, 0); + constexpr int64_t WRAPS_TO_ONE_IN_INT32 = 4294967297LL; + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_fixed_dictionary({WRAPS_TO_ONE_IN_INT32}, {0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_values({WRAPS_TO_ONE_IN_INT32}); + context.encoding = ParquetValueEncoding::PLAIN; + } + IColumn::Filter null_map(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1})); + EXPECT_EQ(assert_cast(*column).get_data()[0].value, 0); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalAcceptsSignExtendedBinaryAfterWideExactScaling) { + const std::vector values {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xCE, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFB, 0x32}; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 8, + .decimal_precision = 19, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_dictionary(values, 8, {1, 0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_bytes(values, 8); + context.encoding = ParquetValueEncoding::PLAIN; + } + ParquetMaterializationState state; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, dictionary ? -123 : 123); + EXPECT_EQ(data[1].value, dictionary ? 123 : -123); + } +} + +TEST(DataTypeSerDeParquetTest, ReusesTypedDictionary) { + TestParquetDecodeSource source; + std::vector dictionary {'a', 'a', 'b', 'b'}; + source.set_dictionary(std::move(dictionary), 2, {1, 0, 1, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2}; + ParquetMaterializationState state; + DataTypeString type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + ASSERT_EQ(state.typed_dictionary->size(), 2); + EXPECT_EQ(column->get_data_at(0).to_string(), "bb"); + EXPECT_EQ(column->get_data_at(1).to_string(), "aa"); + EXPECT_EQ(column->get_data_at(2).to_string(), "bb"); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 1); + EXPECT_EQ(column->get_data_at(3).to_string(), "aa"); + + source.set_dictionary({'c', 'c'}, 2, {0}); + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 2); + EXPECT_EQ(column->get_data_at(4).to_string(), "cc"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDictionaryIndicesWithoutValues) { + TestParquetDecodeSource source; + source.set_dictionary({'a', 'a', 'b', 'b'}, 2, {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2, + .dictionary_index_only = true}; + ParquetMaterializationState state; + DataTypeString type; + auto column = ColumnInt32::create(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = column->get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 1); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 1); + EXPECT_FALSE(state.typed_dictionary); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDateDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, 1, -1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ParquetMaterializationState state; + DataTypeDateV2 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 2), "1969-12-31"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampUnitsAndNegativeEpochDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, -1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1969-12-31 23:59:59.999999"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000001"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimeUnitsDirectly) { + struct TimeUnitCase { + ParquetTimeUnit unit; + int64_t units_per_day; + int64_t expected_last_micros; + }; + const std::vector cases { + {ParquetTimeUnit::MILLIS, 86400000, 86399999000}, + {ParquetTimeUnit::MICROS, 86400000000, 86399999999}, + {ParquetTimeUnit::NANOS, 86400000000000, 86399999999}, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(static_cast(test_case.unit)); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = test_case.unit}; + DataTypeTimeV2 type(6); + + TestParquetDecodeSource source; + source.set_fixed_values({0, test_case.units_per_day - 1}); + ParquetMaterializationState state; + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0], 0); + EXPECT_EQ(data[1], test_case.expected_last_micros); + + TestParquetDecodeSource nullable_source; + nullable_source.set_fixed_values({-1, test_case.units_per_day}); + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState nullable_state; + nullable_state.conversion_failure_null_map = &null_map; + auto nullable_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*nullable_column, nullable_source, context, + 2, nullable_state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + + TestParquetDecodeSource strict_source; + strict_source.set_fixed_values({-1}); + ParquetMaterializationState strict_state; + auto strict_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*strict_column, strict_source, context, 1, + strict_state) + .ok()); + EXPECT_EQ(strict_column->size(), 0); + + TestParquetDecodeSource unused_invalid_dictionary; + unused_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day - 1, test_case.units_per_day}, {1, 2}); + auto dictionary_context = context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState dictionary_state; + auto dictionary_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, + unused_invalid_dictionary, + dictionary_context, 2, dictionary_state) + .ok()); + EXPECT_EQ(dictionary_column->size(), 2); + + TestParquetDecodeSource referenced_invalid_dictionary; + referenced_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day}, {0}); + ParquetMaterializationState invalid_dictionary_state; + auto invalid_dictionary_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet( + *invalid_dictionary_column, referenced_invalid_dictionary, + dictionary_context, 1, invalid_dictionary_state) + .ok()); + EXPECT_EQ(invalid_dictionary_column->size(), 0); + } +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampTzDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1969); + EXPECT_EQ(data[0].microsecond(), 999999); + EXPECT_EQ(data[1].second(), 1); + EXPECT_EQ(data[1].microsecond(), 1); +} + +TEST(DataTypeSerDeParquetTest, ValidatesInt96BeforeDateTimeMaterialization) { + { + TestParquetDecodeSource source; + source.set_fixed_values( + {{0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY - 1, JULIAN_UNIX_EPOCH}}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 23:59:59.999999"); + } + + const std::vector invalid_values { + {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}, + {-1, JULIAN_UNIX_EPOCH}, + {0, std::numeric_limits::max()}}; + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, Int96DictionaryFailuresFollowDecodedIds) { + const std::vector dictionary_values { + {0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + std::vector dictionary(sizeof(TestParquetInt96Timestamp) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(TestParquetInt96Timestamp), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 00:00:00.000000"); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzChecksUnitOverflowAndTargetRange) { + constexpr int64_t MIN_TIMESTAMP_MICROS = -62135596800000000LL; + constexpr int64_t MAX_TIMESTAMP_MICROS = 253402300799999999LL; + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + + { + TestParquetDecodeSource source; + source.set_fixed_values({MIN_TIMESTAMP_MICROS, MAX_TIMESTAMP_MICROS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1); + EXPECT_EQ(data[1].year(), 9999); + EXPECT_EQ(data[1].microsecond(), 999999); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max(), YEAR_10000_MILLIS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDecodedValuesUseTheSameBounds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector values {std::numeric_limits::max(), YEAR_10000_MILLIS, 0}; + NullMap conversion_failures(3, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT64, + .time_unit = DecodedTimeUnit::MILLIS, + .row_count = 3, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(conversion_failures, NullMap({1, 1, 0})); + EXPECT_EQ(assert_cast(*column).get_data()[2].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDictionaryFailuresFollowDecodedIds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector dictionary_values {0, YEAR_10000_MILLIS}; + std::vector dictionary(sizeof(int64_t) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(int64_t), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(assert_cast(*column).get_data()[1].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, Int96DecodedValuesRejectInvalidNanos) { + const std::vector values {{NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + NullMap conversion_failures(1, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT96, + .row_count = 1, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(conversion_failures, NullMap({1})); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFixedBinaryAsVarbinaryDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F}, + 16); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::UUID, + .type_length = 16, + .logical_uuid = true}; + ParquetMaterializationState state; + DataTypeVarbinary type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->get_data_at(0).to_string(), std::string("\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F", + 16)); +} + +TEST(DataTypeSerDeParquetTest, NullableConversionFailuresBecomeNullOutsideStrictMode) { + auto expect_null = [](DataTypeSerDeSPtr serde, MutableColumnPtr column, + TestParquetDecodeSource* source, const ParquetDecodeContext& context) { + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + EXPECT_TRUE(serde->read_column_from_parquet(*column, *source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + }; + + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::min()}); + DataTypeDateV2 type; + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeDateTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({10}); + DataTypeDecimal32 type(1, 0); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 2, + .decimal_scale = 0}); + } +} + +TEST(DataTypeSerDeParquetTest, DictionaryConversionFailuresFollowDecodedIds) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt8 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map[0], 1); + EXPECT_EQ(null_map[1], 1); + EXPECT_EQ(source.dictionary_decode_calls(), 1); +} + +TEST(DataTypeSerDeParquetTest, NonNullableDictionaryConversionFailureRemainsAnError) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ParquetMaterializationState state; + DataTypeInt8 type; + auto column = type.create_column(); + + EXPECT_FALSE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 0); +} + +} // namespace +} // namespace doris diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 762598456e09e1..d28c3d96e74a8d 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -719,6 +719,15 @@ TEST(FileScannerV2Test, FileCacheStatisticsArePublishedToScannerProfile) { EXPECT_EQ(profile.get_counter("BytesWriteIntoCache")->value(), 19); ASSERT_NE(profile.get_info_string("PeerCacheNodes"), nullptr); EXPECT_EQ(*profile.get_info_string("PeerCacheNodes"), "peer-a, peer-b"); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + ASSERT_TRUE(children.contains("IO")); + EXPECT_TRUE(children.at("IO").contains("FileCache")); } TEST(FileScannerV2Test, NotFoundIsSkippedOnlyWhenConfigured) { diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index c0318d451187ed..c5d837bdff756e 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -2410,6 +2410,7 @@ TEST(ColumnMapperScanRequestTest, HiddenTopLevelFilterMappingUsesNameFallback) { EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(1)})); ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(1)).is_local()); EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(1)).local_index(), LocalIndex(1)); } @@ -2935,6 +2936,7 @@ TEST(ColumnMapperScanRequestTest, PredicateOnlyTopLevelColumnUsesHiddenMapping) EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(10)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(10)})); EXPECT_TRUE(request.predicate_columns[0].project_all_children); EXPECT_TRUE(request.predicate_columns[0].children.empty()); diff --git a/be/test/format_v2/delimited_text/csv_reader_test.cpp b/be/test/format_v2/delimited_text/csv_reader_test.cpp index 0ce99630c7af8b..801187474e1be5 100644 --- a/be/test/format_v2/delimited_text/csv_reader_test.cpp +++ b/be/test/format_v2/delimited_text/csv_reader_test.cpp @@ -446,7 +446,7 @@ TEST_F(CsvV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(file_reader_stats.read_rows, 3); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); diff --git a/be/test/format_v2/delimited_text/text_reader_test.cpp b/be/test/format_v2/delimited_text/text_reader_test.cpp index f5f7309e490ff1..97fe3beb0168cb 100644 --- a/be/test/format_v2/delimited_text/text_reader_test.cpp +++ b/be/test/format_v2/delimited_text/text_reader_test.cpp @@ -41,6 +41,7 @@ #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "format_v2/column_mapper.h" +#include "format_v2/parquet/parquet_profile.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" #include "testutil/desc_tbl_builder.h" @@ -414,7 +415,7 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); EXPECT_EQ(counter_value(&_profile, "RowsReturned"), 1); @@ -423,6 +424,33 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6); } +TEST_F(TextV2ReaderTest, FormatProfilesKeepDistinctCountersInBothInitializationOrders) { + auto expect_format_children = [](RuntimeProfile* profile) { + TRuntimeProfileTree tree; + profile->to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("DelimitedTextReader")); + EXPECT_TRUE(children.at("DelimitedTextReader").contains("DelimitedRowsFilteredByConjunct")); + EXPECT_FALSE(children.at("DelimitedTextReader").contains("RowsFilteredByConjunct")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowsFilteredByConjunct")); + EXPECT_FALSE(children.at("ParquetReader").contains("DelimitedRowsFilteredByConjunct")); + }; + + RuntimeProfile parquet_first("parquet_first"); + parquet::ParquetProfile parquet_first_counters; + parquet_first_counters.init(&parquet_first); + auto parquet_first_text = create_reader(_file_path, &_params, _slots, &_state, &parquet_first); + expect_format_children(&parquet_first); + + RuntimeProfile text_first("text_first"); + auto text_first_reader = create_reader(_file_path, &_params, _slots, &_state, &text_first); + parquet::ParquetProfile text_first_parquet_counters; + text_first_parquet_counters.init(&text_first); + expect_format_children(&text_first); +} + // Scenario: Hive text has no embedded nested schema, but TableColumnMapper still needs semantic // children for complex table columns. The reader synthesizes ARRAY/MAP/STRUCT children from the // slot type while keeping the top-level local id as the text field ordinal from column_idxs. diff --git a/be/test/format_v2/jni/jni_table_reader_test.cpp b/be/test/format_v2/jni/jni_table_reader_test.cpp index e3e5e69b825394..95eda4a557e790 100644 --- a/be/test/format_v2/jni/jni_table_reader_test.cpp +++ b/be/test/format_v2/jni/jni_table_reader_test.cpp @@ -19,10 +19,12 @@ #include +#include #include #include #include #include +#include #include #include "io/io_common.h" @@ -40,9 +42,15 @@ class FakeJniTableReader final : public JniTableReader { std::vector propagated_batch_sizes; std::vector close_results; bool next_eof = false; + std::chrono::milliseconds init_delay {0}; + std::chrono::milliseconds open_delay {0}; + std::chrono::milliseconds close_delay {0}; protected: - std::string connector_class() const override { return "test/FakeJniScanner"; } + std::string connector_class() const override { + std::this_thread::sleep_for(init_delay); + return "test/FakeJniScanner"; + } Status build_scanner_params(std::map* params) const override { params->clear(); @@ -57,6 +65,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _close_jni_scanner() override { + std::this_thread::sleep_for(close_delay); if (!TEST_scanner_opened()) { return Status::OK(); } @@ -80,6 +89,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _open_jni_scanner() override { + std::this_thread::sleep_for(open_delay); open_batch_sizes.push_back(TEST_batch_size()); TEST_set_split_state(true, false); return Status::OK(); @@ -191,5 +201,36 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { EXPECT_TRUE(reader.TEST_scanner_opened()); } +TEST(JniTableReaderTest, CommonLifecycleTimersContainJniLifecycleWork) { + constexpr auto delay = std::chrono::milliseconds(8); + RuntimeProfile profile("JniLifecycleContainment"); + FakeJniTableReader reader; + reader.init_delay = delay; + ASSERT_TRUE(init_reader(&reader, nullptr, &profile).ok()); + ASSERT_GE(profile.get_counter("InitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.open_delay = delay; + ASSERT_TRUE(reader.prepare_split({ + .partition_values = {}, + .conjuncts = std::nullopt, + .partition_prune_conjuncts = {}, + .all_runtime_filters_applied = true, + .condition_cache_digest = std::nullopt, + .cache = nullptr, + .current_range = {}, + .current_split_format = FileFormat::JNI, + .global_rowid_context = std::nullopt, + }) + .ok()); + ASSERT_GE(profile.get_counter("PrepareSplitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.close_delay = delay; + ASSERT_TRUE(reader.close().ok()); + EXPECT_GE(profile.get_counter("CloseTime")->value(), + std::chrono::duration_cast(delay).count()); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/orc/orc_file_input_stream_test.cpp b/be/test/format_v2/orc/orc_file_input_stream_test.cpp index 54c89505edc3f5..8d63a4ab21ae65 100644 --- a/be/test/format_v2/orc/orc_file_input_stream_test.cpp +++ b/be/test/format_v2/orc/orc_file_input_stream_test.cpp @@ -329,14 +329,49 @@ TEST(OrcFileInputStreamTest, PublishesClusterProfileExactlyOnce) { input.beforeReadStripe( std::make_unique(200, std::vector {}), selected_columns({}), next_streams); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); } - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); +} + +TEST(OrcFileInputStreamTest, MergedIoChildrenStayIsolatedInBothInitializationOrders) { + for (const bool generic_first : {false, true}) { + auto reader = std::make_shared(256); + RuntimeProfile profile(generic_first ? "generic_first" : "orc_first"); + auto register_generic = [&] { + ADD_TIMER(&profile, "MergedSmallIO"); + return ADD_CHILD_COUNTER_WITH_LEVEL(&profile, "RequestIO", TUnit::UNIT, "MergedSmallIO", + 1); + }; + RuntimeProfile::Counter* generic_request_io = nullptr; + if (generic_first) { + generic_request_io = register_generic(); + } + { + OrcFileInputStream input("test.orc", reader, nullptr, &profile, + {.once_max_read_bytes = 16, .max_merge_distance_bytes = 0}); + if (!generic_first) { + generic_request_io = register_generic(); + } + StripeStreamMap streams; + input.beforeReadStripe( + std::make_unique( + 100, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams); + std::array data {}; + find_stream(streams, 1, ::orc::StreamKind_DATA)->read(data.data(), data.size(), 100); + } + ASSERT_NE(generic_request_io, nullptr); + EXPECT_EQ(generic_request_io->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 1); + } } } // namespace diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 6809186f115b59..20696bcc2ff64a 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -73,6 +73,7 @@ #include "format_v2/expr/cast.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "gen_cpp/Types_types.h" #include "io/fs/buffered_reader.h" #include "io/io_common.h" @@ -5262,16 +5263,17 @@ TEST_F(NewOrcReaderTest, StripePrefetchPublishesMergedReadProfile) { } ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - ASSERT_NE(profile.get_counter("ApplyBytes"), nullptr); - ASSERT_NE(profile.get_counter("ClusterNum"), nullptr); - ASSERT_NE(profile.get_counter("OverReadBytes"), nullptr); - EXPECT_GT(profile.get_counter("RequestIO")->value(), profile.get_counter("MergedIO")->value()); - EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); - EXPECT_GT(profile.get_counter("ApplyBytes")->value(), 0); - EXPECT_GT(profile.get_counter("ClusterNum")->value(), 0); - EXPECT_GE(profile.get_counter("OverReadBytes")->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedApplyBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedClusterNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedOverReadBytes"), nullptr); + EXPECT_GT(profile.get_counter("OrcMergedRequestIO")->value(), + profile.get_counter("OrcMergedIO")->value()); + EXPECT_GT(profile.get_counter("OrcMergedIO")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedApplyBytes")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedClusterNum")->value(), 0); + EXPECT_GE(profile.get_counter("OrcMergedOverReadBytes")->value(), 0); } TEST_F(NewOrcReaderTest, StripePrefetchCanBeDisabledByZeroOnceMaxReadBytes) { @@ -5722,8 +5724,8 @@ TEST_F(NewOrcReaderTest, ClosePublishesOrcLazyStatisticsToRuntimeProfile) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 2); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 2); } TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { @@ -5752,8 +5754,8 @@ TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); } TEST_F(NewOrcReaderTest, ConditionCacheMissMarksSurvivingGranules) { @@ -6701,13 +6703,13 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { ASSERT_EQ(result_rows, 200); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsReadNum"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByGroup"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - ASSERT_NE(profile.get_counter("FilteredBytes"), nullptr); - ASSERT_NE(profile.get_counter("FileNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsReadNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByGroup"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcFileNum"), nullptr); const std::array orc_reader_metric_counters { "ReaderCall", "ReaderInclusiveLatencyUs", @@ -6726,19 +6728,63 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { for (const auto counter_name : orc_reader_metric_counters) { ASSERT_NE(profile.get_counter(std::string(counter_name)), nullptr) << counter_name; } - EXPECT_EQ(profile.get_counter("RowGroupsFiltered")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsReadNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFiltered")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsReadNum")->value(), 1); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 1); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 3); - EXPECT_EQ(profile.get_counter("FilteredRowsByGroup")->value(), 400); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); - EXPECT_GT(profile.get_counter("FilteredBytes")->value(), 0); - EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByGroup")->value(), 400); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); + EXPECT_GT(profile.get_counter("OrcFilteredBytes")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcFileNum")->value(), 1); EXPECT_EQ(profile.get_counter("ReadRowCount")->value(), static_cast(file_reader_stats.read_rows)); } +TEST_F(NewOrcReaderTest, ProfileKeepsPruningCountersBelowOrcReader) { + RuntimeProfile profile("new_orc_reader_hierarchy_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("OrcReader")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsFiltered")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsReadNum")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcFilteredRowsByGroup")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenParquetInitializesFirst) { + RuntimeProfile profile("parquet_then_orc_profile"); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenOrcInitializesFirst) { + RuntimeProfile profile("orc_then_parquet_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { const auto multi_stripe_file_path = (_test_dir / "profile_minmax_disabled.orc").string(); write_multi_stripe_orc_int_file(multi_stripe_file_path, {1, 1000, 2000}); @@ -6775,10 +6821,10 @@ TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { ASSERT_NE(profile.get_counter("SelectedRowGroupCount"), nullptr); ASSERT_NE(profile.get_counter("EvaluatedRowGroupCount"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 0); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 0); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 0); } TEST_F(NewOrcReaderTest, SargConjunctPrunesStripes) { diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp new file mode 100644 index 00000000000000..bcf3e851d765c4 --- /dev/null +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -0,0 +1,3230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "core/custom_allocator.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/column_reader.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" +#include "util/block_compression.h" +#include "util/coding.h" +#include "util/faststring.h" +#include "util/rle_encoding.h" +#include "util/thrift_util.h" +#include "util/timezone_utils.h" + +namespace doris::format::parquet::native { +namespace { + +VExprSPtr create_int32_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t literal, + bool literal_on_left = false) { + const auto int_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + auto slot = VSlotRef::create_shared(column_id, column_id, -1, int_type, "plain_int"); + auto value = VLiteral::create_shared(int_type, Field::create_field(literal)); + if (literal_on_left) { + root->add_child(value); + root->add_child(slot); + } else { + root->add_child(slot); + root->add_child(value); + } + return root; +} + +VExprSPtr create_float_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, float literal) { + const auto float_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({float_type->to_thrift(), float_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + root->add_child(VSlotRef::create_shared(column_id, column_id, -1, float_type, "plain_float")); + root->add_child(VLiteral::create_shared(float_type, Field::create_field(literal))); + return root; +} + +TEST(ParquetV2NativeDecoderTest, RawExprEvaluatesPhysicalValuesWithoutColumn) { + const std::array values = {1, 4, 7, 10, 13, 16}; + std::array matches {}; + matches.fill(1); + const auto greater_equal = create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 7); + const auto less_than = create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 16); + const auto int_type = std::make_shared(); + + ASSERT_TRUE(greater_equal->can_execute_on_raw_fixed_values(int_type, 0)); + ASSERT_TRUE(greater_equal + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + ASSERT_TRUE(less_than + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1, 1, 0})); + + matches.fill(1); + const auto literal_less_than_slot = + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 7, true); + ASSERT_TRUE(literal_less_than_slot + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, RawExprPreservesFloatNanOrdering) { + const float nan = std::numeric_limits::quiet_NaN(); + const std::array values {-1, 0, 1, nan}; + const auto float_type = std::make_shared(); + + std::array matches {}; + matches.fill(1); + const auto greater_than_zero = create_float_raw_comparison(0, "gt", TExprOpcode::GT, 0); + ASSERT_TRUE(greater_than_zero + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1})); + + matches.fill(1); + const auto equals_nan = create_float_raw_comparison(0, "eq", TExprOpcode::EQ, nan); + ASSERT_TRUE(equals_nan + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1})); +} + +class RejectFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return Status::InternalError("Unexpected fixed dictionary"); + } +}; + +class CaptureBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + refs.assign(values, values + num_values); + return Status::OK(); + } + + std::vector refs; +}; + +class CapturePlainBinaryLayoutConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + legacy_consume_called = true; + return Status::InternalError("PLAIN BYTE_ARRAY used the legacy StringRef path"); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + base = encoded_data; + source_offsets.assign(payload_offsets, payload_offsets + num_values); + output_offsets.assign(value_offsets, value_offsets + num_values + 1); + spans = value_spans; + return Status::OK(); + } + + const char* base = nullptr; + bool legacy_consume_called = false; + std::vector source_offsets; + std::vector output_offsets; + std::vector spans; +}; + +class CaptureFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if (width == 0) { + width = value_width; + } + DORIS_CHECK_EQ(width, value_width); + bytes.insert(bytes.end(), values, values + num_values * value_width); + return Status::OK(); + } + + template + std::vector values() const { + DORIS_CHECK_EQ(width, sizeof(T)); + DORIS_CHECK_EQ(bytes.size() % sizeof(T), 0); + std::vector result(bytes.size() / sizeof(T)); + memcpy(result.data(), bytes.data(), bytes.size()); + return result; + } + + size_t width = 0; + std::vector bytes; +}; + +class ScriptedDictionaryMaterializationSource final : public ParquetDecodeSource { +public: + ScriptedDictionaryMaterializationSource(std::vector ids, bool prefer_indices) + : _ids(std::move(ids)), _prefer_indices(prefer_indices) {} + + Status decode_fixed_values(size_t, ParquetFixedValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no fixed values"); + } + + Status decode_binary_values(size_t, ParquetBinaryValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no binary values"); + } + + Status skip_values(size_t) override { return Status::OK(); } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++index_batches; + *indices = _ids; + return Status::OK(); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++direct_batches; + return consumer.consume_indices(_ids.data(), _ids.size()); + } + + bool prefer_dictionary_index_materialization(size_t) const override { return _prefer_indices; } + + size_t direct_batches = 0; + size_t index_batches = 0; + +private: + std::vector _ids; + bool _prefer_indices; +}; + +const RowRanges& scripted_row_ranges() { + static const RowRanges ranges; + return ranges; +} + +class ScriptedColumnReader final : public ColumnReader { +public: + ScriptedColumnReader(size_t rows, size_t values, bool eof, std::vector rep_levels, + std::vector def_levels) + : ColumnReader(scripted_row_ranges(), rows, nullptr, nullptr), + _rows(rows), + _values(values), + _eof(eof), + _rep_levels(std::move(rep_levels)), + _def_levels(std::move(def_levels)) {} + + Status read_column_data(ColumnPtr& column, const DataTypePtr&, + const std::shared_ptr&, FilterMap&, size_t, + size_t* read_rows, bool* eof, bool, int64_t = -1) override { + if (_used) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + column = IColumn::mutate(std::move(column)); + column->assert_mutable()->insert_many_defaults(_values); + *read_rows = _rows; + *eof = _eof; + _used = true; + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t* read_rows, bool* eof) override { + *read_rows = _rows; + *eof = _eof; + return Status::OK(); + } + + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { return {}; } + void close() override {} + void release_batch_scratch(size_t) override {} + void reset_filter_map_index() override {} + +private: + size_t _rows; + size_t _values; + bool _eof; + bool _used = false; + std::vector _rep_levels; + std::vector _def_levels; +}; + +class MemoryBufferedReader final : public io::BufferedStreamReader { +public: + explicit MemoryBufferedReader(std::vector data) : _data(std::move(data)) {} + + Status read_bytes(const uint8_t** buf, uint64_t offset, size_t bytes_to_read, + const io::IOContext*) override { + if (offset > _data.size() || bytes_to_read > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + *buf = _data.data() + offset; + return Status::OK(); + } + Status read_bytes(Slice& slice, uint64_t offset, const io::IOContext*) override { + if (offset > _data.size() || slice.size > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + slice.data = reinterpret_cast(_data.data() + offset); + return Status::OK(); + } + std::string path() override { return "memory.parquet"; } + int64_t mtime() const override { return 0; } + +private: + std::vector _data; +}; + +std::shared_ptr<::parquet::ColumnDescriptor> descriptor(::parquet::Type::type physical_type) { + auto node = ::parquet::schema::PrimitiveNode::Make("value", ::parquet::Repetition::REQUIRED, + physical_type); + return std::make_shared<::parquet::ColumnDescriptor>(node, 0, 0); +} + +DorisUniqueBufferPtr make_byte_array_dictionary(const std::vector& values, + int32_t* length) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + *length = static_cast(total_size); + auto dictionary = make_unique_buffer(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(dictionary.get() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(dictionary.get() + offset, value.data(), value.size()); + offset += value.size(); + } + return dictionary; +} + +std::vector encode_plain_byte_arrays(const std::vector& values) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + std::vector encoded(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(encoded.data() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(encoded.data() + offset, value.data(), value.size()); + offset += value.size(); + } + return encoded; +} + +std::vector serialize_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +Status materialize_level_only_page(bool data_page_v2, tparquet::Type::type physical_type, + tparquet::Encoding::type encoding, bool all_null) { + constexpr size_t VALUE_COUNT = 3; + const std::vector encoded_levels {static_cast(VALUE_COUNT << 1), + static_cast(all_null ? 0 : 1)}; + std::vector payload; + tparquet::PageHeader header; + if (data_page_v2) { + payload = encoded_levels; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(VALUE_COUNT); + header.data_page_header_v2.__set_num_nulls(all_null ? VALUE_COUNT : 0); + header.data_page_header_v2.__set_num_rows(VALUE_COUNT); + header.data_page_header_v2.__set_encoding(encoding); + header.data_page_header_v2.__set_definition_levels_byte_length(encoded_levels.size()); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } else { + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), encoded_levels.size()); + payload.insert(payload.end(), encoded_levels.begin(), encoded_levels.end()); + header.type = tparquet::PageType::DATA_PAGE; + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(VALUE_COUNT); + header.data_page_header.__set_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + } + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(VALUE_COUNT); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = physical_type; + if (physical_type == tparquet::Type::BOOLEAN) { + field.data_type = std::make_shared(); + } else { + field.data_type = std::make_shared(); + } + field.definition_level = 1; + field.parquet_schema.__set_type(physical_type); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, VALUE_COUNT, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + const auto load_status = chunk_reader.load_page_data(); + EXPECT_TRUE(load_status.ok()) << load_status; + RETURN_IF_ERROR(load_status); + + level_t level = -1; + EXPECT_EQ(chunk_reader.def_level_decoder().get_next_run(&level, VALUE_COUNT), VALUE_COUNT); + EXPECT_EQ(level, all_null ? 0 : 1); + FilterMap filter; + RETURN_IF_ERROR(filter.init(nullptr, VALUE_COUNT, false)); + NullMap null_map; + ColumnSelectVector select_vector; + const std::vector null_runs {static_cast(all_null ? 0 : VALUE_COUNT), + static_cast(all_null ? VALUE_COUNT : 0)}; + RETURN_IF_ERROR(select_vector.init(null_runs, VALUE_COUNT, &null_map, &filter, 0)); + auto column = field.data_type->create_column(); + ParquetDecodeContext decode_context; + static const auto utc = cctz::utc_time_zone(); + RETURN_IF_ERROR(init_decode_context_for_test(field, &utc, &decode_context)); + ParquetMaterializationState state; + state.enable_strict_mode = true; + const auto status = chunk_reader.materialize_values(column, *field.data_type->get_serde(), + decode_context, state, select_vector); + if (status.ok()) { + EXPECT_EQ(column->size(), VALUE_COUNT); + EXPECT_EQ(null_map, NullMap(VALUE_COUNT, all_null ? 1 : 0)); + } + return status; +} + +Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, + tparquet::CompressionCodec::type codec, bool preload_page_cache = false) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + RETURN_IF_ERROR(serializer.serialize(&header, &bytes)); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + const size_t chunk_size = bytes.size(); + const std::string page_cache_file_key = "native-scripted-page-" + + std::to_string(static_cast(header.type)) + "-" + + std::to_string(header.compressed_page_size) + "-" + + std::to_string(header.uncompressed_page_size); + if (preload_page_cache) { + auto* page = new DataPage(bytes.size(), true, segment_v2::DATA_PAGE); + memcpy(page->data(), bytes.data(), bytes.size()); + page->reset_size(bytes.size()); + PageCacheHandle handle; + StoragePageCache::instance()->insert( + StoragePageCache::CacheKey(page_cache_file_key, chunk_size, 0), page, &handle, + segment_v2::DATA_PAGE); + } + MemoryBufferedReader reader(std::move(bytes)); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(codec); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(chunk_size); + if (header.type == tparquet::PageType::DICTIONARY_PAGE) { + chunk.meta_data.__set_dictionary_page_offset(0); + chunk.meta_data.__set_data_page_offset(0); + } else { + chunk.meta_data.__set_data_page_offset(0); + } + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(preload_page_cache, page_cache_file_key); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + return chunk_reader.load_page_data(); +} + +Status load_malformed_nested_page(tparquet::PageHeader header, const std::vector& payload, + int64_t metadata_values = std::numeric_limits::max(), + level_t max_repetition_level = 1) { + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(metadata_values); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = max_repetition_level; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + std::vector rep_levels; + size_t result_rows = 0; + bool cross_page = false; + return chunk_reader.load_page_nested_rows(rep_levels, 1, &result_rows, &cross_page); +} + +Status materialize_plain_int96(const std::vector& values, + const std::vector& filter_values = {}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__set_uncompressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values.size()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto bytes = serialize_page( + header, std::vector(reinterpret_cast(values.data()), + reinterpret_cast(values.data()) + + values.size() * sizeof(ParquetInt96Timestamp))); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT96); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(values.size()); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT96; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, values.size(), + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::INT96; + static const auto utc = cctz::utc_time_zone(); + decode_context.timezone = &utc; + ParquetMaterializationState state; + state.enable_strict_mode = true; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.empty() ? nullptr : filter_values.data(), + filter_values.size(), false)); + ColumnSelectVector select_vector; + const std::vector run_length_null_map {static_cast(values.size()), 0}; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, values.size(), nullptr, &filter, 0)); + return chunk_reader.materialize_values(column, *type.get_serde(), decode_context, state, + select_vector); +} + +template +Status materialize_selected_plain_fixed( + const std::vector& physical_values, size_t logical_values, + const std::vector& run_length_null_map, const std::vector& filter_values, + tparquet::Type::type parquet_physical_type, ParquetPhysicalType decode_physical_type, + const DataType& type, MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics, bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(PhysicalType)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(PhysicalType)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(static_cast(logical_values)); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(PhysicalType)); + if (!payload.empty()) { + memcpy(payload.data(), physical_values.data(), payload.size()); + } + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(parquet_physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = parquet_physical_type; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = decode_physical_type; + if (context_override != nullptr) { + decode_context = *context_override; + } + ParquetMaterializationState state; + state.enable_strict_mode = strict_mode; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +template +Status materialize_selected_plain_int32(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT32, + ParquetPhysicalType::INT32, type, column, null_map, + statistics, strict_mode, context_override); +} + +template +Status materialize_selected_plain_int64(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + const ParquetDecodeContext& context) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT64, + ParquetPhysicalType::INT64, type, column, null_map, + statistics, false, &context); +} + +template +Status materialize_selected_dictionary_int32( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics) { + std::vector dictionary_payload(dictionary.size() * sizeof(int32_t)); + memcpy(dictionary_payload.data(), dictionary.data(), dictionary_payload.size()); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + // Real Parquet files place column chunks after the file header. Keep the dictionary offset + // non-zero because zero is the metadata sentinel for "no dictionary page". + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 4); + for (const auto id : physical_ids) { + encoder.Put(id); + } + // Parquet bit-packed dictionary runs declare groups of eight; emit the trailing padding so a + // decoder that buffers the declared run never reads beyond the synthetic page. + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 4; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::INT32; + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +Status materialize_selected_dictionary_strings(const std::vector& dictionary, + const std::vector& physical_ids, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics) { + auto dictionary_payload = encode_plain_byte_arrays(dictionary); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (const auto id : physical_ids) { + encoder.Put(id); + } + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 2; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::BYTE_ARRAY); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = tparquet::Type::BYTE_ARRAY; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeString type; + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::BYTE_ARRAY; + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +TEST(ParquetV2NativeDecoderTest, RawExprMapsNullableSparseRowsDirectly) { + const std::vector physical_values {1, 4, 7, 10, 13}; + constexpr size_t LOGICAL_VALUES = 7; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(LOGICAL_VALUES); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(int32_t)); + memcpy(payload.data(), physical_values.data(), payload.size()); + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(LOGICAL_VALUES); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, LOGICAL_VALUES, + nullptr, page_context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + + const std::vector null_runs {1, 1, 2, 1, 2}; + const std::vector input_filter {1, 0, 1, 1, 1, 0, 1}; + FilterMap filter; + ASSERT_TRUE(filter.init(input_filter.data(), input_filter.size(), false).ok()); + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector.init(null_runs, LOGICAL_VALUES, nullptr, &filter, 0).ok()); + const VExprSPtrs predicates {create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 4), + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 13)}; + NullMap selected_nulls; + IColumn::Filter physical_matches; + IColumn::Filter row_filter; + bool used_filter = false; + ASSERT_TRUE(chunk_reader + .filter_plain_values(predicates, 0, select_vector, &selected_nulls, + &physical_matches, &row_filter, &used_filter) + .ok()); + + EXPECT_TRUE(used_filter); + EXPECT_EQ(selected_nulls, (NullMap {0, 0, 0, 1, 0})); + EXPECT_EQ(physical_matches, (IColumn::Filter {0, 1, 1, 0})); + EXPECT_EQ(row_filter, (IColumn::Filter {0, 1, 1, 0, 0})); + EXPECT_EQ(chunk_reader.remaining_num_values(), 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainPrimitiveSelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 0, 500, 0, 1000, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDecimalSelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeDecimal32 type(9, 0); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 9, + .decimal_scale = 0}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + const auto& values = assert_cast(*column).get_data(); + ASSERT_EQ(values.size(), 5); + EXPECT_EQ(values[0].value, 0); + EXPECT_EQ(values[1].value, 500); + EXPECT_EQ(values[2].value, 0); + EXPECT_EQ(values[3].value, 1000); + EXPECT_EQ(values[4].value, 0); + EXPECT_EQ(null_map, (NullMap {0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateSelectionBatchesPhysicalPayload) { + const std::vector physical_days {0, 1, 2, 3, 4}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateV2 type; + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_days, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-04"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateTimeSelectionBatchesPhysicalPayload) { + const std::vector physical_micros {0, 1'000'000, 2'000'000, 3'000'000, 4'000'000}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS}; + ASSERT_TRUE(materialize_selected_plain_int64(physical_micros, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000000"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-01 00:00:03.000000"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseDictionarySelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector dictionary(16); + std::iota(dictionary.begin(), dictionary.end(), 100); + // One value followed by nineteen NULLs exercises the sparse-null threshold (<10%). + std::vector null_runs; + for (size_t row = 0; row < LOGICAL_VALUES;) { + null_runs.push_back(1); + ++row; + const auto null_count = std::min(19, LOGICAL_VALUES - row); + null_runs.push_back(cast_set(null_count)); + row += null_count; + } + std::vector physical_ids((LOGICAL_VALUES + 19) / 20); + for (size_t index = 0; index < physical_ids.size(); ++index) { + physical_ids[index] = static_cast(index % dictionary.size()); + } + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + const auto status = materialize_selected_dictionary_int32( + dictionary, physical_ids, LOGICAL_VALUES, null_runs, filter, type, &column, &null_map, + &statistics); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 100, 102, 0, 104, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationSkipsFailureScanWhenDictionaryIsClean) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + auto& dictionary = assert_cast(*state.typed_dictionary).get_data(); + dictionary = {10, 20, 30, 40}; + state.dictionary_indices = {3, 0, 2, 1, 3}; + state.dictionary_conversion_failures.resize_fill(dictionary.size(), 0); + + auto output = ColumnInt32::create(); + ASSERT_TRUE(state.materialize_dictionary(*output).ok()); + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_failure_scan_rows, 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationUsesCacheAwareExecutionShape) { + auto verify_strategy = [](bool prefer_indices, + ParquetDictionaryMaterializationStrategy expected_strategy, + size_t expected_direct_batches, size_t expected_index_batches) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + assert_cast(*state.typed_dictionary).get_data() = {10, 20, 30, 40}; + ScriptedDictionaryMaterializationSource source({3, 0, 2, 1, 3}, prefer_indices); + auto output = ColumnInt32::create(); + + const auto status = state.materialize_dictionary(*output, source, 5); + EXPECT_TRUE(status.ok()) << status; + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_materialization_strategy, expected_strategy); + EXPECT_EQ(source.direct_batches, expected_direct_batches); + EXPECT_EQ(source.index_batches, expected_index_batches); + }; + + verify_strategy(false, ParquetDictionaryMaterializationStrategy::DIRECT, 1, 0); + verify_strategy(true, ParquetDictionaryMaterializationStrategy::INDICES, 0, 1); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryRepeatedRunsGatherDirectlyIntoDestination) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 1); + for (size_t row = 0; row < 64; ++row) { + encoder.Put(1); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 1; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 64, state).ok()); + EXPECT_EQ(state.dictionary_materialization_strategy, + ParquetDictionaryMaterializationStrategy::DIRECT); + ASSERT_EQ(output->size(), 64); + for (size_t row = 0; row < output->size(); ++row) { + EXPECT_EQ(assert_cast(*output).get_element(row), 20); + } +} + +TEST(ParquetV2NativeDecoderTest, DictionaryDirectGatherRollsBackLateCorruptRun) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (size_t row = 0; row < 8; ++row) { + encoder.Put(0); + } + for (size_t row = 0; row < 8; ++row) { + encoder.Put(3); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 2; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + const auto status = + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 16, state); + EXPECT_TRUE(status.is()) << status; + EXPECT_TRUE(output->empty()); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryIndexBoundsCheckHandlesVectorTailAndUnsignedIds) { + const std::vector valid {0, 7, 1, 6, 2, 5, 3, 4, 7, 0, 6}; + EXPECT_TRUE(native::dictionary_indices_in_bounds(valid.data(), valid.size(), 8)); + + auto invalid_tail = valid; + invalid_tail.back() = 8; + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_tail.data(), invalid_tail.size(), 8)); + + auto invalid_unsigned = valid; + invalid_unsigned[3] = std::numeric_limits::max(); + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_unsigned.data(), + invalid_unsigned.size(), 8)); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionRemapsConversionFailures) { + const std::vector dictionary {1, 1000, 2, 3}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeInt8 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(9); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_int32(dictionary, physical_ids, 7, null_runs, + filter, type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt8::Container {9, 0, 0, 2, 0})); + EXPECT_EQ(null_map, (NullMap {0, 1, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionExpandsStringsInPlace) { + const std::vector dictionary {"a", "bbb", "cc", "dddd"}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeString type; + auto column = type.create_column(); + assert_cast(*column).insert_data("prefix", 6); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_strings(dictionary, physical_ids, 7, null_runs, + filter, &column, &null_map, &statistics) + .ok()); + + ASSERT_EQ(column->size(), 5); + EXPECT_EQ(column->get_data_at(0).to_string_view(), "prefix"); + EXPECT_EQ(column->get_data_at(1).to_string_view(), ""); + EXPECT_EQ(column->get_data_at(2).to_string_view(), "bbb"); + EXPECT_EQ(column->get_data_at(3).to_string_view(), "cc"); + EXPECT_EQ(column->get_data_at(4).to_string_view(), ""); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionHandlesEmptyOutputShapes) { + DataTypeInt32 type; + ColumnChunkReaderStatistics statistics; + + auto all_null_column = type.create_column(); + assert_cast(*all_null_column).get_data().push_back(8); + NullMap all_null_map; + all_null_map.push_back(0); + const std::vector select_alternating {1, 0, 1, 0, 1}; + ASSERT_TRUE(materialize_selected_plain_int32({}, 5, {0, 5}, select_alternating, type, + &all_null_column, &all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto dictionary_all_null_column = type.create_column(); + assert_cast(*dictionary_all_null_column).get_data().push_back(8); + NullMap dictionary_all_null_map; + dictionary_all_null_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10}, {}, 5, {0, 5}, select_alternating, type, + &dictionary_all_null_column, + &dictionary_all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(dictionary_all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto all_filtered_column = type.create_column(); + assert_cast(*all_filtered_column).get_data().push_back(9); + NullMap all_filtered_map; + all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_plain_int32( + {10, 20, 30}, 5, {1, 1, 1, 1, 1}, std::vector(5, 0), type, + &all_filtered_column, &all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto dictionary_all_filtered_column = type.create_column(); + assert_cast(*dictionary_all_filtered_column).get_data().push_back(9); + NullMap dictionary_all_filtered_map; + dictionary_all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10, 20, 30}, {0, 1, 2}, 5, {1, 1, 1, 1, 1}, + std::vector(5, 0), type, + &dictionary_all_filtered_column, + &dictionary_all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(dictionary_all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); + const uint8_t* dictionary_address = dictionary.get(); + ByteArrayDictDecoder decoder; + + ASSERT_TRUE(decoder.set_dict(dictionary, dictionary_length, 2).ok()); + EXPECT_EQ(dictionary.get(), nullptr); + + RejectFixedConsumer fixed_consumer; + CaptureBinaryConsumer binary_consumer; + ASSERT_TRUE(decoder.decode_dictionary(fixed_consumer, binary_consumer).ok()); + ASSERT_EQ(binary_consumer.refs.size(), 2); + EXPECT_EQ(binary_consumer.refs[0].to_string_view(), "alpha"); + EXPECT_EQ(binary_consumer.refs[1].to_string_view(), "beta"); + EXPECT_EQ(binary_consumer.refs[0].data, + reinterpret_cast(dictionary_address + sizeof(uint32_t))); + + // bit width 1, an RLE run of three values (header = 3 << 1), dictionary id 1. + char valid_indices[] = {1, 6, 1}; + Slice valid_slice(valid_indices, sizeof(valid_indices)); + ASSERT_TRUE(decoder.set_data(&valid_slice).ok()); + std::vector decoded_indices; + ASSERT_TRUE(decoder.decode_dictionary_indices(3, &decoded_indices).ok()); + EXPECT_EQ(decoded_indices, std::vector({1, 1, 1})); + + // bit width 2, one RLE value with dictionary id 3. Skipping still validates the encoded id so + // filter selection cannot hide a corrupt dictionary stream. + char invalid_indices[] = {2, 2, 3}; + Slice invalid_slice(invalid_indices, sizeof(invalid_indices)); + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetDecodeSource& source = decoder; + EXPECT_TRUE(source.skip_values(1).is()); + + // Sparse decode must validate filtered dictionary ids too. Predicate selection must never + // turn a corrupt page into a successful read merely because the bad row was not selected. + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetSelection filtered_corrupt {.total_values = 1, .selected_values = 0, .ranges = {}}; + EXPECT_TRUE(decoder.decode_selected_dictionary_indices(filtered_corrupt, &decoded_indices) + .is()); + + Slice empty_indices; + EXPECT_TRUE(decoder.set_data(&empty_indices).is()); + + // Dictionary indices are uint32_t. Wider external bit widths would make the RLE decoder copy + // a five-byte repeated value into four-byte state before it can validate an index. + char oversized_bit_width[] = {33, 2, 0}; + Slice oversized_bit_width_slice(oversized_bit_width, sizeof(oversized_bit_width)); + EXPECT_TRUE(decoder.set_data(&oversized_bit_width_slice).is()); +} + +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryBoundsEntryCountBeforeReserve) { + auto dictionary = make_unique_buffer(1); + dictionary.get()[0] = 0; + ByteArrayDictDecoder decoder; + EXPECT_TRUE(decoder.set_dict(dictionary, 1, std::numeric_limits::max()) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, LegacyConvertedTimestampsRemainUtcAdjusted) { + TimezoneUtils::load_timezones_to_cache(); + for (const auto converted_type : + {tparquet::ConvertedType::TIMESTAMP_MILLIS, tparquet::ConvertedType::TIMESTAMP_MICROS}) { + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT64; + field.parquet_schema.__set_converted_type(converted_type); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.logical_type, ParquetLogicalType::TIMESTAMP); + EXPECT_TRUE(context.timestamp_is_adjusted_to_utc); + + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("Asia/Shanghai", shanghai)); + ASSERT_TRUE(init_decode_context_for_test(field, &shanghai, &context).ok()); + int64_t epoch = 0; + Slice epoch_slice(reinterpret_cast(&epoch), sizeof(epoch)); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(sizeof(epoch)); + ASSERT_TRUE(decoder->set_data(&epoch_slice).ok()); + ParquetMaterializationState state; + DataTypeDateTimeV2 type(0); + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, *decoder, context, 1, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 08:00:00"); + } +} + +TEST(ParquetV2NativeDecoderTest, GeospatialByteArrayAnnotationsDecodeAsRawBytes) { + const std::string wkb("\x01\x01\x00\x00\x00", 5); + for (const bool geometry : {true, false}) { + tparquet::LogicalType logical; + if (geometry) { + logical.__set_GEOMETRY(tparquet::GeometryType()); + } else { + logical.__set_GEOGRAPHY(tparquet::GeographyType()); + } + + NativeFieldSchema field; + field.name = geometry ? "geometry" : "geography"; + field.physical_type = tparquet::Type::BYTE_ARRAY; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.physical_type, ParquetPhysicalType::BYTE_ARRAY); + EXPECT_EQ(context.logical_type, ParquetLogicalType::NONE); + + DataTypeString type; + auto plain_column = type.create_column(); + auto encoded = encode_plain_byte_arrays({wkb}); + Slice plain_slice(encoded.data(), encoded.size()); + std::unique_ptr plain_decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, + plain_decoder) + .ok()); + ASSERT_TRUE(plain_decoder->set_data(&plain_slice).ok()); + ParquetMaterializationState plain_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*plain_column, *plain_decoder, context, 1, + plain_state) + .ok()); + ASSERT_EQ(plain_column->size(), 1); + EXPECT_EQ(plain_column->get_data_at(0).to_string_view(), wkb); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({wkb, "unused"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 2).ok()); + char dictionary_index[] = {1, 2, 0}; + Slice dictionary_slice(dictionary_index, sizeof(dictionary_index)); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + context.encoding = ParquetValueEncoding::DICTIONARY; + auto dictionary_column = type.create_column(); + ParquetMaterializationState dictionary_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, dictionary_decoder, + context, 1, dictionary_state) + .ok()); + ASSERT_EQ(dictionary_column->size(), 1); + EXPECT_EQ(dictionary_column->get_data_at(0).to_string_view(), wkb); + + field.physical_type = tparquet::Type::INT32; + EXPECT_TRUE( + init_decode_context_for_test(field, nullptr, &context).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, InvalidLogicalPhysicalPairsFailBeforeDecode) { + auto invalid = [](tparquet::Type::type physical, const tparquet::LogicalType& logical) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + + tparquet::LogicalType date; + date.__set_DATE(tparquet::DateType()); + EXPECT_FALSE(invalid(tparquet::Type::BOOLEAN, date).ok()); + + tparquet::LogicalType timestamp; + timestamp.__set_TIMESTAMP(tparquet::TimestampType()); + timestamp.TIMESTAMP.__set_unit(tparquet::TimeUnit()); + timestamp.TIMESTAMP.unit.__set_MICROS(tparquet::MicroSeconds()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, timestamp).ok()); + + tparquet::LogicalType integer; + integer.__set_INTEGER(tparquet::IntType()); + integer.INTEGER.__set_bitWidth(64); + integer.INTEGER.__set_isSigned(true); + EXPECT_FALSE(invalid(tparquet::Type::INT32, integer).ok()); + + tparquet::LogicalType string; + string.__set_STRING(tparquet::StringType()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, string).ok()); + + tparquet::LogicalType uuid; + uuid.__set_UUID(tparquet::UUIDType()); + EXPECT_FALSE(invalid(tparquet::Type::BYTE_ARRAY, uuid).ok()); + + auto invalid_converted = [](tparquet::Type::type physical, + tparquet::ConvertedType::type converted, int type_length = -1, + int precision = -1, int scale = -1) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_converted_type(converted); + if (type_length >= 0) { + field.parquet_schema.__set_type_length(type_length); + } + if (precision >= 0) { + field.parquet_schema.__set_precision(precision); + } + if (scale >= 0) { + field.parquet_schema.__set_scale(scale); + } + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UTF8).ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::TIME_MICROS).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UINT_64).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::ConvertedType::INTERVAL, 8) + .ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::DECIMAL, -1, 10, 2) + .ok()); + + NativeFieldSchema fixed; + fixed.name = "fixed"; + fixed.physical_type = tparquet::Type::FIXED_LEN_BYTE_ARRAY; + ParquetDecodeContext context; + EXPECT_FALSE(init_decode_context_for_test(fixed, nullptr, &context).ok()); + fixed.parquet_schema.__set_type_length(4); + EXPECT_TRUE(init_decode_context_for_test(fixed, nullptr, &context).ok()); +} + +TEST(ParquetV2NativeDecoderTest, NonStrictLegacyTimestampsKeepDefaultOnOverflow) { + DataTypePtr datetime_type = make_nullable(std::make_shared(6)); + NativeFieldSchema int96_field; + int96_field.physical_type = tparquet::Type::INT96; + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, false)); + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, true)); + + NativeFieldSchema utc_field; + utc_field.physical_type = tparquet::Type::INT64; + utc_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + utc_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + utc_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(utc_field, datetime_type, false)); + + NativeFieldSchema converted_utc_field; + converted_utc_field.physical_type = tparquet::Type::INT64; + converted_utc_field.parquet_schema.__set_converted_type( + tparquet::ConvertedType::TIMESTAMP_MICROS); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(converted_utc_field, datetime_type, + false)); + + NativeFieldSchema local_field = utc_field; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + EXPECT_FALSE( + preserves_timestamp_conversion_default_for_test(local_field, datetime_type, false)); + + NativeFieldSchema integer_field; + integer_field.physical_type = tparquet::Type::INT64; + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test( + integer_field, make_nullable(std::make_shared()), false)); +} + +TEST(ParquetV2NativeDecoderTest, FastInt96NormalizesLegacyOutOfDayNanos) { + EXPECT_TRUE(materialize_plain_int96({{-1, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{86400000000000LL, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{0, 2440588}, {-1, 2440588}}, {0, 1}).ok()); +} + +TEST(ParquetV2NativeDecoderTest, NonStrictLocalTimestampDefaultsBecomeNull) { + DataTypePtr datetime_type = make_nullable(std::make_shared(6)); + NativeFieldSchema local_field; + local_field.physical_type = tparquet::Type::INT64; + local_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + local_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + const uint64_t invalid_datetime = 0; + column->insert_data(reinterpret_cast(&invalid_datetime), sizeof(invalid_datetime)); + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 1); + + null_map[0] = 0; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 0); +} + +TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCursor) { + const ParquetSelection selection { + .total_values = 7, + .selected_values = 3, + .ranges = {{.first = 1, .count = 2}, {.first = 6, .count = 1}}}; + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {10, 11, 12, 13, 14, 15, 16, 17}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({11, 12, 16})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({17})); + + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded_strings = encode_plain_byte_arrays(strings); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + Slice string_slice(encoded_strings.data(), encoded_strings.size()); + ASSERT_TRUE(decoder->set_data(&string_slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 3); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "one"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "two"); + EXPECT_EQ(selected_strings.refs[2].to_string_view(), "six"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "seven"); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char booleans[] = {static_cast(0b10001101)}; + Slice boolean_slice(booleans, sizeof(booleans)); + ASSERT_TRUE(decoder->set_data(&boolean_slice).ok()); + CaptureFixedConsumer selected_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_booleans).ok()); + EXPECT_EQ(selected_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_boolean).ok()); + EXPECT_EQ(trailing_boolean.values(), std::vector({1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_booleans[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_booleans, sizeof(rle_booleans)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + CaptureFixedConsumer selected_rle_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_rle_booleans).ok()); + EXPECT_EQ(selected_rle_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_rle_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_rle_boolean).ok()); + EXPECT_EQ(trailing_rle_boolean.values(), std::vector({1})); +} + +TEST(ParquetV2NativeDecoderTest, PlainByteArrayPublishesOffsetsAndCoalescedSelectionSpans) { + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded = encode_plain_byte_arrays(strings); + Slice slice(encoded.data(), encoded.size()); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + + const ParquetSelection selection { + .total_values = 7, + .selected_values = 4, + .ranges = {{.first = 1, .count = 2}, {.first = 5, .count = 2}}}; + CapturePlainBinaryLayoutConsumer consumer; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, consumer).ok()); + EXPECT_FALSE(consumer.legacy_consume_called); + EXPECT_EQ(consumer.output_offsets, std::vector({0, 3, 6, 10, 13})); + ASSERT_EQ(consumer.source_offsets.size(), selection.selected_values); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[0], 3), "one"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[1], 3), "two"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[2], 4), "five"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[3], 3), "six"); + ASSERT_EQ(consumer.spans.size(), 2); + EXPECT_EQ(consumer.spans[0].first, 0); + EXPECT_EQ(consumer.spans[0].count, 2); + EXPECT_EQ(consumer.spans[1].first, 2); + EXPECT_EQ(consumer.spans[1].count, 2); + + CaptureBinaryConsumer trailing; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing).ok()); + ASSERT_EQ(trailing.refs.size(), 1); + EXPECT_EQ(trailing.refs[0].to_string_view(), "seven"); +} + +TEST(ParquetV2NativeDecoderTest, SparsePlainFixedDecodeDoesNotRetainGatherBuffer) { + constexpr size_t value_count = 1UL << 18; + std::vector integers(value_count); + std::iota(integers.begin(), integers.end(), 0); + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + const ParquetSelection selection { + .total_values = value_count, + .selected_values = 2, + .ranges = {{.first = 1, .count = 1}, {.first = value_count - 1, .count = 1}}}; + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), + std::vector({1, static_cast(value_count - 1)})); + // Sparse PLAIN spans can be consumed directly from the encoded page. Retaining a gather buffer + // makes a highly selective batch allocate in proportion to its selected width for no benefit. + EXPECT_EQ(decoder->retained_scratch_bytes(), 0); +} + +TEST(ParquetV2NativeDecoderTest, FixedPlainLargeSkipCannotWrapPageOffset) { + std::vector values {11, 22}; + Slice slice(reinterpret_cast(values.data()), values.size() * sizeof(int64_t)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int64_t)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(536870913).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({11})); +} + +TEST(ParquetV2NativeDecoderTest, PlainAndBooleanRleExposeRawValuesAndPreserveCursor) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {11, 22, 33}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer integer_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, integer_consumer).ok()); + EXPECT_EQ(integer_consumer.values(), std::vector({22, 33})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0b10001101)}; + Slice plain_boolean_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_boolean_slice).ok()); + CaptureFixedConsumer plain_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(8, plain_boolean_consumer).ok()); + EXPECT_EQ(plain_boolean_consumer.values(), + std::vector({1, 0, 1, 1, 0, 0, 0, 1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_boolean[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_boolean, sizeof(rle_boolean)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + ASSERT_TRUE(decoder->skip_values(3).ok()); + CaptureFixedConsumer rle_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(5, rle_boolean_consumer).ok()); + EXPECT_EQ(rle_boolean_consumer.values(), std::vector({1, 0, 0, 0, 1})); +} + +TEST(ParquetV2NativeDecoderTest, DeltaEncodingsExposeValuesAfterSkip) { + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer first_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, first_integer).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer remaining_integers; + ASSERT_TRUE(decoder->decode_fixed_values(2, remaining_integers).ok()); + EXPECT_EQ(first_integer.values(), std::vector({100})); + EXPECT_EQ(remaining_integers.values(), std::vector({99, 1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(3, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 3); + EXPECT_EQ(consumer.refs[0].to_string_view(), "prefix-b"); + EXPECT_EQ(consumer.refs[1].to_string_view(), "other"); + EXPECT_EQ(consumer.refs[2].to_string_view(), "other-tail"); + } +} + +TEST(ParquetV2NativeDecoderTest, SparseStatefulEncodingsBatchDecodeAndCompact) { + const ParquetSelection selection { + .total_values = 3, + .selected_values = 2, + .ranges = {{.first = 0, .count = 1}, {.first = 2, .count = 1}}}; + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({100, 99})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 2); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "prefix-a"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "other"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "other-tail"); + } + + const std::vector floats {1.0F, -2.5F, 3.25F, 9.5F}; + std::vector encoded_floats(floats.size() * sizeof(float)); + for (size_t row = 0; row < floats.size(); ++row) { + const auto* bytes = reinterpret_cast(&floats[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded_floats[byte * floats.size() + row] = bytes[byte]; + } + } + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice float_slice(encoded_floats.data(), encoded_floats.size()); + ASSERT_TRUE(decoder->set_data(&float_slice).ok()); + CaptureFixedConsumer selected_floats; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_floats).ok()); + EXPECT_EQ(selected_floats.values(), std::vector({1.0F, 3.25F})); + CaptureFixedConsumer trailing_float; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_float).ok()); + EXPECT_EQ(trailing_float.values(), std::vector({9.5F})); +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRestoresFixedWidthRows) { + const std::vector values {1.0F, -2.5F, 3.25F}; + std::vector encoded(values.size() * sizeof(float)); + for (size_t row = 0; row < values.size(); ++row) { + const auto* bytes = reinterpret_cast(&values[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded[byte * values.size() + row] = bytes[byte]; + } + } + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({-2.5F, 3.25F})); +} + +TEST(ParquetV2NativeDecoderTest, BitPackedLevelCursorOperationsPreservePosition) { + char encoded[] = {static_cast(0b00001101)}; + Slice levels(encoded, sizeof(encoded)); + LevelDecoder decoder; + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 1, 4).ok()); + + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 4), 1); + EXPECT_EQ(value, 1); + EXPECT_EQ(decoder.get_next(), 0); + decoder.rewind_one(); + EXPECT_EQ(decoder.get_next(), 0); + level_t tail[2] = {-1, -1}; + EXPECT_EQ(decoder.get_levels(tail, 2), 2); + EXPECT_EQ(tail[0], 1); + EXPECT_EQ(tail[1], 1); + + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice truncated_levels(truncated_rle, sizeof(truncated_rle)); + LevelDecoder rle_decoder; + ASSERT_TRUE(rle_decoder.init(&truncated_levels, tparquet::Encoding::RLE, 1, 1).ok()); + level_t truncated_value = -1; + EXPECT_EQ(rle_decoder.get_levels(&truncated_value, 1), 0); +} + +TEST(ParquetV2NativeDecoderTest, BitPackedLevelByteCountDoesNotWrapAtLargeCounts) { + char placeholder[8] = {}; + constexpr uint32_t num_levels = 1'500'000'000; + constexpr size_t expected_bytes = 562'500'000; + Slice levels(placeholder, expected_bytes); + LevelDecoder decoder; + + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 4, num_levels).ok()); + EXPECT_EQ(levels.size, 0); +} + +TEST(ParquetV2NativeDecoderTest, RejectsLevelsAboveSchemaMaximumOnEveryDecodePath) { + auto make_decoder = [](tparquet::Encoding::type encoding) { + static char encoded[] = {0x03}; // width=2 value=3, while the schema maximum is 2. + static char rle_encoded[] = {0x02, 0x00, 0x00, 0x00, 0x02, 0x03}; + Slice levels = encoding == tparquet::Encoding::BIT_PACKED + ? Slice(encoded, sizeof(encoded)) + : Slice(rle_encoded, sizeof(rle_encoded)); + LevelDecoder decoder; + EXPECT_TRUE(decoder.init(&levels, encoding, 2, 1).ok()); + return decoder; + }; + + for (auto encoding : {tparquet::Encoding::BIT_PACKED, tparquet::Encoding::RLE}) { + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_levels(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + EXPECT_EQ(decoder.get_next(), -1); + } + } +} + +TEST(ParquetV2NativeDecoderTest, NestedReadersRejectRleAndBitPackedLevelsAboveMaximum) { + for (auto encoding : {tparquet::Encoding::RLE, tparquet::Encoding::BIT_PACKED}) { + const std::vector payload = encoding == tparquet::Encoding::RLE + ? std::vector {2, 0, 0, 0, 2, 3} + : std::vector {3}; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + + EXPECT_TRUE(load_malformed_nested_page(header, payload, 1, 2).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, TruncatedBooleanStreamsFailWhileSkipping) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0xFF)}; + Slice plain_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_slice).ok()); + EXPECT_FALSE(decoder->skip_values(9).ok()); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + // A bit-packed run header for eight values without its required payload byte. + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice rle_slice(truncated_rle, sizeof(truncated_rle)); + ASSERT_TRUE(decoder->set_data(&rle_slice).ok()); + EXPECT_FALSE(decoder->skip_values(1).ok()); + + // The first literal group is complete but the second has only its header. Exact-count checks + // must reject both dense and selected decodes instead of exposing the retained vector tail. + char partially_truncated_rle[] = {0x03, 0x00, 0x00, 0x00, 0x03, static_cast(0xFF), 0x03}; + Slice partially_truncated_slice(partially_truncated_rle, sizeof(partially_truncated_rle)); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer dense_consumer; + EXPECT_FALSE(decoder->decode_fixed_values(9, dense_consumer).ok()); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_tail { + .total_values = 9, .selected_values = 1, .ranges = {{.first = 8, .count = 1}}}; + EXPECT_FALSE(decoder->decode_selected_fixed_values(select_tail, selected_consumer).ok()); +} + +TEST(ParquetV2NativeDecoderTest, CompactRleSkipsKeepScratchBounded) { + constexpr uint32_t value_count = 1U << 20; + uint8_t header[16]; + uint8_t* header_end = encode_varint32(header, value_count << 1); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"only"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 1).ok()); + std::vector dictionary_indices {0}; + dictionary_indices.insert(dictionary_indices.end(), reinterpret_cast(header), + reinterpret_cast(header_end)); + Slice dictionary_slice(dictionary_indices.data(), dictionary_indices.size()); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + ParquetDecodeSource& dictionary_source = dictionary_decoder; + ASSERT_TRUE(dictionary_source.skip_values(value_count).ok()); + EXPECT_LE(dictionary_decoder.retained_scratch_bytes(), 4096 * sizeof(uint32_t)); + + std::vector boolean_payload(reinterpret_cast(header), + reinterpret_cast(header_end)); + boolean_payload.push_back(0); + std::vector boolean_page(sizeof(uint32_t) + boolean_payload.size()); + encode_fixed32_le(reinterpret_cast(boolean_page.data()), boolean_payload.size()); + memcpy(boolean_page.data() + sizeof(uint32_t), boolean_payload.data(), boolean_payload.size()); + Slice boolean_slice(boolean_page.data(), boolean_page.size()); + std::unique_ptr boolean_decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, boolean_decoder) + .ok()); + ASSERT_TRUE(boolean_decoder->set_data(&boolean_slice).ok()); + ASSERT_TRUE(boolean_decoder->skip_values(value_count).ok()); + EXPECT_LE(boolean_decoder->retained_scratch_bytes(), 4096); +} + +TEST(ParquetV2NativeDecoderTest, CompactDeltaSkipKeepsScratchBounded) { + constexpr uint32_t delta_count = 1U << 20; + std::vector encoded(64); + uint8_t* cursor = encoded.data(); + cursor = encode_varint32(cursor, delta_count); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, delta_count + 1); + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta + *cursor++ = 0; // zero-width miniblock + encoded.resize(cursor - encoded.data()); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(delta_count + 1); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(delta_count + 1).ok()); + EXPECT_LE(decoder->retained_scratch_bytes(), 4096 * sizeof(int32_t) + 1); +} + +TEST(ParquetV2NativeDecoderTest, DeltaPaddingBitCountIsWidenedBeforeCursorAdvance) { + int64_t padding_bits = 0; + ASSERT_TRUE(detail::checked_delta_padding_bits(64, std::numeric_limits::max(), + &padding_bits) + .ok()); + EXPECT_EQ(padding_bits, 64LL * std::numeric_limits::max()); + EXPECT_GT(padding_bits, std::numeric_limits::max()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArraySkipsKeepScratchBounded) { + constexpr size_t value_count = 1U << 16; + std::vector<::parquet::ByteArray> values(value_count); + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + static_cast(encoding), decoder) + .ok()); + decoder->set_expected_values(value_count); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(value_count).ok()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); + } +} + +TEST(ParquetV2NativeDecoderTest, DeltaFixedWidthValidatesFilteredAndSkippedValues) { + const std::vector values {"good", "bad"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_type_length(4); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_good { + .total_values = 2, .selected_values = 1, .ranges = {{.first = 0, .count = 1}}}; + EXPECT_TRUE(decoder->decode_selected_fixed_values(select_good, selected_consumer) + .is()); + + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_TRUE(decoder->skip_values(2).is()); +} + +TEST(ParquetV2NativeDecoderTest, TruncatedFixedWidthPlainKeepsPublicErrorKeyword) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(4); + char truncated[] = {'a', 'b', 'c'}; + Slice slice(truncated, sizeof(truncated)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + const auto status = decoder->decode_fixed_values(1, consumer); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Unexpected end of stream"), std::string::npos); +} + +TEST(ParquetV2NativeDecoderTest, DeltaSkipRequiresTheRequestedValueCount) { + const std::vector integers {7}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(integers.data(), static_cast(integers.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(2).ok()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaHeadersAndLengthsAreBoundedBeforeAllocation) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(1); + // block_size=128, miniblocks=4, total_values=INT_MAX, first_value=0. + char oversized_count[] = {static_cast(0x80), + 0x01, + 0x04, + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + 0x07, + 0x00}; + Slice oversized_count_slice(oversized_count, sizeof(oversized_count)); + EXPECT_TRUE(decoder->set_data(&oversized_count_slice).is()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + // A single decoded length of 1 GiB with no following payload bytes must fail before the + // decoder resizes its backing data buffer. + char oversized_length[] = {static_cast(0x80), + 0x01, + 0x04, + 0x01, + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + 0x08}; + Slice oversized_length_slice(oversized_length, sizeof(oversized_length)); + CaptureBinaryConsumer consumer; + EXPECT_TRUE(decoder->set_data(&oversized_length_slice).is()); + EXPECT_TRUE(consumer.refs.empty()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + std::vector huge_prefix(64); + uint8_t* cursor = huge_prefix.data(); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, std::numeric_limits::max() - 1); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, 0); // one empty suffix + huge_prefix.resize(cursor - huge_prefix.data()); + Slice huge_prefix_slice(huge_prefix.data(), huge_prefix.size()); + ASSERT_TRUE(decoder->set_data(&huge_prefix_slice).ok()); + EXPECT_TRUE(decoder->decode_binary_values(1, consumer).is()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); +} + +TEST(ParquetV2NativeDecoderTest, EmptyDeltaLengthPageResetsDecoderState) { + const std::vector strings {"old", "state"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + Slice valid(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&valid).ok()); + Slice empty; + EXPECT_TRUE(decoder->set_data(&empty).is()); +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRejectsPartialRowsAtPageBoundary) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + char partial_row[] = {0, 1, 2, 3, 4}; + Slice slice(partial_row, sizeof(partial_row)); + EXPECT_TRUE(decoder->set_data(&slice).is()); +} + +Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t value_rows, + size_t key_values, size_t value_values, + std::vector key_rep_levels, std::vector value_rep_levels, + bool value_eof, std::vector key_def_levels = {}, + std::vector value_def_levels = {}) { + auto value_type = make_nullable(std::make_shared()); + auto map_type = std::make_shared(key_type, value_type); + ColumnPtr column = map_type->create_column(); + NativeFieldSchema field; + field.name = "m"; + field.data_type = map_type; + field.definition_level = 1; + field.repetition_level = 1; + field.repeated_parent_def_level = 0; + + if (key_def_levels.empty()) { + key_def_levels.assign(key_rep_levels.size(), 1); + } + if (value_def_levels.empty()) { + value_def_levels.assign(value_rep_levels.size(), 1); + } + auto key_reader = std::make_unique( + key_rows, key_values, true, std::move(key_rep_levels), std::move(key_def_levels)); + auto value_reader = std::make_unique(value_rows, value_values, value_eof, + std::move(value_rep_levels), + std::move(value_def_levels)); + MapColumnReader reader(scripted_row_ranges(), key_rows, nullptr, nullptr); + RETURN_IF_ERROR(reader.init(std::move(key_reader), std::move(value_reader), &field)); + + auto root = std::make_shared(std::make_shared(), + std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + return reader.read_column_data(column, map_type, root, filter, key_rows, &read_rows, &eof, + false); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderUsesKeyShapeForNestedValues) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {0, 1}, {0, 2, 1}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryDistribution) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 4, 4, {0, 1, 0, 1}, {0, 1, 1, 0}, true) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryPresence) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 2, 2, {0, 0}, {0, 0}, true, {0, 1}, {1, 0}) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, ComplexReadersRejectMalformedSiblingCounts) { + auto int_type = std::make_shared(); + EXPECT_TRUE( + read_scripted_map(int_type, 2, 1, 2, 1, {0, 0}, {0}, true).is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderPreservesNullableKeysWrittenByDoris) { + auto nullable_key = make_nullable(std::make_shared()); + EXPECT_TRUE(read_scripted_map(nullable_key, 1, 1, 1, 1, {0}, {0}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShortSibling) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 0}); + children["b"] = std::make_unique(1, 1, true, std::vector {0}, + std::vector {0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedRepeatedParentShape) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.repetition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique( + 2, 3, true, std::vector {0, 1, 0}, std::vector {0, 0, 0}); + children["b"] = std::make_unique( + 2, 3, true, std::vector {0, 0, 1}, std::vector {0, 0, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedOptionalParentPresence) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.definition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 1}); + children["b"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {1, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { + constexpr size_t value_count = 1UL << 20; + std::vector encoded(value_count * sizeof(float), 0); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(value_count, consumer).ok()); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + EXPECT_EQ(decoder->active_scratch_bytes(), value_count * sizeof(float)); + + std::vector ordinary_encoded(sizeof(float), 0); + Slice ordinary_slice(ordinary_encoded.data(), ordinary_encoded.size()); + ASSERT_TRUE(decoder->set_data(&ordinary_slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + // Capacity records the high-water allocation while size records this batch. The reader needs + // both to distinguish stable large batches from a one-off outlier before releasing capacity. + EXPECT_EQ(decoder->active_scratch_bytes(), sizeof(float)); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + decoder->release_scratch(64UL << 10); + EXPECT_LE(decoder->retained_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArrayScratchReleasePreservesPrefixState) { + const std::vector values {std::string(4096, 'x'), "shared-prefix-a", + "shared-prefix-b", "shared-prefix-c"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + decoder->release_scratch(64); + ASSERT_TRUE(decoder->decode_binary_values(2, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 2); + EXPECT_EQ(consumer.refs[0].to_string_view(), values[2]); + EXPECT_EQ(consumer.refs[1].to_string_view(), values[3]); +} + +TEST(ParquetV2NativeDecoderTest, DeltaBitPackScratchReleasePreservesMiniblockState) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + + std::vector outlier_header(64); + uint8_t* cursor = outlier_header.data(); + cursor = encode_varint32(cursor, 512); // values per block + cursor = encode_varint32(cursor, 16); // miniblocks per block + cursor = encode_varint32(cursor, 2); // total values + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta, zig-zag encoded + memset(cursor, 0, 16); // one zero bit width per miniblock + cursor += 16; + outlier_header.resize(cursor - outlier_header.data()); + decoder->set_expected_values(2); + Slice outlier_slice(outlier_header.data(), outlier_header.size()); + ASSERT_TRUE(decoder->set_data(&outlier_slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + + std::vector values(40); + std::iota(values.begin(), values.end(), 0); + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + decoder->release_scratch(8); + ASSERT_TRUE(decoder->decode_fixed_values(values.size() - 1, consumer).ok()); + EXPECT_EQ(consumer.values().size(), values.size() + 1); +} + +TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption) { + auto parse_header = [](tparquet::PageHeader header) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + if (header.compressed_page_size > 0) { + bytes.resize(bytes.size() + header.compressed_page_size); + } + MemoryBufferedReader reader(bytes); + tparquet::ColumnMetaData metadata; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, bytes.size(), 1, metadata, + context); + return page_reader.parse_page_header(); + }; + + tparquet::PageHeader negative; + negative.type = tparquet::PageType::DATA_PAGE; + negative.__set_compressed_page_size(-1); + negative.__set_uncompressed_page_size(1); + negative.__isset.data_page_header = true; + negative.data_page_header.__set_num_values(1); + EXPECT_TRUE(parse_header(negative).is()); + + tparquet::PageHeader oversized_levels; + oversized_levels.type = tparquet::PageType::DATA_PAGE_V2; + oversized_levels.__set_compressed_page_size(1); + oversized_levels.__set_uncompressed_page_size(1); + oversized_levels.__isset.data_page_header_v2 = true; + oversized_levels.data_page_header_v2.__set_num_values(1); + oversized_levels.data_page_header_v2.__set_num_rows(1); + oversized_levels.data_page_header_v2.__set_repetition_levels_byte_length(2); + oversized_levels.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(oversized_levels).is()); + + tparquet::PageHeader impossible_counts = oversized_levels; + impossible_counts.__set_compressed_page_size(0); + impossible_counts.__set_uncompressed_page_size(0); + impossible_counts.data_page_header_v2.__set_repetition_levels_byte_length(0); + impossible_counts.data_page_header_v2.__set_num_nulls(2); + EXPECT_TRUE(parse_header(impossible_counts).is()); + + tparquet::PageHeader missing_layout; + missing_layout.type = tparquet::PageType::DATA_PAGE; + missing_layout.__set_compressed_page_size(0); + missing_layout.__set_uncompressed_page_size(0); + EXPECT_TRUE(parse_header(missing_layout).is()); + + auto swapped_layout = oversized_levels; + swapped_layout.type = tparquet::PageType::DATA_PAGE; + EXPECT_TRUE(parse_header(swapped_layout).is()); + + auto competing_layouts = negative; + competing_layouts.__set_compressed_page_size(0); + competing_layouts.__set_uncompressed_page_size(0); + competing_layouts.__isset.data_page_header_v2 = true; + competing_layouts.data_page_header_v2.__set_num_values(0); + competing_layouts.data_page_header_v2.__set_num_rows(0); + competing_layouts.data_page_header_v2.__set_num_nulls(0); + competing_layouts.data_page_header_v2.__set_repetition_levels_byte_length(0); + competing_layouts.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(competing_layouts).is()); +} + +TEST(ParquetV2NativeDecoderTest, FlatPagesRejectLogicalAndPhysicalCardinalityMismatch) { + auto init_chunk = [](tparquet::PageHeader header, bool with_offset_index) { + std::vector payload(static_cast(header.compressed_page_size), 0); + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(bytes.size()); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + if (with_offset_index) { + ColumnChunkReader reader_with_index(&reader, &chunk, &field, &offset_index, + 1, nullptr, context); + return reader_with_index.init(); + } + ColumnChunkReader sequential_reader(&reader, &chunk, &field, nullptr, 1, + nullptr, context); + return sequential_reader.init(); + }; + + tparquet::PageHeader v2; + v2.type = tparquet::PageType::DATA_PAGE_V2; + v2.__set_compressed_page_size(8); + v2.__set_uncompressed_page_size(8); + v2.__isset.data_page_header_v2 = true; + v2.data_page_header_v2.__set_num_values(2); + v2.data_page_header_v2.__set_num_rows(1); + v2.data_page_header_v2.__set_num_nulls(0); + v2.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + v2.data_page_header_v2.__set_repetition_levels_byte_length(0); + v2.data_page_header_v2.__set_definition_levels_byte_length(0); + v2.data_page_header_v2.__set_is_compressed(false); + auto status = init_chunk(v2, false); + EXPECT_TRUE(status.is()) << status; + status = init_chunk(v2, true); + EXPECT_TRUE(status.is()) << status; + + tparquet::PageHeader v1; + v1.type = tparquet::PageType::DATA_PAGE; + v1.__set_compressed_page_size(8); + v1.__set_uncompressed_page_size(8); + v1.__isset.data_page_header = true; + v1.data_page_header.__set_num_values(2); + v1.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + v1.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + v1.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + status = init_chunk(v1, true); + EXPECT_TRUE(status.is()) << status; +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyAllNullPagesInitializeZeroValueDecoders) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, true); + EXPECT_TRUE(status.ok()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyPagesRejectDefinitionLevelsThatRequireValues) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, false); + EXPECT_TRUE(status.is()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + +TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsMissingAdvertisedRowStarts) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(12); + header.__set_uncompressed_page_size(12); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(2); + header.data_page_header_v2.__set_num_rows(2); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(4); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + // Two values [0, 1] contain only one repetition-level zero, so they describe one row. + EXPECT_TRUE(load_malformed_nested_page(header, {2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0}, 2) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, FirstNestedV1PageRejectsOrphanContinuation) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + // A two-value RLE run at repetition level 1 has no level-0 row start. + const std::vector payload {2, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0}; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(2); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + EXPECT_TRUE(load_malformed_nested_page(header, payload, 2).is()); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1ContinuationRemainsValidAfterFirstRowStart) { + auto make_page = [](int values, std::vector payload) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, payload); + }; + auto first_page = make_page(2, {2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0}); + auto second_page = make_page(1, {2, 0, 0, 0, 2, 1, 0, 0, 0, 0}); + first_page.insert(first_page.end(), second_page.begin(), second_page.end()); + + MemoryBufferedReader reader(first_page); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(3); + chunk.meta_data.__set_total_compressed_size(first_page.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + ASSERT_TRUE(cross_page); + ASSERT_TRUE(chunk_reader.load_cross_page_nested_row(levels, &cross_page).ok()); + EXPECT_FALSE(cross_page); + EXPECT_EQ(levels, std::vector({0, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, HugeNestedPageCountsDoNotPreallocateFromHeaders) { + for (auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(std::numeric_limits::max()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(std::numeric_limits::max()); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } + // The four-byte V1 level length (or V2 value payload) contains no advertised levels. The + // reader must report corruption without reserving INT_MAX level slots first. + EXPECT_TRUE(load_malformed_nested_page(header, {0, 0, 0, 0}).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, PageDecompressionRejectsBothSizeMismatchDirections) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + std::vector input(8, 7); + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(input.data(), input.size()), &compressed).ok()); + std::vector payload(compressed.data(), compressed.data() + compressed.size()); + + for (const int32_t advertised_size : {4, 12}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(advertised_size); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + EXPECT_FALSE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY).ok()); + } +} + +TEST(ParquetV2NativeDecoderTest, UncompressedDictionaryRequiresEqualPhysicalAndLogicalSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(1); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE(load_scripted_page(header, std::vector(8), + tparquet::CompressionCodec::UNCOMPRESSED) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, UncompressedDataPagesRequireEqualPhysicalAndLogicalSizes) { + for (const auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + } + const std::vector payload(8); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED) + .is()); + EXPECT_TRUE( + load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED, true) + .is()); + } +} + +TEST(ParquetV2NativeDecoderTest, LegacyV2CompressedOverrideAllowsDifferentSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(16); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, true) + .ok()); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, EmptyOffsetIndexCannotSelectIndexedPageReader) { + MemoryBufferedReader reader(std::vector {0}); + tparquet::ColumnMetaData metadata; + tparquet::OffsetIndex empty_index; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, 1, 1, metadata, context, &empty_index); + EXPECT_FALSE(page_reader.has_next_page()); + EXPECT_TRUE(page_reader.next_page().is()); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkRangeRejectsSignedOverflowAndBoundsLegacyPadding) { + tparquet::ColumnMetaData metadata; + metadata.__set_data_page_offset(-1); + metadata.__set_total_compressed_size(10); + ColumnChunkRange range; + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(95); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(10); + metadata.__set_total_compressed_size(std::numeric_limits::max()); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_total_compressed_size(20); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, true, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 25); + + metadata.__set_dictionary_page_offset(0); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 20); + + metadata.__set_dictionary_page_offset(5); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 5); + EXPECT_EQ(range.length, 20); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsBackwardAndOverlappingLocations) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(110); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(120); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); + + second.__set_offset(140); + second.__set_first_row_index(0); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); + + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_TRUE(validate_offset_index(index, range, 110, 20)); + + range = {.offset = std::numeric_limits::max(), .length = 2}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsShiftedFirstDataPage) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(120); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(140); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + + EXPECT_FALSE(validate_offset_index(index, range, 100, 20)); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingDataDecoder) { + tparquet::PageHeader index_header; + index_header.type = tparquet::PageType::INDEX_PAGE; + index_header.__set_compressed_page_size(3); + index_header.__set_uncompressed_page_size(3); + index_header.__set_index_page_header(tparquet::IndexPageHeader()); + auto bytes = serialize_page(index_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsUnknownAuxiliaryPage) { + tparquet::PageHeader unknown_header; + unknown_header.type = static_cast(127); + unknown_header.__set_compressed_page_size(3); + unknown_header.__set_uncompressed_page_size(3); + auto bytes = serialize_page(unknown_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); +} + +TEST(ParquetV2NativeDecoderTest, LegacyDataPageV2OverridesFalseCompressedFlag) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(1024); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + EXPECT_TRUE(should_cache_decompressed(&header, metadata, false)); + EXPECT_FALSE(should_cache_decompressed(&header, metadata, true)); + + EXPECT_TRUE(parquet_reader_compat("parquet-cpp version 1.5.0").data_page_v2_always_compressed); + EXPECT_FALSE(parquet_reader_compat("parquet-cpp version 2.0.0").data_page_v2_always_compressed); + EXPECT_TRUE(parquet_reader_compat("parquet-mr version 1.2.8").parquet_816_padding); + EXPECT_FALSE(parquet_reader_compat("parquet-mr version 1.2.9").parquet_816_padding); +} + +TEST(ParquetV2NativeDecoderTest, NullableNumericOverflowIsNullOnlyOutsideStrictMode) { + const int32_t overflow = 1000; + auto decode = [&](bool strict, MutableColumnPtr* column, IColumn::Filter* null_map) { + std::unique_ptr decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder)); + decoder->set_type_length(sizeof(overflow)); + Slice slice(reinterpret_cast(&overflow), sizeof(overflow)); + RETURN_IF_ERROR(decoder->set_data(&slice)); + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + MutableColumnPtr column; + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ASSERT_TRUE(decode(false, &column, &null_map).ok()); + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + + null_map.clear(); + null_map.resize_fill(1, 0); + EXPECT_FALSE(decode(true, &column, &null_map).ok()); + EXPECT_EQ(null_map[0], 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryConversionFailureIsDeferredUntilReferenced) { + auto decode_dictionary_id = [](uint8_t dictionary_id, bool strict, IColumn::Filter* null_map, + MutableColumnPtr* column) { + const std::array dictionary_values {1, 1000}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + RETURN_IF_ERROR(Decoder::get_decoder(tparquet::Type::INT32, + tparquet::Encoding::RLE_DICTIONARY, decoder)); + decoder->set_type_length(sizeof(int32_t)); + RETURN_IF_ERROR( + decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size())); + char encoded_id[] = {1, 2, static_cast(dictionary_id)}; + Slice id_slice(encoded_id, sizeof(encoded_id)); + RETURN_IF_ERROR(decoder->set_data(&id_slice)); + + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + for (const bool strict : {false, true}) { + IColumn::Filter null_map; + IColumn::Filter* output_null_map = nullptr; + if (strict) { + null_map.resize_fill(1, 0); + output_null_map = &null_map; + } + + MutableColumnPtr column; + const auto unused_bad = decode_dictionary_id(0, strict, output_null_map, &column); + ASSERT_TRUE(unused_bad.ok()) << unused_bad; + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(assert_cast(*column).get_element(0), 1); + + const auto referenced_bad = decode_dictionary_id(1, strict, output_null_map, &column); + EXPECT_TRUE(referenced_bad.is()) << referenced_bad; + EXPECT_TRUE(column->empty()); + if (output_null_map != nullptr) { + EXPECT_EQ((*output_null_map)[0], 0); + } + } +} + +TEST(ParquetV2NativeDecoderTest, FixedLengthStringsAppendAsOneContiguousSpan) { + ColumnString column; + const std::string values = "aaabbbccc"; + column.insert_many_fixed_length_data(values.data(), 3, 3); + ASSERT_EQ(column.size(), 3); + EXPECT_EQ(column.get_data_at(0).to_string_view(), "aaa"); + EXPECT_EQ(column.get_data_at(1).to_string_view(), "bbb"); + EXPECT_EQ(column.get_data_at(2).to_string_view(), "ccc"); +} + +TEST(ParquetV2NativeDecoderTest, ComplexPageStatisticsPreservePerLeafCrossings) { + ColumnChunkReaderStatistics first_chunk; + first_chunk.page_read_counter = 1; + first_chunk.data_page_read_counter = 1; + ColumnChunkReaderStatistics second_chunk; + second_chunk.page_read_counter = 1; + second_chunk.data_page_read_counter = 1; + ColumnReader::ColumnStatistics combined; + ColumnReader::ColumnStatistics first(first_chunk, 0); + ColumnReader::ColumnStatistics second(second_chunk, 0); + combined.merge(first); + combined.merge(second); + + EXPECT_EQ(combined.page_read_counter, 2); + ASSERT_EQ(combined.leaf_page_read_counters.size(), 2); + EXPECT_EQ(combined.leaf_page_read_counters[0], 1); + EXPECT_EQ(combined.leaf_page_read_counters[1], 1); +} + +TEST(ParquetV2NativeDecoderTest, NativePageCacheUsesStableFileDescriptionIdentity) { + ParquetPageCacheKeyBuilder first; + ParquetPageCacheKeyBuilder replaced; + first.init("s3://bucket/object::etag-v1::1234"); + replaced.init("s3://bucket/object::etag-v2::1234"); + EXPECT_EQ(first.make_key(4096, 128).fname, "s3://bucket/object::etag-v1::1234"); + EXPECT_NE(first.make_key(4096, 128).encode(), replaced.make_key(4096, 128).encode()); +} + +TEST(ParquetV2NativeDecoderTest, UncompressedV2PageCachePayloadIsAlwaysDecompressed) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(100); + header.__set_uncompressed_page_size(1000); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + + // The representation is explicit in V2; a codec and compression ratio cannot override it on + // a warm cache hit. + EXPECT_TRUE(should_cache_decompressed(&header, metadata)); +} + +TEST(ParquetV2NativeDecoderTest, CacheDisabledPagesDoNotPrepareCopiedPayload) { + EXPECT_FALSE(can_prepare_page_cache_payload(false, false, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, true, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, false, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, true, false)); + EXPECT_TRUE(can_prepare_page_cache_payload(true, false, true, true)); +} + +TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchUsesIdleBatchHysteresis) { + ::doris::RowRanges row_ranges; + tparquet::ColumnChunk chunk; + ScalarColumnReader reader(row_ranges, 1, chunk, nullptr, nullptr, nullptr); + + constexpr size_t max_retained_bytes = 64UL << 10; + reader.reserve_batch_scratch_for_test(1UL << 16); + const size_t oversized_bytes = reader.retained_batch_scratch_bytes_for_test(); + ASSERT_GT(oversized_bytes, max_retained_bytes); + + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); + reader.release_batch_scratch(max_retained_bytes); + const size_t released_bytes = reader.retained_batch_scratch_bytes_for_test(); + EXPECT_LT(released_bytes, oversized_bytes); + EXPECT_LE(released_bytes, max_retained_bytes + sizeof(void*)); +} + +} // namespace +} // namespace doris::format::parquet::native diff --git a/be/test/format_v2/parquet/parquet_column_reader_test.cpp b/be/test/format_v2/parquet/parquet_column_reader_test.cpp deleted file mode 100644 index fb4cd129e1b03d..00000000000000 --- a/be/test/format_v2/parquet/parquet_column_reader_test.cpp +++ /dev/null @@ -1,3682 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_decimal.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_struct.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "core/types.h" -#include "format_v2/file_reader.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/selection_vector.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -template -const ColumnType& get_nullable_nested_column(const IColumn& column) { - // File-local schema exposed by the parquet reader follows Doris external-table semantics: - // nested STRUCT fields, LIST elements, and MAP keys/values are nullable even when the parquet - // field is required. - const auto& nullable_column = assert_cast(column); - return assert_cast(nullable_column.get_nested_column()); -} - -ParquetColumnSchema mock_column_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "mock"; - schema.type = std::make_shared(); - return schema; -} - -class BaseUnsupportedReader final : public ParquetColumnReader { -public: - BaseUnsupportedReader() - : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } -}; - -class DefaultSelectReader final : public ParquetColumnReader { -public: - DefaultSelectReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - auto& values = assert_cast(*column); - for (int64_t row = 0; row < rows; ++row) { - values.insert_value(static_cast(_cursor + row)); - } - _cursor += rows; - *rows_read = rows; - _read_ranges.push_back(rows); - return Status::OK(); - } - - Status skip(int64_t rows) override { - _cursor += rows; - _skip_ranges.push_back(rows); - return Status::OK(); - } - - const std::vector& read_ranges() const { return _read_ranges; } - const std::vector& skip_ranges() const { return _skip_ranges; } - -private: - int64_t _cursor = 0; - std::vector _read_ranges; - std::vector _skip_ranges; -}; - -class NestedSkipReader final : public ParquetColumnReader { -public: - NestedSkipReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - *values_consumed = length_upper_bound; - return Status::OK(); - } -}; - -class ParquetColumnReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_column_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "reader.parquet").string(); - _plain_file_path = (_test_dir / "plain_reader.parquet").string(); - write_parquet_file(); - _file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto metadata = _file_reader->metadata(); - ASSERT_EQ(metadata->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(_fields.size(), _expected_by_field.size()); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_all_null_int32_array() { - arrow::Int32Builder builder; - for (int64_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(builder.AppendNull().ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_required_struct_array() { - auto struct_type = arrow::struct_({arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - const std::vector a_values = {101, 102, 103, 104, 105}; - const std::vector b_values = {"sa", "sb", "sc", "sd", "se"}; - for (size_t row = 0; row < a_values.size(); ++row) { - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(a_values[row]).ok()); - EXPECT_TRUE(b_builder->Append(b_values[row]).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(201).ok()); - EXPECT_TRUE(b_builder->Append("nsa").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(203).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(204).ok()); - EXPECT_TRUE(b_builder->Append("nsd").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_decimal_array() { - auto decimal_type = arrow::decimal128(38, 6); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("d", decimal_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto d_array_builder = std::make_unique( - decimal_type, arrow::default_memory_pool()); - field_builders.push_back(std::shared_ptr(std::move(d_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* d_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(123456789)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(d_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(-987654321)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("xs", list_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - field_builders.push_back(list_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(305).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_map_array() { - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("kv", map_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - field_builders.push_back(map_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(401).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("one").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(403).ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(404).ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(405).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->Append("five").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_nested_struct_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto nested_type = arrow::struct_({arrow::field("xs", list_type, true)}); - auto struct_type = arrow::struct_({arrow::field("nested", nested_type, true)}); - - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - std::vector> nested_field_builders; - nested_field_builders.push_back(list_builder); - auto nested_builder = std::make_shared( - nested_type, arrow::default_memory_pool(), std::move(nested_field_builders)); - std::vector> field_builders; - field_builders.push_back(nested_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(7).ok()); - EXPECT_TRUE(value_builder->Append(8).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), false))); - const std::vector> values = { - {1, 2}, {3}, {4, 5, 6}, {7}, {8, 9}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto value : row) { - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append(20).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(30).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(110).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(120).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(130).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_list_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - auto* a_builder = assert_cast(struct_builder->field_builder(0)); - auto* b_builder = assert_cast(struct_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(11).ok()); - EXPECT_TRUE(b_builder->Append("la").ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(12).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(13).ok()); - EXPECT_TRUE(b_builder->Append("ld").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(14).ok()); - EXPECT_TRUE(b_builder->Append("le").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_list_list_int_array() { - auto value_builder = std::make_shared(); - auto inner_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto inner_list_builder = std::make_shared( - arrow::default_memory_pool(), value_builder, inner_list_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), inner_list_builder, - arrow::list(arrow::field("element", inner_list_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->AppendNull().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - const std::vector>> values = { - {{1, "a"}, {2, "b"}}, {{3, "c"}}, {{4, "d"}, {5, "e"}, {6, "f"}}, - {{7, "g"}}, {{8, "h"}, {9, "i"}}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto& [key, value] : row) { - EXPECT_TRUE(key_builder->Append(key).ok()); - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append("bb").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_struct_map_array() { - auto key_builder = std::make_shared(); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto value_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", struct_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - auto* a_builder = assert_cast(value_builder->field_builder(0)); - auto* b_builder = assert_cast(value_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(21).ok()); - EXPECT_TRUE(b_builder->Append("ma").ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(22).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(24).ok()); - EXPECT_TRUE(b_builder->Append("me").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(201).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(key_builder->Append(202).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(203).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(204).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(205).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_map_list_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), map_builder, - arrow::list(arrow::field("element", map_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("a").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(value_builder->Append("d").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_map_map_array() { - auto key_builder = std::make_shared(); - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, nested_map_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(101).ok()); - EXPECT_TRUE(nested_value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(401).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_list_struct_map_list_array() { - auto element_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - element_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - auto map_builder = std::make_shared(arrow::default_memory_pool(), - key_builder, list_builder, map_type); - auto struct_type = arrow::struct_({arrow::field("kv", map_type, true)}); - std::vector> struct_field_builders; - struct_field_builders.push_back(map_builder); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(struct_field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(10).ok()); - EXPECT_TRUE(element_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_map_list_map_array() { - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto list_type = arrow::list(arrow::field("element", nested_map_type, true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - nested_map_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(1).ok()); - EXPECT_TRUE(nested_value_builder->Append("a").ok()); - EXPECT_TRUE(nested_key_builder->Append(2).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(3).ok()); - EXPECT_TRUE(nested_value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(50).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(4).ok()); - EXPECT_TRUE(nested_value_builder->Append("d").ok()); - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, std::shared_ptr array, - std::function validator) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - _expected_by_field.push_back(std::move(validator)); - } - - void write_parquet_file() { - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); - add_field(arrow::field("all_null_int_col", arrow::int32(), true), - build_all_null_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - for (size_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(nullable_column.is_null_at(row)); - } - }); - add_field(arrow::field("struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false), - }), - false), - build_required_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_STRUCT); - const auto& struct_column = assert_cast(column); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - get_nullable_nested_column(struct_column.get_column(1)); - EXPECT_EQ(a_values.get_element(0), 101); - EXPECT_EQ(a_values.get_element(4), 105); - EXPECT_EQ(b_values.get_data_at(1).to_string(), "sb"); - EXPECT_EQ(b_values.get_data_at(4).to_string(), "se"); - }); - add_field(arrow::field("nullable_struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }), - true), - build_nullable_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_nested = - assert_cast(b_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 201); - EXPECT_EQ(a_values.get_element(2), 203); - EXPECT_EQ(a_values.get_element(3), 204); - EXPECT_FALSE(b_values.is_null_at(0)); - EXPECT_TRUE(b_values.is_null_at(2)); - EXPECT_FALSE(b_values.is_null_at(3)); - EXPECT_EQ(b_nested.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(b_nested.get_data_at(3).to_string(), "nsd"); - }); - add_field(arrow::field("nullable_struct_decimal_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("d", arrow::decimal128(38, 6), true), - }), - true), - build_nullable_struct_with_decimal_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& d_values = - assert_cast(struct_column.get_column(1)); - const auto& d_nested = - assert_cast(d_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_FALSE(d_values.is_null_at(0)); - EXPECT_TRUE(d_values.is_null_at(2)); - EXPECT_FALSE(d_values.is_null_at(3)); - EXPECT_EQ(d_nested.get_element(0), Decimal128V3(123456789)); - EXPECT_EQ(d_nested.get_element(3), Decimal128V3(-987654321)); - }); - auto struct_list_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("xs", arrow::list(arrow::field("element", arrow::int32(), true)), - true), - }); - add_field(arrow::field("nullable_struct_list_col", struct_list_type, true), - build_nullable_struct_with_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_EQ(a_values.get_element(4), 305); - - const auto& xs_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = - assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = - assert_cast(xs_array.get_data()); - ASSERT_EQ(elements.size(), 4); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - const auto& values = - assert_cast(elements.get_nested_column()); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_EQ(values.get_element(3), 5); - }); - auto struct_map_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("kv", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - }); - add_field(arrow::field("nullable_struct_map_col", struct_map_type, true), - build_nullable_struct_with_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(2), 403); - EXPECT_EQ(a_values.get_element(3), 404); - EXPECT_EQ(a_values.get_element(4), 405); - - const auto& kv_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = - assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); - }); - auto nested_struct_list_type = arrow::struct_({ - arrow::field("nested", - arrow::struct_({ - arrow::field("xs", - arrow::list(arrow::field("element", - arrow::int32(), true)), - true), - }), - true), - }); - add_field(arrow::field("nullable_struct_nested_struct_list_col", nested_struct_list_type, - true), - build_nullable_struct_with_nested_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = - assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); - }); - add_field(arrow::field("list_int_col", - arrow::list(arrow::field("element", arrow::int32(), false)), false), - build_required_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_ARRAY); - const auto* array_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ( - remove_nullable(array_type->get_nested_type())->get_primitive_type(), - TYPE_INT); - const auto& array_column = assert_cast(column); - ASSERT_EQ(array_column.size(), ROW_COUNT); - const auto array_size_at = [&array_column](size_t row_idx) { - return array_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : array_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(array_size_at(0), 2); - EXPECT_EQ(array_size_at(1), 1); - EXPECT_EQ(array_size_at(2), 3); - EXPECT_EQ(array_size_at(4), 2); - const auto& values = - get_nullable_nested_column(array_column.get_data()); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(5), 6); - EXPECT_EQ(values.get_element(8), 9); - }); - add_field(arrow::field("nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), true), - build_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 30); - EXPECT_EQ(values.get_element(4), 40); - }); - add_field(arrow::field("required_nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), false), - build_required_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& array_column = assert_cast(column); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 5); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - ASSERT_EQ(elements.size(), 5); - EXPECT_TRUE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(4)); - }); - auto list_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field("nullable_list_struct_col", - arrow::list(arrow::field("element", list_struct_type, true)), true), - build_nullable_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - - const auto& elements = - assert_cast(array_column.get_data()); - const auto& struct_column = - assert_cast(elements.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 11); - EXPECT_EQ(a_values.get_element(1), 12); - EXPECT_EQ(a_values.get_element(3), 13); - EXPECT_EQ(a_values.get_element(4), 14); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); - }); - auto nested_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field(arrow::field("nullable_list_list_int_col", - arrow::list(arrow::field("element", nested_list_type, true)), true), - build_nullable_list_list_int_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 4); - EXPECT_EQ(outer_offsets[3], 5); - EXPECT_EQ(outer_offsets[4], 7); - - const auto& inner_nullable = - assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_FALSE(inner_nullable.is_null_at(6)); - - const auto& inner_array = - assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[5], 5); - EXPECT_EQ(inner_offsets[6], 7); - - const auto& elements = - assert_cast(inner_array.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 7); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 3); - EXPECT_EQ(values.get_element(4), 4); - EXPECT_EQ(values.get_element(5), 5); - EXPECT_TRUE(elements.is_null_at(6)); - }); - add_field(arrow::field( - "map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)), - false), - build_required_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_MAP); - const auto* map_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), - TYPE_INT); - EXPECT_EQ(remove_nullable(map_type->get_value_type())->get_primitive_type(), - TYPE_STRING); - const auto& map_column = assert_cast(column); - ASSERT_EQ(map_column.size(), ROW_COUNT); - const auto map_size_at = [&map_column](size_t row_idx) { - return map_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : map_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(map_size_at(0), 2); - EXPECT_EQ(map_size_at(1), 1); - EXPECT_EQ(map_size_at(2), 3); - EXPECT_EQ(map_size_at(4), 2); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(keys.size(), 9); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(5), 6); - EXPECT_EQ(keys.get_element(8), 9); - EXPECT_EQ(values.get_data_at(0).to_string(), "a"); - EXPECT_EQ(values.get_data_at(5).to_string(), "f"); - EXPECT_EQ(values.get_data_at(8).to_string(), "i"); - }); - add_field( - arrow::field("nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - build_nullable_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 10); - EXPECT_EQ(keys.get_element(1), 20); - EXPECT_EQ(keys.get_element(3), 40); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(values.is_null_at(3)); - }); - add_field( - arrow::field("required_nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - false), - build_required_nullable_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& map_column = assert_cast(column); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 4); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(3)); - }); - auto map_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field( - "nullable_map_int_struct_col", - arrow::map(arrow::int32(), arrow::field("value", map_struct_type, true)), - true), - build_nullable_int_struct_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& struct_column = - assert_cast(values.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 21); - EXPECT_EQ(a_values.get_element(1), 22); - EXPECT_EQ(a_values.get_element(3), 24); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); - }); - auto map_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field( - arrow::field("nullable_map_int_list_col", - arrow::map(arrow::int32(), arrow::field("value", map_list_type, true)), - true), - build_nullable_int_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), ROW_COUNT); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 4); - EXPECT_EQ(map_offsets[4], 5); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 5); - EXPECT_EQ(keys.get_element(0), 201); - EXPECT_EQ(keys.get_element(1), 202); - EXPECT_EQ(keys.get_element(2), 203); - EXPECT_EQ(keys.get_element(3), 204); - EXPECT_EQ(keys.get_element(4), 205); - - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_FALSE(values.is_null_at(4)); - - const auto& list_column = - assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); - - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(element_values.get_element(0), 1); - EXPECT_EQ(element_values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(element_values.get_element(3), 3); - EXPECT_EQ(element_values.get_element(4), 4); - }); - auto list_map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field("nullable_list_map_int_string_col", - arrow::list(arrow::field("element", list_map_type, true)), true), - build_nullable_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& map_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 3); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(value_data.get_data_at(3).to_string(), "d"); - }); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field( - "nullable_map_int_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)), - true), - build_nullable_int_map_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 4); - - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 4); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - - const auto& inner_values = - assert_cast(outer_map.get_values()); - ASSERT_EQ(inner_values.size(), 4); - EXPECT_FALSE(inner_values.is_null_at(0)); - EXPECT_FALSE(inner_values.is_null_at(1)); - EXPECT_TRUE(inner_values.is_null_at(2)); - EXPECT_FALSE(inner_values.is_null_at(3)); - - const auto& inner_map = - assert_cast(inner_values.get_nested_column()); - const auto& inner_offsets = inner_map.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 4); - EXPECT_EQ(inner_offsets[0], 1); - EXPECT_EQ(inner_offsets[1], 1); - EXPECT_EQ(inner_offsets[2], 1); - EXPECT_EQ(inner_offsets[3], 2); - const auto& inner_keys = - get_nullable_nested_column(inner_map.get_keys()); - const auto& inner_strings = - assert_cast(inner_map.get_values()); - const auto& inner_string_data = - assert_cast(inner_strings.get_nested_column()); - ASSERT_EQ(inner_keys.size(), 2); - EXPECT_EQ(inner_keys.get_element(0), 101); - EXPECT_EQ(inner_keys.get_element(1), 401); - EXPECT_EQ(inner_string_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(inner_strings.is_null_at(1)); - }); - auto deep_list_value_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto deep_list_map_type = - arrow::map(arrow::int32(), arrow::field("value", deep_list_value_type, true)); - auto deep_list_struct_type = arrow::struct_({arrow::field("kv", deep_list_map_type, true)}); - add_field(arrow::field("nullable_list_struct_map_list_col", - arrow::list(arrow::field("element", deep_list_struct_type, true)), - true), - build_deep_list_struct_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& struct_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - - const auto& struct_column = - assert_cast(struct_values.get_nested_column()); - const auto& map_values = - assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - - const auto& lists = - assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 4); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 4); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 3); - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 3); - EXPECT_EQ(element_values.get_element(0), 10); - EXPECT_TRUE(elements.is_null_at(1)); - EXPECT_EQ(element_values.get_element(2), 40); - }); - auto deep_map_nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto deep_map_list_type = - arrow::list(arrow::field("element", deep_map_nested_map_type, true)); - add_field( - arrow::field( - "nullable_map_int_list_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", deep_map_list_type, true)), - true), - build_deep_map_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); - - const auto& inner_maps = - assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 6); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_FALSE(inner_maps.is_null_at(1)); - EXPECT_TRUE(inner_maps.is_null_at(2)); - EXPECT_FALSE(inner_maps.is_null_at(3)); - EXPECT_TRUE(inner_maps.is_null_at(4)); - EXPECT_FALSE(inner_maps.is_null_at(5)); - const auto& inner_map_column = - assert_cast(inner_maps.get_nested_column()); - const auto& inner_offsets = inner_map_column.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 6); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 3); - EXPECT_EQ(inner_offsets[4], 3); - EXPECT_EQ(inner_offsets[5], 4); - const auto& inner_keys = - get_nullable_nested_column(inner_map_column.get_keys()); - ASSERT_EQ(inner_keys.size(), 4); - EXPECT_EQ(inner_keys.get_element(0), 1); - EXPECT_EQ(inner_keys.get_element(1), 2); - EXPECT_EQ(inner_keys.get_element(2), 3); - EXPECT_EQ(inner_keys.get_element(3), 4); - const auto& strings = - assert_cast(inner_map_column.get_values()); - const auto& string_data = - assert_cast(strings.get_nested_column()); - ASSERT_EQ(strings.size(), 4); - EXPECT_EQ(string_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(strings.is_null_at(1)); - EXPECT_EQ(string_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(string_data.get_data_at(3).to_string(), "d"); - }); - - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - - auto file_result = arrow::io::FileOutputStream::Open(_file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ROW_COUNT, builder.build())); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_plain_reader(size_t field_idx) { - // Keep the normal fixture dictionary encoded. This one test writes a plain-encoded copy - // because Arrow BinaryRecordReader has a stricter reset contract than - // DictionaryRecordReader. - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - auto plain_file_result = arrow::io::FileOutputStream::Open(_plain_file_path); - DORIS_CHECK(plain_file_result.ok()); - std::shared_ptr plain_out = *plain_file_result; - ::parquet::WriterProperties::Builder plain_builder; - plain_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - plain_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - plain_builder.compression(::parquet::Compression::UNCOMPRESSED); - plain_builder.disable_dictionary(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), plain_out, ROW_COUNT, plain_builder.build())); - DORIS_CHECK(plain_out->Close().ok()); - - _plain_file_reader = ::parquet::ParquetFileReader::OpenFile(_plain_file_path, false); - auto metadata = _plain_file_reader->metadata(); - DORIS_CHECK(metadata != nullptr); - DORIS_CHECK(metadata->num_row_groups() == 1); - _plain_row_group = _plain_file_reader->RowGroup(0); - DORIS_CHECK(_plain_row_group != nullptr); - - ParquetColumnReaderFactory factory(_plain_row_group, metadata->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_child_reader(size_t field_idx, - size_t child_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[child_idx]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_grandchild_reader( - size_t field_idx, size_t child_idx, size_t grandchild_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - const auto& child_schema = *struct_schema.children[child_idx]; - EXPECT_LT(grandchild_idx, child_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = child_schema.local_id; - child_projection.project_all_children = false; - format::LocalColumnIndex grandchild_projection; - grandchild_projection.index = child_schema.children[grandchild_idx]->local_id; - child_projection.children.push_back(std::move(grandchild_projection)); - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - void read_and_validate(size_t field_idx) const { - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - _expected_by_field[field_idx](*_fields[field_idx], *column); - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet test field " << name; - return _fields.size(); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::string _plain_file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::unique_ptr<::parquet::ParquetFileReader> _plain_file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::shared_ptr<::parquet::RowGroupReader> _plain_row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; - std::vector> _expected_by_field; -}; - -TEST(ParquetColumnReaderBaseTest, SelectionVectorRangesAndValidation) { - SelectionVector identity; - ASSERT_TRUE(identity.verify(4, 5).ok()); - auto ranges = selection_to_ranges(identity, 4); - ASSERT_EQ(ranges.size(), 1); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 4); - - std::array selected = {0, 2, 3, 6, 6}; - SelectionVector external(selected.data(), 4); - auto status = external.verify(3, 7); - ASSERT_TRUE(status.ok()) << status; - ranges = selection_to_ranges(external, 3); - ASSERT_EQ(ranges.size(), 2); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 1); - EXPECT_EQ(ranges[1].start, 2); - EXPECT_EQ(ranges[1].length, 2); - - EXPECT_FALSE(external.verify(8, 7).ok()); - EXPECT_FALSE(external.verify(5, 7).ok()); - EXPECT_FALSE(external.verify(4, 6).ok()); - - std::array duplicate = {0, 2, 2}; - SelectionVector non_strict(duplicate.data(), duplicate.size()); - EXPECT_FALSE(non_strict.verify(3, 5).ok()); - EXPECT_FALSE(identity.verify(1, -1).ok()); -} - -TEST(ParquetColumnReaderBaseTest, DefaultSelectUsesSkipReadRangesAndNestedConsumeIsExplicit) { - DefaultSelectReader reader; - std::array selected = {1, 3, 4}; - SelectionVector selection(selected.data(), selected.size()); - auto column = ColumnInt32::create(); - MutableColumnPtr mutable_column = std::move(column); - auto status = reader.select(selection, selected.size(), 6, mutable_column); - ASSERT_TRUE(status.ok()) << status; - - const auto& values = assert_cast(*mutable_column); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 3); - EXPECT_EQ(values.get_element(2), 4); - EXPECT_EQ(reader.skip_ranges(), std::vector({1, 1, 1})); - EXPECT_EQ(reader.read_ranges(), std::vector({1, 2})); - - BaseUnsupportedReader unsupported_reader; - auto skip_status = unsupported_reader.skip(1); - EXPECT_FALSE(skip_status.ok()); - EXPECT_NE(skip_status.to_string().find("skip is not implemented"), std::string::npos); - EXPECT_FALSE(unsupported_reader.load_nested_batch(1).ok()); - int64_t values_read = 0; - EXPECT_FALSE(unsupported_reader.build_nested_column(1, mutable_column, &values_read).ok()); - EXPECT_FALSE(unsupported_reader.consume_nested_column(1, &values_read).ok()); - - NestedSkipReader nested_reader; - auto nested_status = nested_reader.consume_nested_column(3, &values_read); - ASSERT_TRUE(nested_status.ok()) << nested_status; - EXPECT_EQ(values_read, 3); -} - -TEST_F(ParquetColumnReaderTest, ScalarReadCoversRequiredNullableAllNullAndMultipleBatches) { - read_and_validate(find_field_idx("int32_col")); - read_and_validate(find_field_idx("string_col")); - read_and_validate(find_field_idx("nullable_int_col")); - read_and_validate(find_field_idx("all_null_int_col")); - - auto reader = create_reader(find_field_idx("int32_col")); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - ASSERT_TRUE(reader->read(3, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 3); - const auto& values = assert_cast(*column); - ASSERT_EQ(values.size(), ROW_COUNT); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_EQ(values.get_element(2), 30); - EXPECT_EQ(values.get_element(4), 50); -} - -TEST_F(ParquetColumnReaderTest, ScalarSkipCoversZeroSomeAllAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(0).ok()); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& first_value = assert_cast(*column); - EXPECT_EQ(first_value.get_element(0), 10); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(2).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& skipped_values = assert_cast(*column); - EXPECT_EQ(skipped_values.get_element(0), 30); - EXPECT_EQ(skipped_values.get_element(1), 40); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(ROW_COUNT).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - EXPECT_EQ(rows_read, 0); - EXPECT_EQ(column->size(), 0); - - reader = create_reader(find_field_idx("nullable_int_col")); - ASSERT_TRUE(reader->skip(1).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); -} - -TEST_F(ParquetColumnReaderTest, ScalarSelectCoversAllDisjointSingleZeroThenReadAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector all_selected(ROW_COUNT); - auto column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(all_selected, ROW_COUNT, ROW_COUNT, column).ok()); - const auto& all_values = assert_cast(*column); - ASSERT_EQ(all_values.size(), ROW_COUNT); - EXPECT_EQ(all_values.get_element(0), 10); - EXPECT_EQ(all_values.get_element(4), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array disjoint = {0, 2, 4}; - SelectionVector disjoint_selection(disjoint.data(), disjoint.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(disjoint_selection, disjoint.size(), ROW_COUNT, column).ok()); - const auto& disjoint_values = assert_cast(*column); - ASSERT_EQ(disjoint_values.size(), 3); - EXPECT_EQ(disjoint_values.get_element(0), 10); - EXPECT_EQ(disjoint_values.get_element(1), 30); - EXPECT_EQ(disjoint_values.get_element(2), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array single = {2}; - SelectionVector single_selection(single.data(), single.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(single_selection, single.size(), ROW_COUNT, column).ok()); - const auto& single_value = assert_cast(*column); - ASSERT_EQ(single_value.size(), 1); - EXPECT_EQ(single_value.get_element(0), 30); - - reader = create_reader(find_field_idx("int32_col")); - std::array first_last = {0, 4}; - SelectionVector first_last_selection(first_last.data(), first_last.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(first_last_selection, first_last.size(), ROW_COUNT, column).ok()); - const auto& first_last_values = assert_cast(*column); - ASSERT_EQ(first_last_values.size(), 2); - EXPECT_EQ(first_last_values.get_element(0), 10); - EXPECT_EQ(first_last_values.get_element(1), 50); - - reader = create_reader(find_field_idx("int32_col")); - SelectionVector empty_selection; - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(empty_selection, 0, 2, column).ok()); - ASSERT_EQ(column->size(), 0); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& after_empty_select = assert_cast(*column); - ASSERT_EQ(after_empty_select.size(), 1); - EXPECT_EQ(after_empty_select.get_element(0), 30); - - reader = create_reader(find_field_idx("nullable_int_col")); - std::array nullable_rows = {0, 1, 2}; - SelectionVector nullable_selection(nullable_rows.data(), nullable_rows.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(nullable_selection, nullable_rows.size(), ROW_COUNT, column).ok()); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidScalarInputsAndNestedScalarProjection) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& int_schema = *_fields[find_field_idx("int32_col")]; - ParquetColumnSchema invalid_leaf; - invalid_leaf.kind = ParquetColumnSchemaKind::PRIMITIVE; - invalid_leaf.name = "invalid_leaf"; - invalid_leaf.type = int_schema.type; - invalid_leaf.type_descriptor = int_schema.type_descriptor; - invalid_leaf.descriptor = int_schema.descriptor; - invalid_leaf.leaf_column_id = _file_reader->metadata()->num_columns(); - auto status = factory.create(invalid_leaf, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); - - ParquetColumnSchema null_descriptor; - null_descriptor.kind = ParquetColumnSchemaKind::PRIMITIVE; - null_descriptor.name = "null_descriptor"; - null_descriptor.type = int_schema.type; - null_descriptor.type_descriptor = int_schema.type_descriptor; - null_descriptor.leaf_column_id = int_schema.leaf_column_id; - status = factory.create(null_descriptor, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("descriptor is null"), std::string::npos); - - const auto& list_element_schema = - *_fields[find_field_idx("nullable_list_int_col")]->children[0]; - status = factory.create(list_element_schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("flat primitive columns"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex projection = - format::LocalColumnIndex::partial_local(list_schema.local_id); - format::LocalColumnIndex element_projection = - format::LocalColumnIndex::partial_local(list_element_schema.local_id); - projection.children.push_back(std::move(element_projection)); - status = factory.create(list_schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("scalar projection is invalid"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidComplexProjections) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& struct_schema = *_fields[find_field_idx("struct_col")]; - format::LocalColumnIndex struct_empty = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - auto status = factory.create(struct_schema, &struct_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); - - format::LocalColumnIndex struct_invalid = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - struct_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(struct_schema, &struct_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex list_empty = - format::LocalColumnIndex::partial_local(list_schema.local_id); - status = factory.create(list_schema, &list_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); - - const auto& map_schema = *_fields[find_field_idx("nullable_map_int_struct_col")]; - const auto& value_schema = *map_schema.children[1]; - format::LocalColumnIndex map_invalid = - format::LocalColumnIndex::partial_local(map_schema.local_id); - map_invalid.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - map_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(map_schema, &map_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadSupportedComplexTypes) { - read_and_validate(find_field_idx("struct_col")); - read_and_validate(find_field_idx("nullable_struct_col")); - read_and_validate(find_field_idx("nullable_struct_decimal_col")); - read_and_validate(find_field_idx("list_int_col")); - read_and_validate(find_field_idx("nullable_list_int_col")); - read_and_validate(find_field_idx("required_nullable_list_int_col")); - read_and_validate(find_field_idx("nullable_list_struct_col")); - read_and_validate(find_field_idx("nullable_list_list_int_col")); - read_and_validate(find_field_idx("map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_string_col")); - read_and_validate(find_field_idx("required_nullable_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_struct_col")); - read_and_validate(find_field_idx("nullable_map_int_list_col")); - read_and_validate(find_field_idx("nullable_list_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_map_int_string_col")); - read_and_validate(find_field_idx("nullable_list_struct_map_list_col")); - read_and_validate(find_field_idx("nullable_map_int_list_map_int_string_col")); -} - -TEST_F(ParquetColumnReaderTest, SkipThenRead) { - auto reader = create_reader(find_field_idx("int32_col")); - auto st = reader->skip(2); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 2); - EXPECT_EQ(int_values.get_element(0), 30); - EXPECT_EQ(int_values.get_element(1), 40); -} - -TEST_F(ParquetColumnReaderTest, SelectReadsOnlySelectedRanges) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 2); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 3); - EXPECT_EQ(int_values.get_element(0), 10); - EXPECT_EQ(int_values.get_element(1), 30); - EXPECT_EQ(int_values.get_element(2), 50); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructChildren) { - const auto field_idx = find_field_idx("struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& struct_column = assert_cast(*column); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(values.get_data_at(0).to_string(), "sa"); - EXPECT_EQ(values.get_data_at(4).to_string(), "se"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedNullableStructChildren) { - const auto field_idx = find_field_idx("nullable_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& nullable_column = assert_cast(*column); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = assert_cast(struct_column.get_column(0)); - const auto& nested_values = assert_cast(values.get_nested_column()); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(nested_values.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(nested_values.get_data_at(3).to_string(), "nsd"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedListStructElementChildren) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& list_schema = *_fields[field_idx]; - ASSERT_EQ(list_schema.name, "nullable_list_struct_col"); - ASSERT_EQ(list_schema.children.size(), 1); - const auto& element_schema = *list_schema.children[0]; - ASSERT_EQ(element_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = list_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex element_projection; - element_projection.index = element_schema.local_id; - element_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = element_schema.children[1]->local_id; - element_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(element_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(list_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* array_type = - assert_cast(remove_nullable(reader->type()).get()); - const auto* element_type = assert_cast( - remove_nullable(array_type->get_nested_type()).get()); - ASSERT_EQ(element_type->get_elements().size(), 1); - EXPECT_EQ(element_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& elements = assert_cast(array_column.get_data()); - const auto& struct_column = assert_cast(elements.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedMapStructValueChildren) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.name, "nullable_map_int_struct_col"); - ASSERT_EQ(map_schema.children.size(), 2); - const auto& value_schema = *map_schema.children[1]; - ASSERT_EQ(value_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = map_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex value_projection; - value_projection.index = value_schema.local_id; - value_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = value_schema.children[1]->local_id; - value_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(value_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* map_type = assert_cast(remove_nullable(reader->type()).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), TYPE_INT); - const auto* value_type = - assert_cast(remove_nullable(map_type->get_value_type()).get()); - ASSERT_EQ(value_type->get_elements().size(), 1); - EXPECT_EQ(value_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - const auto& values = assert_cast(map_column.get_values()); - const auto& struct_column = assert_cast(values.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); -} - -TEST_F(ParquetColumnReaderTest, AllowsMapKeyWithValueProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - const auto& value_schema = *map_schema.children[1]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - projection.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_NE(reader, nullptr); -} - -TEST_F(ParquetColumnReaderTest, RejectMapKeyOnlyProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_FALSE(st.ok()); - EXPECT_NE(st.to_string().find("contains no value"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_list_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "xs"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = assert_cast(xs_array.get_data()); - const auto& values = assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 4); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 5); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructListChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_map_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "kv"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesListChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesMapChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesNestedStructComplexChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_nested_struct_list_col"); - auto reader = create_projected_grandchild_reader(field_idx, 0, 0); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructMapChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadStructListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructListWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(1), 304); - EXPECT_EQ(a_values.get_element(2), 305); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadStructMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(1), 404); - EXPECT_EQ(a_values.get_element(2), 405); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, ReadListStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 1); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 1); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 1); - EXPECT_EQ(inner_offsets[0], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectListListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 5); - EXPECT_EQ(outer_offsets[2], 7); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[6], 7); -} - -TEST_F(ParquetColumnReaderTest, ReadMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SkipPlainBinaryMapThenReadResetsArrowBuilder) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_plain_reader(field_idx); - - // Row 0 contains two STRING values. The levels-only skip must release (and discard) those - // Arrow BinaryRecordReader builder chunks before the next normal read. If they leak into the - // next batch, ParquetLeafReader observes more values than current definition/repetition levels. - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 3); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(map_column.get_offsets()[1], 0); - EXPECT_EQ(map_column.get_offsets()[2], 1); - const auto& values = get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(values.size(), 1); - EXPECT_EQ(values.get_data_at(0).to_string(), "cc"); -} - -TEST_F(ParquetColumnReaderTest, SelectMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectMapStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 2); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectMapListWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 4); - EXPECT_EQ(map_offsets[2], 5); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(4)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepListStructMapListAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepListStructMapListThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 3); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_FALSE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 3); - EXPECT_TRUE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_FALSE(map_values.is_null_at(2)); - - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 2); - EXPECT_EQ(keys.get_element(0), 3); - EXPECT_EQ(keys.get_element(1), 4); - const auto& lists = assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 2); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepListStructMapList) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepMapListMapAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepMapListMapThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 3); - EXPECT_EQ(outer_keys.get_element(0), 30); - EXPECT_EQ(outer_keys.get_element(1), 40); - EXPECT_EQ(outer_keys.get_element(2), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 3); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_FALSE(lists.is_null_at(2)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 3); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); - EXPECT_EQ(list_offsets[2], 3); - const auto& inner_maps = assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 3); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_TRUE(inner_maps.is_null_at(1)); - EXPECT_FALSE(inner_maps.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepMapListMap) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp b/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp deleted file mode 100644 index 0d0f9a2f8567cc..00000000000000 --- a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp +++ /dev/null @@ -1,506 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" - -namespace doris::format::parquet { -namespace { - -std::shared_ptr fixed_binary_array(const std::vector& values, - int byte_width) { - auto type = arrow::fixed_size_binary(byte_width); - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetLeafReader make_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type) { - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr); -} - -struct CapturedDecodedView { - DecodedValueKind value_kind = DecodedValueKind::INT32; - DecodedTimeUnit time_unit = DecodedTimeUnit::UNKNOWN; - int64_t row_count = 0; - int decimal_precision = -1; - int decimal_scale = -1; - int fixed_length = -1; - bool timestamp_is_adjusted_to_utc = false; - bool enable_strict_mode = false; - const cctz::time_zone* timezone = nullptr; - bool null_map_is_null = true; - std::vector null_map; - std::vector fixed_values; - std::vector binary_values; - std::vector owned_binary_values; -}; - -ParquetLeafReader make_spy_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type, - CapturedDecodedView* captured, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false) { - auto appender = [captured](MutableColumnPtr&, const DecodedColumnView& view) { - captured->value_kind = view.value_kind; - captured->time_unit = view.time_unit; - captured->row_count = view.row_count; - captured->decimal_precision = view.decimal_precision; - captured->decimal_scale = view.decimal_scale; - captured->fixed_length = view.fixed_length; - captured->timestamp_is_adjusted_to_utc = view.timestamp_is_adjusted_to_utc; - captured->enable_strict_mode = view.enable_strict_mode; - captured->timezone = view.timezone; - captured->null_map_is_null = view.null_map == nullptr; - captured->null_map.clear(); - if (view.null_map != nullptr) { - captured->null_map.assign(view.null_map, view.null_map + view.row_count); - } - captured->fixed_values.clear(); - if (view.values != nullptr && view.value_kind == DecodedValueKind::INT64) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 8); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::FLOAT) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::INT32) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } - captured->binary_values.clear(); - captured->owned_binary_values.clear(); - if (view.binary_values != nullptr) { - captured->owned_binary_values.reserve(view.binary_values->size()); - for (const auto& value : *view.binary_values) { - captured->owned_binary_values.emplace_back( - value.data == nullptr ? std::string() - : std::string(value.data, value.size)); - } - captured->binary_values.reserve(captured->owned_binary_values.size()); - for (const auto& value : captured->owned_binary_values) { - captured->binary_values.emplace_back(value.data(), value.size()); - } - } - return Status::OK(); - }; - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr, {}, timezone, - enable_strict_mode, std::move(appender)); -} - -} // namespace - -struct ParquetLeafReaderTestAccess { - static ParquetLeafBatch make_fixed_batch(const std::vector& def_levels, - const std::vector& rep_levels, - const std::vector& values, - bool read_dense_for_nullable = false) { - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._consumed_level_count = static_cast(def_levels.size()); - batch._decoded_level_count = static_cast(def_levels.size()); - batch._values_written = static_cast(values.size()); - batch._def_levels = def_levels.data(); - batch._rep_levels = rep_levels.data(); - batch._fixed_values = reinterpret_cast(values.data()); - batch._read_dense_for_nullable = read_dense_for_nullable; - return batch; - } - - static Status build_nested_batch(const ParquetLeafReader& reader, - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, - int16_t value_slot_repetition_level, - ParquetNestedScalarBatch* nested_batch) { - return reader.build_nested_batch_from_leaf_batch(leaf_batch, records_read, - value_slot_definition_level, nested_batch, - value_slot_repetition_level); - } -}; - -std::shared_ptr<::parquet::ColumnDescriptor> int32_column_descriptor(int16_t max_definition_level, - int16_t max_repetition_level) { - auto node = ::parquet::schema::PrimitiveNode::Make("leaf", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32); - return std::make_shared<::parquet::ColumnDescriptor>(node, max_definition_level, - max_repetition_level); -} - -ParquetLeafReader make_nested_leaf_reader( - const std::shared_ptr<::parquet::ColumnDescriptor>& descriptor, DataTypePtr type) { - ParquetTypeDescriptor type_descriptor; - type_descriptor.physical_type = ::parquet::Type::INT32; - type_descriptor.doris_type = type; - return ParquetLeafReader(descriptor.get(), type_descriptor, std::move(type), "nested_leaf", - nullptr); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesAreSpacedBeforeSerde) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30, 50}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& nullable = assert_cast(*column); - ASSERT_EQ(nullable.size(), 5); - EXPECT_FALSE(nullable.is_null_at(0)); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_FALSE(nullable.is_null_at(2)); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_FALSE(nullable.is_null_at(4)); - const auto& nested = assert_cast(nullable.get_nested_column()); - EXPECT_EQ(nested.get_element(0), 10); - EXPECT_EQ(nested.get_element(2), 30); - EXPECT_EQ(nested.get_element(4), 50); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesRejectCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet values"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, Float16BinaryValuesAreConvertedToFloat) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - descriptor.fixed_length = 2; - auto type = std::make_shared(); - auto reader = make_leaf_reader(descriptor, type); - - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::FIXED_BINARY; - batch._binary_chunks = {fixed_binary_array( - {half(0x0000), half(0x8000), half(0x3E00), half(0x0001), half(0x7E00)}, 2)}; - batch._values_written = 5; - - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, nullptr, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& floats = assert_cast(*column); - ASSERT_EQ(floats.size(), 5); - EXPECT_FLOAT_EQ(floats.get_element(0), 0.0F); - EXPECT_TRUE(std::signbit(floats.get_element(1))); - EXPECT_FLOAT_EQ(floats.get_element(2), 1.5F); - EXPECT_NEAR(floats.get_element(3), 5.9604645e-8F, 1e-12F); - EXPECT_TRUE(std::isnan(floats.get_element(4))); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableValuesAreSpacedWithNullRefs) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("aa").ok()); - ASSERT_TRUE(builder.Append("cc").ok()); - ASSERT_TRUE(builder.Append("ee").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 3; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& nullable = assert_cast(*column); - const auto& strings = assert_cast(nullable.get_nested_column()); - ASSERT_EQ(nullable.size(), 5); - EXPECT_EQ(strings.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_EQ(strings.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_EQ(strings.get_data_at(4).to_string(), "ee"); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableRejectsCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("only_one").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 1; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 3, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet binary values"), - std::string::npos); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCarriesDescriptorSessionAndNullMapFields) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT64; - descriptor.time_unit = ParquetTimeUnit::NANOS; - descriptor.decimal_precision = 18; - descriptor.decimal_scale = 4; - descriptor.fixed_length = 12; - descriptor.timestamp_is_adjusted_to_utc = true; - auto type = make_nullable(std::make_shared()); - cctz::time_zone shanghai; - ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); - - CapturedDecodedView captured; - auto reader = make_spy_leaf_reader(descriptor, type, &captured, &shanghai, true); - const std::vector values = {100, 200, 300}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT64; - batch._fixed_values = reinterpret_cast(values.data()); - batch._values_written = values.size(); - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, &null_map, column).ok()); - EXPECT_EQ(captured.value_kind, DecodedValueKind::INT64); - EXPECT_EQ(captured.time_unit, DecodedTimeUnit::NANOS); - EXPECT_EQ(captured.row_count, 3); - EXPECT_EQ(captured.decimal_precision, 18); - EXPECT_EQ(captured.decimal_scale, 4); - EXPECT_EQ(captured.fixed_length, 12); - EXPECT_TRUE(captured.timestamp_is_adjusted_to_utc); - EXPECT_TRUE(captured.enable_strict_mode); - EXPECT_EQ(captured.timezone, &shanghai); - EXPECT_FALSE(captured.null_map_is_null); - EXPECT_EQ(captured.null_map, std::vector({0, 1, 0})); - - auto required_column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, nullptr, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); - - const NullMap empty_null_map; - ASSERT_TRUE(reader.append_values(batch, 3, &empty_null_map, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCapturesBinaryFixedLengthAndFloat16Override) { - ParquetTypeDescriptor binary_descriptor; - binary_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - binary_descriptor.fixed_length = 4; - auto type = std::make_shared(); - - CapturedDecodedView binary_view; - auto binary_reader = make_spy_leaf_reader(binary_descriptor, type, &binary_view); - ParquetLeafBatch binary_batch; - binary_batch._value_kind = DecodedValueKind::FIXED_BINARY; - binary_batch._binary_chunks = {fixed_binary_array({"abcd", "wxyz"}, 4)}; - binary_batch._values_written = 2; - auto binary_column = type->create_column(); - ASSERT_TRUE(binary_reader.append_values(binary_batch, 2, nullptr, binary_column).ok()); - EXPECT_EQ(binary_view.value_kind, DecodedValueKind::FIXED_BINARY); - EXPECT_EQ(binary_view.fixed_length, 4); - ASSERT_EQ(binary_view.owned_binary_values.size(), 2); - EXPECT_EQ(binary_view.owned_binary_values[0], "abcd"); - EXPECT_EQ(binary_view.owned_binary_values[1], "wxyz"); - - ParquetTypeDescriptor float16_descriptor; - float16_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - float16_descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - float16_descriptor.fixed_length = 2; - CapturedDecodedView float16_view; - auto float16_reader = make_spy_leaf_reader(float16_descriptor, - std::make_shared(), &float16_view); - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - ParquetLeafBatch float16_batch; - float16_batch._value_kind = DecodedValueKind::FIXED_BINARY; - float16_batch._binary_chunks = {fixed_binary_array({half(0x3E00), half(0x4000)}, 2)}; - float16_batch._values_written = 2; - auto float16_column = std::make_shared()->create_column(); - ASSERT_TRUE(float16_reader.append_values(float16_batch, 2, nullptr, float16_column).ok()); - EXPECT_EQ(float16_view.value_kind, DecodedValueKind::FLOAT); - ASSERT_EQ(float16_view.fixed_values.size(), sizeof(float) * 2); - const auto* floats = reinterpret_cast(float16_view.fixed_values.data()); - EXPECT_FLOAT_EQ(floats[0], 1.5F); - EXPECT_FLOAT_EQ(floats[1], 2.0F); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLevels) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 2, 2}; - const std::vector rep_levels = {0, 1, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 2, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.records_read, 2); - EXPECT_EQ(nested_batch.levels_written, 3); - EXPECT_EQ(nested_batch.value_indices, std::vector({0, 1, 2})); - const auto& nested_values = assert_cast(*nested_batch.values_column); - ASSERT_EQ(nested_values.size(), 3); - EXPECT_EQ(nested_values.get_element(0), 10); - EXPECT_EQ(nested_values.get_element(2), 30); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutValueSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 777, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 2, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLeafValues) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 1, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutPayloadSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {1, 2, 0, 2}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {777, 10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({-1, 1, -1, 2})); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsMismatchedValueLayout) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 0, 2, 0}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("inconsistent value count"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsDenseNullable) { - auto descriptor = int32_column_descriptor(1, 0); - auto reader = - make_nested_leaf_reader(descriptor, make_nullable(std::make_shared())); - const std::vector def_levels = {1}; - const std::vector rep_levels = {0}; - const std::vector values = {10}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values, true); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 1, 0, 0, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Dense nullable parquet nested reader is not supported"), - std::string::npos); -} - -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp index 940f94a373a013..cb54cab86b27d8 100644 --- a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp +++ b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp @@ -26,134 +26,6 @@ namespace doris::format::parquet { namespace { -void expect_plan_entry(const ParquetPageCacheReadPlanEntry& entry, - const ParquetPageCacheRange& cached_range, int64_t copy_offset_in_cache, - int64_t output_offset, int64_t copy_size) { - EXPECT_EQ(entry.cached_range.offset, cached_range.offset); - EXPECT_EQ(entry.cached_range.size, cached_range.size); - EXPECT_EQ(entry.copy_offset_in_cache, copy_offset_in_cache); - EXPECT_EQ(entry.output_offset, output_offset); - EXPECT_EQ(entry.copy_size, copy_size); -} - -TEST(ParquetPageCacheRangeTest, SubsetRequestHitsSingleCachedRange) { - const std::vector cached_ranges = { - {100, 100}, - }; - - // Request [120, 150) is fully inside cached [100, 200). The reader should lookup - // the exact cached key [100, 200), then copy from cached offset 20 into output offset 0. - auto plan = detail::plan_page_cache_range_read(120, 30, cached_ranges); - - ASSERT_EQ(plan.size(), 1); - expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestHitsMultipleAdjacentCachedRanges) { - const std::vector cached_ranges = { - {180, 80}, - {100, 80}, - }; - - // Request [100, 260) is larger than either cached entry, but the two cached ranges - // exactly cover it. The copy plan stitches the two exact cache entries together. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 80}, 0, 0, 80); - expect_plan_entry(plan[1], {180, 80}, 0, 80, 80); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestCanUseOverlappingCachedRanges) { - const std::vector cached_ranges = { - {150, 110}, - {100, 100}, - }; - - // Request [100, 260) is covered by overlapping cached ranges. The first copy uses - // [100, 200); the second resumes at cursor 200 and copies the tail from [150, 260). - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 100}, 0, 0, 100); - expect_plan_entry(plan[1], {150, 110}, 50, 100, 60); -} - -TEST(ParquetPageCacheRangeTest, PartialOverlapWithoutFullCoverageMisses) { - const std::vector cached_ranges = { - {100, 80}, - {200, 60}, - }; - - // Cached ranges cover [100, 180) and [200, 260), but [180, 200) is missing. - // The caller must read the whole request from the file instead of returning - // a partially cached result. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, NonCoveringAndInvalidRangesAreIgnored) { - const std::vector cached_ranges = { - {50, 20}, {100, 0}, {100, -1}, {180, 20}, {120, 30}, - }; - - // Only [120, 150) intersects the request, but it does not cover the request start - // [100, 120), so this is still a miss. - auto plan = detail::plan_page_cache_range_read(100, 50, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, InvalidRequestMisses) { - const std::vector cached_ranges = { - {100, 100}, - }; - - EXPECT_TRUE(detail::plan_page_cache_range_read(-1, 10, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, 0, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, -1, cached_ranges).empty()); -} - -TEST(ParquetPageCacheRangeTest, PerFileRangeIndexDeduplicatesAndEvictsAtCapacity) { - detail::ParquetPageCacheRangeIndex index(3); - index.insert({200, 20}); - index.insert({100, 30}); - index.insert({100, 10}); - index.insert({100, 30}); - - EXPECT_EQ(index.size(), 3); - index.insert({300, 40}); - const auto ranges = index.ranges(); - ASSERT_EQ(ranges.size(), 3); - EXPECT_EQ(ranges[0].offset, 100); - EXPECT_EQ(ranges[0].size, 30); - EXPECT_EQ(ranges[1].offset, 200); - EXPECT_EQ(ranges[1].size, 20); - EXPECT_EQ(ranges[2].offset, 300); - - index.erase({100, 30}); - EXPECT_EQ(index.size(), 2); -} - -TEST(ParquetPageCacheRangeTest, DirectorySharesBoundedIndexAcrossReaderLifetimes) { - detail::ParquetPageCacheRangeDirectory directory(2); - auto first_reader_index = directory.get_or_create("file-a"); - first_reader_index->insert({100, 100}); - first_reader_index.reset(); - - // The directory owns the per-file index, so reader B still discovers reader A's wider cache - // entry and can plan a subset hit after A closes. - auto second_reader_index = directory.get_or_create("file-a"); - const auto plan = detail::plan_page_cache_range_read(120, 30, second_reader_index->ranges()); - ASSERT_EQ(plan.size(), 1); - expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); - - directory.get_or_create("file-b"); - directory.get_or_create("file-c"); - EXPECT_EQ(directory.size(), 2); -} - TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) { const std::vector ranges = { {100, 50}, @@ -173,6 +45,19 @@ TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) EXPECT_EQ(valid_ranges[1].size, 60); } +TEST(ParquetPageCacheRangeTest, SerializedIndexesAreBoundedIndividuallyAndWhenCoalesced) { + constexpr size_t file_size = 1ULL << 30; + const auto budget = detail::MAX_SERIALIZED_PARQUET_INDEX_BYTES; + + EXPECT_TRUE(detail::is_serialized_index_range_safe(file_size, 0, budget)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, 0, budget + 1)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, -1, 1)); + + EXPECT_TRUE(detail::is_serialized_index_span_safe(100, 100 + budget)); + // Individually small adjacent indexes must not combine into one unbounded allocation. + EXPECT_FALSE(detail::is_serialized_index_span_safe(100, 101 + budget)); +} + TEST(ParquetPageCacheRangeTest, AveragePrefetchRangeSizeUsesOnlyValidRanges) { const std::vector ranges = { {0, 512}, diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 36b7cebdaa9cb9..5c434a346bd2c9 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -5,9 +5,7 @@ // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at -// // http://www.apache.org/licenses/LICENSE-2.0 -// // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,188 +13,48 @@ // specific language governing permissions and limitations // under the License. -#include #include -#include #include #include #include #include -#include #include #include "common/consts.h" #include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" #include "core/column/column_string.h" -#include "core/column/column_struct.h" #include "core/column/column_vector.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" #include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "format_v2/parquet/selection_vector.h" #include "storage/utils.h" namespace doris::format::parquet { namespace { -ParquetColumnSchema int64_schema(std::string name = "mock") { +ParquetColumnSchema int64_schema() { ParquetColumnSchema schema; schema.local_id = 0; - schema.name = std::move(name); + schema.name = "mock"; schema.type = std::make_shared(); return schema; } -ParquetColumnSchema nested_int64_schema(std::string name, int16_t nullable_definition_level, - int16_t definition_level, int16_t repetition_level = 0, - int16_t repeated_ancestor_definition_level = 0) { - ParquetColumnSchema schema = int64_schema(std::move(name)); - schema.type = make_nullable(std::make_shared()); - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - return schema; -} - -ParquetColumnSchema nested_struct_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "struct"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})); - return schema; -} - -ParquetColumnSchema nested_list_schema(std::string name, DataTypePtr element_type, - int16_t nullable_definition_level, int16_t definition_level, - int16_t repetition_level, - int16_t repeated_ancestor_definition_level) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = std::move(name); - schema.kind = ParquetColumnSchemaKind::LIST; - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - schema.type = make_nullable(std::make_shared(std::move(element_type))); - return schema; -} - -ParquetColumnSchema nested_map_schema( - DataTypePtr value_type = make_nullable(std::make_shared())) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "map"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.repetition_level = 1; - schema.repeated_ancestor_definition_level = 2; - schema.type = make_nullable(std::make_shared( - make_nullable(std::make_shared()), std::move(value_type))); - return schema; -} - -ParquetColumnSchema bare_repeated_int64_list_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "repeated"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.definition_level = 1; - schema.repetition_level = 1; - schema.repeated_repetition_level = 1; - schema.repeated_ancestor_definition_level = 1; - schema.type = std::make_shared(std::make_shared()); - return schema; -} - -std::unique_ptr primitive_child(int local_id, std::string name, - DataTypePtr type) { - auto child = std::make_unique(); - child->local_id = local_id; - child->name = std::move(name); - child->kind = ParquetColumnSchemaKind::PRIMITIVE; - child->leaf_column_id = local_id; - child->type = std::move(type); - child->type_descriptor.physical_type = ::parquet::Type::INT32; - child->type_descriptor.doris_type = child->type; - return child; -} - -ParquetColumnSchema struct_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "s"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.children.push_back(primitive_child(0, "a", std::make_shared())); - schema.children.push_back(primitive_child(1, "b", std::make_shared())); - DataTypes types = {make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)}; - Strings names = {"a", "b"}; - schema.type = std::make_shared(types, names); - return schema; -} - -ParquetColumnSchema list_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "xs"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.children.push_back(primitive_child(0, "element", std::make_shared())); - schema.type = std::make_shared(schema.children[0]->type); - return schema; -} - -ParquetColumnSchema map_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "m"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.children.push_back(primitive_child(0, "key", std::make_shared())); - schema.children.push_back(primitive_child(1, "value", std::make_shared())); - schema.type = std::make_shared(make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)); - return schema; -} - class CursorColumnReader final : public ParquetColumnReader { public: CursorColumnReader() : ParquetColumnReader(int64_schema(), std::make_shared()) {} Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("invalid mock read arguments"); - } - auto* values = assert_cast(column.get()); + DORIS_CHECK(column); + DORIS_CHECK(rows_read != nullptr); + auto& values = assert_cast(*column); for (int64_t row = 0; row < rows; ++row) { - values->insert_value(_cursor + row); + values.insert_value(_cursor + row); } _read_lengths.push_back(rows); _cursor += rows; @@ -205,271 +63,33 @@ class CursorColumnReader final : public ParquetColumnReader { } Status skip(int64_t rows) override { + DORIS_CHECK(rows >= 0); _skip_lengths.push_back(rows); _cursor += rows; return Status::OK(); } + void flush_profile() override { ++_profile_flushes; } + bool crossed_page_since_last_batch() override { + ++_page_crossing_checks; + return _crossed_page; + } + + void set_crossed_page(bool crossed_page) { _crossed_page = crossed_page; } + int64_t cursor() const { return _cursor; } const std::vector& skip_lengths() const { return _skip_lengths; } const std::vector& read_lengths() const { return _read_lengths; } + int profile_flushes() const { return _profile_flushes; } + int page_crossing_checks() const { return _page_crossing_checks; } private: int64_t _cursor = 0; std::vector _skip_lengths; std::vector _read_lengths; -}; - -class ScriptedNestedReader final : public ParquetColumnReader { -public: - ScriptedNestedReader(ParquetColumnSchema schema, DataTypePtr type, - std::vector def_levels, std::vector rep_levels, - bool has_repeated_child = false, bool build_nulls = false) - : ParquetColumnReader(schema, std::move(type)), - _def_levels(std::move(def_levels)), - _rep_levels(std::move(rep_levels)), - _has_repeated_child(has_repeated_child), - _build_nulls(build_nulls) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - _build_lengths.push_back(length_upper_bound); - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("invalid scripted nested build arguments"); - } - for (int64_t row = 0; row < length_upper_bound; ++row) { - insert_value(column, _next_value++, _build_nulls); - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - set_nested_build_level_cursor(std::min(nested_build_level_cursor() + length_upper_bound, - static_cast(_def_levels.size()))); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return _has_repeated_child; } - - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - -private: - static void insert_value(MutableColumnPtr& column, int64_t value, bool is_null) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - if (is_null) { - nullable_column->insert_default(); - return; - } - assert_cast(nullable_column->get_nested_column()).insert_value(value); - nullable_column->get_null_map_data().push_back(0); - return; - } - assert_cast(*column).insert_value(value); - } - - std::vector _def_levels; - std::vector _rep_levels; - bool _has_repeated_child = false; - bool _build_nulls = false; - int64_t _next_value = 0; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; -}; - -class ChunkedNestedLeafReader final : public ParquetColumnReader { -public: - ChunkedNestedLeafReader() - : ParquetColumnReader(nested_int64_schema("element", 0, 1, 1, 1), - std::make_shared()) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(values_read != nullptr); - _initial_column_sizes.push_back(column->size()); - _build_lengths.push_back(length_upper_bound); - if (auto* nullable = check_and_get_column(*column); nullable != nullptr) { - auto& values = assert_cast(nullable->get_nested_column()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values.insert_value(row); - nullable->get_null_map_data().push_back(0); - } - } else { - auto* values = assert_cast(column.get()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values->insert_value(row); - } - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return true; } - - const std::vector& load_lengths() const { return _load_lengths; } - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - const std::vector& initial_column_sizes() const { return _initial_column_sizes; } - -private: - std::vector _def_levels; - std::vector _rep_levels; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; - std::vector _initial_column_sizes; -}; - -} // namespace - -struct ScalarColumnReaderTestAccess { - static void set_nested_batch(ScalarColumnReader* reader, - std::unique_ptr batch) { - reader->_nested_batch = std::move(batch); - } - - static int64_t page_filtered_rows_to_skip(const ScalarColumnReader& reader, int64_t rows) { - return reader.page_filtered_rows_to_skip(rows); - } - - static void set_row_group_rows_read(ScalarColumnReader* reader, int64_t rows) { - reader->_row_group_rows_read = rows; - } - - static Status append_dictionary_filtered_values( - const ScalarColumnReader& reader, - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) { - return reader.append_dictionary_filtered_values(chunks, dictionary_filter, column, - row_filter, matched_rows, used_filter); - } -}; - -namespace { - -ParquetColumnSchema string_schema(std::string name = "string") { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = std::move(name); - schema.type = std::make_shared(); - schema.type_descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - schema.type_descriptor.doris_type = schema.type; - return schema; -} - -std::shared_ptr<::arrow::Array> dictionary_array(const std::vector& indices, - const std::vector& values) { - ::arrow::Int8Builder index_builder; - EXPECT_TRUE(index_builder.AppendValues(indices).ok()); - auto index_result = index_builder.Finish(); - EXPECT_TRUE(index_result.ok()) << index_result.status(); - - ::arrow::StringBuilder dictionary_builder; - EXPECT_TRUE(dictionary_builder.AppendValues(values).ok()); - auto dictionary_result = dictionary_builder.Finish(); - EXPECT_TRUE(dictionary_result.ok()) << dictionary_result.status(); - - auto result = ::arrow::DictionaryArray::FromArrays( - ::arrow::dictionary(::arrow::int8(), ::arrow::utf8()), *index_result, - *dictionary_result); - EXPECT_TRUE(result.ok()) << result.status(); - return *result; -} - -std::unique_ptr make_scripted_scalar_reader( - ParquetColumnSchema schema, std::unique_ptr batch) { - auto reader = std::make_unique(schema, nullptr); - ScalarColumnReaderTestAccess::set_nested_batch(reader.get(), std::move(batch)); - return reader; -} - -std::unique_ptr scalar_batch(std::vector def_levels, - std::vector rep_levels, - std::vector value_indices, - std::vector values) { - auto batch = std::make_unique(); - batch->levels_written = static_cast(def_levels.size()); - batch->def_levels = std::move(def_levels); - batch->rep_levels = std::move(rep_levels); - batch->value_indices = std::move(value_indices); - auto column = ColumnInt64::create(); - for (const auto value : values) { - column->insert_value(value); - } - batch->values_column = std::move(column); - return batch; -} - -class DefaultOnlyReader final : public ParquetColumnReader { -public: - DefaultOnlyReader() - : ParquetColumnReader(int64_schema("default_only"), std::make_shared()) { - } - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } + int _profile_flushes = 0; + bool _crossed_page = false; + int _page_crossing_checks = 0; }; GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { @@ -480,21 +100,6 @@ GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { return location; } -TEST(ParquetScalarColumnReaderTest, DictionaryIndexOutsideFilterIsCorruption) { - ScalarColumnReader reader(string_schema("dictionary_value"), nullptr); - MutableColumnPtr column = ColumnString::create(); - IColumn::Filter row_filter; - int64_t matched_rows = 0; - bool used_filter = false; - const std::vector> chunks = { - dictionary_array({0, 1}, {"keep", "out-of-range"})}; - - const auto status = ScalarColumnReaderTestAccess::append_dictionary_filtered_values( - reader, chunks, IColumn::Filter {1}, column, &row_filter, &matched_rows, &used_filter); - EXPECT_EQ(ErrorCode::CORRUPTION, status.code()) << status; - EXPECT_NE(status.to_string().find("Invalid parquet dictionary index 1"), std::string::npos); -} - } // namespace TEST(SelectionVectorTest, IdentitySelectionToRanges) { @@ -520,6 +125,21 @@ TEST(SelectionVectorTest, ExternalBufferSelectionToRanges) { EXPECT_TRUE(selection.verify(std::size(indices), 8).ok()); } +TEST(SelectionVectorTest, OutputRangesReuseCapacity) { + SelectionVector::Index indices[] = {1, 2, 5}; + SelectionVector selection(indices, std::size(indices)); + std::vector ranges; + ranges.reserve(8); + const auto retained_capacity = ranges.capacity(); + + selection_to_ranges(selection, std::size(indices), &ranges); + ASSERT_EQ(ranges.size(), 2); + EXPECT_EQ(ranges.capacity(), retained_capacity); + selection_to_ranges(selection, 1, &ranges); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges.capacity(), retained_capacity); +} + TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { SelectionVector selection(2); EXPECT_FALSE(selection.verify(3, 3).ok()); @@ -534,6 +154,28 @@ TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { EXPECT_FALSE(selection.verify(2, 3).ok()); } +TEST(SelectionVectorTest, MaterializedFilterIsReusedUntilSelectionChanges) { + SelectionVector selection(4); + selection.set_index(0, 1); + selection.set_index(1, 3); + const uint8_t* first_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &first_filter).ok()); + ASSERT_NE(first_filter, nullptr); + EXPECT_EQ(std::vector(first_filter, first_filter + 4), + std::vector({0, 1, 0, 1})); + + const uint8_t* reused_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &reused_filter).ok()); + EXPECT_EQ(reused_filter, first_filter); + + selection.set_index(1, 2); + const uint8_t* updated_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &updated_filter).ok()); + EXPECT_EQ(updated_filter, first_filter); + EXPECT_EQ(std::vector(updated_filter, updated_filter + 4), + std::vector({0, 1, 1, 0})); +} + TEST(ParquetColumnReaderControlTest, BaseSelectUsesSkipReadRanges) { CursorColumnReader reader; SelectionVector selection(3); @@ -559,571 +201,38 @@ TEST(ParquetColumnReaderControlTest, BaseSelectZeroRowsConsumesBatch) { SelectionVector selection; auto column = std::make_shared()->create_column(); ASSERT_TRUE(reader.select(selection, 0, 4, column).ok()); - EXPECT_EQ(column->size(), 0); + EXPECT_TRUE(column->empty()); EXPECT_EQ(reader.cursor(), 4); EXPECT_TRUE(reader.read_lengths().empty()); EXPECT_EQ(reader.skip_lengths(), std::vector({4})); } -TEST(ParquetColumnReaderControlTest, BaseNestedDefaultsAndSkipNested) { - DefaultOnlyReader base_reader; - EXPECT_FALSE(base_reader.skip(1).ok()); - EXPECT_FALSE(base_reader.load_nested_batch(1).ok()); - - auto column = std::make_shared()->create_column(); - int64_t values_read = 0; - EXPECT_FALSE(base_reader.build_nested_column(1, column, &values_read).ok()); - - int64_t values_consumed = 0; - EXPECT_FALSE(base_reader.consume_nested_column(1, &values_consumed).ok()); -} - -TEST(ParquetColumnReaderControlTest, NestedSkipConsumesBoundedBatchesWithoutMaterializing) { - auto element_reader = std::make_unique(); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); - - ASSERT_TRUE(reader.skip(8193).ok()); - EXPECT_TRUE(element_reader_ptr->load_lengths().empty()); - EXPECT_EQ(element_reader_ptr->level_load_lengths(), std::vector({4096, 4096, 1})); - EXPECT_EQ(element_reader_ptr->consume_lengths(), std::vector({4096, 4096, 1})); - EXPECT_TRUE(element_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(element_reader_ptr->initial_column_sizes().empty()); -} - -TEST(ParquetColumnReaderControlTest, MapSkipConsumesBothStreamsWithoutMaterializing) { - const std::vector def_levels {3, 3, 1, 3}; - const std::vector rep_levels {0, 1, 0, 0}; - auto key_reader = std::make_unique( - nested_int64_schema("key", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* key_reader_ptr = key_reader.get(); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); - - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(key_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(key_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_TRUE(key_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(value_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, StructSkipConsumesNullSeparatedChildSpans) { - const std::vector def_levels {2, 0, 2}; - const std::vector rep_levels {0, 0, 0}; - auto shape_reader = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* shape_reader_ptr = shape_reader.get(); - auto child_reader = std::make_unique( - nested_int64_schema("child", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* child_reader_ptr = child_reader.get(); - std::vector> children; - children.push_back(std::move(shape_reader)); - children.push_back(std::move(child_reader)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {-1, 0}); - - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(shape_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(child_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(shape_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_EQ(child_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_TRUE(shape_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(child_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, NestedListSkipConsumesRecursivelyWithoutMaterializing) { - auto leaf_reader = std::make_unique( - nested_int64_schema("leaf", 0, 1, 1, 1), std::make_shared(), - std::vector {1, 1}, std::vector {0, 0}); - auto* leaf_reader_ptr = leaf_reader.get(); - const auto inner_type = std::make_shared(std::make_shared()); - auto inner_reader = std::make_unique(bare_repeated_int64_list_schema(), - inner_type, std::move(leaf_reader)); - const auto outer_type = std::make_shared(inner_type); - ListColumnReader reader(bare_repeated_int64_list_schema(), outer_type, std::move(inner_reader)); - - ASSERT_TRUE(reader.skip(2).ok()); - EXPECT_EQ(leaf_reader_ptr->level_load_lengths(), std::vector({2})); - EXPECT_EQ(leaf_reader_ptr->consume_lengths(), std::vector({2})); - EXPECT_TRUE(leaf_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, NestedMaterializerHelpersAppendOffsetsAndParentNulls) { - ColumnArray::Offsets64 offsets; - append_offsets(offsets, {3, 0, 2}); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 3); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 5); - append_offsets(offsets, {1, 4}); - ASSERT_EQ(offsets.size(), 5); - EXPECT_EQ(offsets[3], 6); - EXPECT_EQ(offsets[4], 10); - - const NullMap parent_nulls = {0, 1, 0}; - append_parent_nulls(nullptr, parent_nulls); - NullMap dst = {1}; - append_parent_nulls(&dst, parent_nulls); - EXPECT_EQ(dst, NullMap({1, 0, 1, 0})); -} - -TEST(ParquetColumnReaderControlTest, PageFilteredRowsToSkipUsesOnlyFullSkippedRanges) { - ParquetPageSkipPlan page_skip_plan; - page_skip_plan.skipped_ranges = {RowRange {0, 3}, RowRange {5, 2}, RowRange {10, 4}}; - - auto schema = nested_int64_schema("page_filtered", 0, 0); - ScalarColumnReader reader(schema, nullptr, &page_skip_plan); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 3), 3); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 3); - - ScalarColumnReaderTestAccess::set_row_group_rows_read(&reader, 5); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 2), 2); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 2); -} - -TEST(ParquetColumnReaderControlTest, StructSkipsNullParentForRepeatedChildAndBatchesPresentRows) { - auto repeated_child = std::make_unique( - nested_int64_schema("repeated_shape", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2, 2, 2}, - std::vector {0, 0, 0, 0}, true); - auto* repeated_child_ptr = repeated_child.get(); - auto scalar_child = make_scripted_scalar_reader( - nested_int64_schema("scalar_child", 1, 2), - scalar_batch({2, 0, 2, 2}, {0, 0, 0, 0}, {0, -1, 1, 2}, {10, 20, 30})); - auto* scalar_child_ptr = scalar_child.get(); - - std::vector> children; - children.push_back(std::move(repeated_child)); - children.push_back(std::move(scalar_child)); - StructColumnReader reader(nested_struct_schema(), - make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})), - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_EQ(repeated_child_ptr->build_lengths(), std::vector({1, 2})); - EXPECT_EQ(scalar_child_ptr->nested_build_level_cursor(), 4); -} - -TEST(ParquetColumnReaderControlTest, StructFallsBackToFirstChildWhenAllChildrenAreRepeated) { - auto first_child = std::make_unique( - nested_int64_schema("first", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 0}, std::vector {0, 0}, true); - auto second_child = std::make_unique( - nested_int64_schema("second", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2}, - std::vector {0, 0}, true); - - std::vector> children; - children.push_back(std::move(first_child)); - children.push_back(std::move(second_child)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(rows_read, 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesComplexChildShapeOnly) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 2, 0, 0, 2, 2}, std::vector {0, 0, 0, 0, 0, 0}); - - ParquetColumnSchema map_schema = nested_map_schema(); - map_schema.nullable_definition_level = 2; - map_schema.definition_level = 3; - map_schema.repeated_ancestor_definition_level = 0; - auto key_reader = std::make_unique( - nested_int64_schema("key", 3, 3, 1, 0), - make_nullable(std::make_shared()), - std::vector {3, 3, 0, 0, 3, 3}, std::vector {0, 0, 0, 0, 0, 0}); - auto value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 4, 4, 1, 0), - scalar_batch({4, 4, 0, 0, 4, 4}, {0, 0, 0, 0, 0, 0}, - {0, 1, -1, -1, 2, 3}, {10, 20, 30, 40})); - auto map_reader = std::make_unique( - map_schema, map_schema.type, std::move(key_reader), std::move(value_reader)); - - std::vector> children; - children.push_back(std::move(shape_child)); - children.push_back(std::move(map_reader)); - auto struct_type = make_nullable(std::make_shared(DataTypes {map_schema.type}, - Strings {"partitionValues"})); - StructColumnReader reader(nested_struct_schema(), struct_type, std::move(children), {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(6, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 6); - - const auto& nullable_struct = assert_cast(*column); - ASSERT_EQ(nullable_struct.size(), 6); - EXPECT_FALSE(nullable_struct.is_null_at(0)); - EXPECT_FALSE(nullable_struct.is_null_at(1)); - EXPECT_TRUE(nullable_struct.is_null_at(2)); - EXPECT_TRUE(nullable_struct.is_null_at(3)); - EXPECT_FALSE(nullable_struct.is_null_at(4)); - EXPECT_FALSE(nullable_struct.is_null_at(5)); - - const auto& struct_column = - assert_cast(nullable_struct.get_nested_column()); - const auto& map_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_nullable.size(), 6); - EXPECT_FALSE(map_nullable.is_null_at(0)); - EXPECT_FALSE(map_nullable.is_null_at(1)); - EXPECT_TRUE(map_nullable.is_null_at(2)); - EXPECT_TRUE(map_nullable.is_null_at(3)); - EXPECT_FALSE(map_nullable.is_null_at(4)); - EXPECT_FALSE(map_nullable.is_null_at(5)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 6); - EXPECT_EQ(map_column.get_offsets()[0], 1); - EXPECT_EQ(map_column.get_offsets()[1], 2); - EXPECT_EQ(map_column.get_offsets()[2], 2); - EXPECT_EQ(map_column.get_offsets()[3], 2); - EXPECT_EQ(map_column.get_offsets()[4], 3); - EXPECT_EQ(map_column.get_offsets()[5], 4); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesNestedStructDescendants) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 0, 2}, std::vector {0, 0, 0}); - - auto id_batch = scalar_batch({4, 3, 4}, {0, 0, 0}, {0, -1, 1}, {10, 20}); - id_batch->value_slot_definition_level = 3; - auto id_reader = - make_scripted_scalar_reader(nested_int64_schema("id", 3, 4), std::move(id_batch)); - - ParquetColumnSchema inner_schema; - inner_schema.local_id = 0; - inner_schema.name = "stats_parsed"; - inner_schema.kind = ParquetColumnSchemaKind::STRUCT; - inner_schema.nullable_definition_level = 2; - inner_schema.definition_level = 3; - inner_schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared())}, Strings {"id"})); - - std::vector> inner_children; - inner_children.push_back(std::move(id_reader)); - auto inner_reader = std::make_unique( - inner_schema, inner_schema.type, std::move(inner_children), std::vector {0}); - - std::vector> outer_children; - outer_children.push_back(std::move(shape_child)); - outer_children.push_back(std::move(inner_reader)); - auto outer_type = make_nullable(std::make_shared(DataTypes {inner_schema.type}, - Strings {"stats_parsed"})); - StructColumnReader reader(nested_struct_schema(), outer_type, std::move(outer_children), - {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); - - const auto& outer_nullable = assert_cast(*column); - ASSERT_EQ(outer_nullable.size(), 3); - EXPECT_FALSE(outer_nullable.is_null_at(0)); - EXPECT_TRUE(outer_nullable.is_null_at(1)); - EXPECT_FALSE(outer_nullable.is_null_at(2)); - - const auto& outer_struct = assert_cast(outer_nullable.get_nested_column()); - const auto& inner_nullable = assert_cast(outer_struct.get_column(0)); - ASSERT_EQ(inner_nullable.size(), 3); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_TRUE(inner_nullable.is_null_at(1)); - EXPECT_FALSE(inner_nullable.is_null_at(2)); - - const auto& inner_struct = assert_cast(inner_nullable.get_nested_column()); - const auto& id_nullable = assert_cast(inner_struct.get_column(0)); - const auto& id_values = assert_cast(id_nullable.get_nested_column()); - EXPECT_EQ(id_values.get_element(0), 10); - EXPECT_EQ(id_values.get_element(2), 20); -} - -TEST(ParquetColumnReaderControlTest, ListKeepsEmptyBareRepeatedPrimitiveRows) { - auto element_reader = std::make_unique( - nested_int64_schema("element", 0, 1, 1, 1), std::make_shared(), - std::vector {0, 1, 1, 0}, std::vector {0, 0, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); - - const auto& array_column = assert_cast(*column); - ASSERT_EQ(array_column.get_offsets().size(), 3); - EXPECT_EQ(array_column.get_offsets()[0], 0); - EXPECT_EQ(array_column.get_offsets()[1], 2); - EXPECT_EQ(array_column.get_offsets()[2], 2); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({2})); -} - -TEST(ParquetColumnReaderControlTest, NestedListSkipsAncestorEmptyRowsButKeepsNullElements) { - auto element_reader = - std::make_unique(nested_int64_schema("element", 5, 5, 2, 4), - make_nullable(std::make_shared()), - std::vector {1, 5, 5, 5, 2, 5, 2, 0}, - std::vector {0, 0, 2, 1, 0, 1, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - - const auto inner_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()))); - auto inner_reader = std::make_unique( - nested_list_schema("inner", make_nullable(std::make_shared()), 3, 4, 2, - 2), - inner_type, std::move(element_reader)); - auto outer_type = make_nullable(std::make_shared(inner_type)); - ListColumnReader reader(nested_list_schema("outer", inner_type, 1, 2, 1, 2), outer_type, - std::move(inner_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 5); - EXPECT_EQ(outer_offsets[3], 5); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 5); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_TRUE(inner_nullable.is_null_at(4)); - - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 5); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 3); - EXPECT_EQ(inner_offsets[2], 3); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 4); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({4})); -} - -TEST(ParquetColumnReaderControlTest, MapKeepsEmptyMapRows) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 1); - - const auto& nullable_map = assert_cast(*column); - EXPECT_FALSE(nullable_map.is_null_at(0)); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(value_reader_ptr->build_lengths(), std::vector({0})); -} - -TEST(ParquetColumnReaderControlTest, ListMapSkipsAncestorEmptyRowsBeforeScalarValues) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 4, 4, 2, 4), - make_nullable(std::make_shared()), std::vector {1, 4}, - std::vector {0, 0}); - auto value_reader = make_scripted_scalar_reader(nested_int64_schema("value", 5, 5, 2, 4), - scalar_batch({1, 5}, {0, 0}, {-1, 0}, {100})); +TEST(ParquetColumnReaderControlTest, SchedulerFlushesReaderProfilesAtBatchBoundary) { + ParquetScanScheduler scheduler; + auto reader = std::make_unique(); + auto* reader_ptr = reader.get(); + scheduler._current_predicate_columns.emplace(0, std::move(reader)); - const auto map_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()), - make_nullable(std::make_shared()))); - auto map_reader = std::make_unique( - nested_map_schema(make_nullable(std::make_shared())), map_type, - std::move(key_reader), std::move(value_reader)); - auto outer_type = make_nullable(std::make_shared(map_type)); - ListColumnReader reader(nested_list_schema("outer", map_type, 1, 2, 1, 2), outer_type, - std::move(map_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 2); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 2); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 1); - - const auto& map_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(map_nullable.size(), 1); - EXPECT_FALSE(map_nullable.is_null_at(0)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 1); - - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 1); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_EQ(value_data.get_element(0), 100); + scheduler.flush_current_reader_profiles(); + EXPECT_EQ(reader_ptr->profile_flushes(), 1); } -TEST(ParquetColumnReaderControlTest, MapRejectsNullKeysAndMisalignedScalarValueRepLevels) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}, false, true); - auto value_reader = std::make_unique( - nested_int64_schema("value", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}); - MapColumnReader null_key_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(value_reader)); - auto column = null_key_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = null_key_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null key"), std::string::npos); - - auto aligned_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(aligned_key_reader), - std::move(misaligned_value_reader)); - column = misaligned_reader.type()->create_column(); - status = misaligned_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} - -TEST(ParquetColumnReaderControlTest, MapConsumePreservesKeyAndValueCorruptionChecks) { - auto null_key_reader = - make_scripted_scalar_reader(nested_int64_schema("key", 2, 3, 1, 2), - scalar_batch({2}, {0}, {-1}, std::vector {})); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {2}, - std::vector {0}); - MapColumnReader null_key_reader_map(nested_map_schema(), nested_map_schema().type, - std::move(null_key_reader), std::move(value_reader)); - int64_t values_consumed = 0; - auto status = null_key_reader_map.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null"), std::string::npos); +TEST(ParquetColumnReaderControlTest, SchedulerOrsPageCrossingOncePerBatch) { + ParquetScanScheduler scheduler; + auto predicate_reader = std::make_unique(); + auto* predicate_ptr = predicate_reader.get(); + predicate_ptr->set_crossed_page(true); + scheduler._current_predicate_columns.emplace(0, std::move(predicate_reader)); - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(misaligned_value_reader)); - status = misaligned_reader.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} - -TEST(ParquetColumnReaderControlTest, MapBuildsScalarAndComplexValuePaths) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto scalar_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 1}, {0, 1}, {100, 200})); - MapColumnReader scalar_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(scalar_value_reader)); - auto column = scalar_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = scalar_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_map = assert_cast(*column); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 2); - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 2); - EXPECT_EQ(value_data.get_element(0), 100); - EXPECT_EQ(value_data.get_element(1), 200); + auto lazy_reader = std::make_unique(); + auto* lazy_ptr = lazy_reader.get(); + lazy_ptr->set_crossed_page(true); + scheduler._current_non_predicate_columns.emplace(1, std::move(lazy_reader)); - auto complex_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto complex_value_reader = std::make_unique( - nested_int64_schema("complex_value", 2, 3, 1), - make_nullable(std::make_shared()), std::vector {3, 3}, - std::vector {0, 1}); - auto* complex_value_reader_ptr = complex_value_reader.get(); - MapColumnReader complex_reader(nested_map_schema(), nested_map_schema().type, - std::move(complex_key_reader), std::move(complex_value_reader)); - column = complex_reader.type()->create_column(); - status = complex_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(complex_value_reader_ptr->build_lengths(), std::vector({2})); + // Both readers are sampled even after the OR becomes true so their next batch starts cleanly. + EXPECT_TRUE(scheduler.finish_current_reader_batch_profiles()); + EXPECT_EQ(predicate_ptr->page_crossing_checks(), 1); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 1); } TEST(ParquetVirtualColumnReaderTest, RowPositionReadSkipAndInvalidArgs) { @@ -1192,79 +301,4 @@ TEST(ParquetVirtualColumnReaderTest, GlobalRowIdReadSkipSelectAndInvalidArgs) { EXPECT_FALSE(reader.read(1, column, nullptr).ok()); } -TEST(ParquetColumnReaderFactoryTest, RejectsInvalidLeafIdBeforeCreatingRecordReader) { - ParquetColumnSchema schema = int64_schema("bad_leaf"); - schema.kind = ParquetColumnSchemaKind::PRIMITIVE; - schema.leaf_column_id = 3; - schema.type_descriptor.physical_type = ::parquet::Type::INT64; - schema.type_descriptor.doris_type = schema.type; - - ParquetColumnReaderFactory factory(nullptr, 1); - std::unique_ptr reader; - const auto status = factory.create(schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsProjectedUnsupportedLogicalType) { - ParquetColumnSchema schema = int64_schema("unsupported_time"); - schema.kind = ParquetColumnSchemaKind::PRIMITIVE; - schema.type_descriptor.unsupported_reason = - "Parquet TIME with isAdjustedToUTC=true is not supported"; - - ParquetColumnReaderFactory factory(nullptr, 1); - std::unique_ptr reader; - const auto status = factory.create(schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find(schema.type_descriptor.unsupported_reason), - std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsStructInvalidAndEmptyProjection) { - auto schema = struct_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto empty_projection = format::LocalColumnIndex::partial_local(0); - status = factory.create(schema, &empty_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsListProjectionWithoutElement) { - auto schema = list_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto projection = format::LocalColumnIndex::partial_local(0); - const auto status = factory.create(schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsMapInvalidAndKeyOnlyProjection) { - auto schema = map_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(1)); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto key_only_projection = format::LocalColumnIndex::partial_local(0); - key_only_projection.children.push_back(format::LocalColumnIndex::local(0)); - status = factory.create(schema, &key_only_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no value"), std::string::npos); -} - } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 7402595fc37d06..b63c161b0b6200 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -35,10 +35,15 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_decimal.h" +#include "core/column/column_map.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -58,6 +63,7 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -66,6 +72,7 @@ #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -466,6 +473,14 @@ std::shared_ptr build_string_array(const std::vector& return finish_array(&builder); } +std::shared_ptr build_binary_array(const std::vector& values) { + arrow::BinaryBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_timestamp_array(const std::shared_ptr& type, const std::vector& values) { arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); @@ -475,6 +490,27 @@ std::shared_ptr build_timestamp_array(const std::shared_ptr build_decimal_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); + for (const auto value : values) { + EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_fixed_binary_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); + const int32_t byte_width = + std::static_pointer_cast(type)->byte_width(); + for (const auto& value : values) { + EXPECT_EQ(value.size(), byte_width); + EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_struct_array(const std::vector& ids, const std::vector& names) { auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false), @@ -517,6 +553,41 @@ void write_parquet_file(const std::string& file_path, int64_t row_group_size = R row_group_size, builder.build())); } +void write_unannotated_binary_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("raw_bytes", arrow::binary(), false)}); + auto table = arrow::Table::Make(schema, {build_binary_array({"否", "是", "测试"})}); + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 3, + builder.build())); +} + +void write_decimal_and_fixed_binary_parquet_file(const std::string& file_path) { + auto decimal_type = arrow::decimal128(38, 6); + auto fixed_type = arrow::fixed_size_binary(4); + auto schema = arrow::schema({arrow::field("decimal_value", decimal_type, false), + arrow::field("fixed_value", fixed_type, false)}); + auto table = arrow::Table::Make( + schema, + {build_decimal_array(decimal_type, {1234567, -1, 0, -987654321, 42}), + build_fixed_binary_array(fixed_type, {"ABCD", std::string("\0x\0y", 4), "wxyz", "1234", + std::string("\xff\x00\x7f\x80", 4)})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.disable_dictionary(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + std::shared_ptr build_nullable_int_string_map_array() { auto key_builder = std::make_shared(); auto value_builder = std::make_shared(); @@ -686,6 +757,27 @@ void write_nullable_string_struct_parquet_file(const std::string& file_path) { ROW_COUNT, builder.build())); } +void write_nullable_complex_parquet_file(const std::string& file_path) { + auto map_array = build_nullable_int_string_map_array(); + auto list_array = build_nullable_string_list_array(); + auto struct_array = build_nullable_string_struct_array(); + auto table = arrow::Table::Make(arrow::schema({arrow::field("m", map_array->type(), true), + arrow::field("a", list_array->type(), true), + arrow::field("s", struct_array->type(), true)}), + {map_array, list_array, struct_array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_nullable_struct_with_list_parquet_file(const std::string& file_path) { auto scalar_first = build_nullable_struct_with_list_array(false); auto list_first = build_nullable_struct_with_list_array(true); @@ -706,6 +798,34 @@ void write_nullable_struct_with_list_parquet_file(const std::string& file_path) ROW_COUNT, builder.build())); } +void write_nested_complex_under_struct_parquet_file(const std::string& file_path) { + auto nested_struct = build_nullable_string_struct_array(); + auto nested_array = build_nullable_string_list_array(); + auto nested_map = build_nullable_int_string_map_array(); + auto marker = build_int32_array({10, 20, 30, 40, 50}); + auto struct_field = arrow::field("nested_struct", nested_struct->type(), true); + auto array_field = arrow::field("nested_array", nested_array->type(), true); + auto map_field = arrow::field("nested_map", nested_map->type(), true); + auto marker_field = arrow::field("marker", arrow::int32(), false); + auto outer_result = + arrow::StructArray::Make({nested_struct, nested_array, nested_map, marker}, + {struct_field, array_field, map_field, marker_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + auto outer = *outer_result; + auto table = arrow::Table::Make(arrow::schema({arrow::field("outer", outer->type(), false)}), + {outer}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_int96_timestamp_parquet_file(const std::string& file_path) { auto field = arrow::field("ts_tz", arrow::timestamp(arrow::TimeUnit::MICRO), true); auto array = @@ -792,14 +912,21 @@ void write_struct_filter_parquet_file(const std::string& file_path) { builder.build())); } -void write_dictionary_filter_parquet_file(const std::string& file_path) { +void write_dictionary_filter_parquet_file( + const std::string& file_path, + ::parquet::Compression::type compression = ::parquet::Compression::UNCOMPRESSED) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("value", arrow::utf8(), false), }); - auto table = - arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), - build_string_array({"aa", "az", "lm", "lz", "za", "zz"})}); + const std::vector values = + compression == ::parquet::Compression::UNCOMPRESSED + ? std::vector {"aa", "az", "lm", "lz", "za", "zz"} + : std::vector {std::string(4096, 'a'), std::string(4096, 'b'), + std::string(4096, 'c'), std::string(4096, 'd'), + std::string(4096, 'e'), std::string(4096, 'f')}; + auto table = arrow::Table::Make( + schema, {build_int32_array({1, 2, 3, 4, 5, 6}), build_string_array(values)}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -808,7 +935,7 @@ void write_dictionary_filter_parquet_file(const std::string& file_path) { ::parquet::WriterProperties::Builder builder; builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.compression(compression); builder.enable_dictionary("value"); builder.disable_dictionary("id"); builder.disable_statistics(); @@ -867,31 +994,6 @@ void write_dictionary_filter_with_trailing_column_parquet_file(const std::string builder.build())); } -void write_nested_dictionary_filter_parquet_file(const std::string& file_path) { - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); - auto schema = arrow::schema({ - arrow::field("s", struct_type, false), - }); - auto table = arrow::Table::Make( - schema, {build_struct_array({1, 2, 3, 4, 5, 6}, {"aa", "az", "lm", "lz", "za", "zz"})}); - - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.enable_dictionary("s.name"); - builder.disable_dictionary("s.identifier.field_id"); - builder.disable_statistics(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, - builder.build())); -} - void write_dictionary_edge_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), @@ -917,62 +1019,6 @@ void write_dictionary_edge_parquet_file(const std::string& file_path) { builder.build())); } -void write_nested_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); - std::vector names; - names.reserve(ids.size()); - for (const auto id : ids) { - names.push_back("name-" + std::to_string(id)); - } - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); - auto schema = arrow::schema({ - arrow::field("s", struct_type, false), - }); - auto table = arrow::Table::Make(schema, {build_struct_array(ids, names)}); - - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); -} - -void write_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), false), - }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); - - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); -} - void write_page_index_filter_pair_parquet_file(const std::string& file_path) { std::vector ids(128); std::iota(ids.begin(), ids.end(), 0); @@ -1136,7 +1182,8 @@ class NewParquetReaderTest : public testing::Test { RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, - bool is_immutable = false) const { + bool is_immutable = false, bool enable_mapping_varbinary = false, + std::string fs_name = {}, int64_t mtime = 0) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); @@ -1145,9 +1192,11 @@ class NewParquetReaderTest : public testing::Test { file_description->range_start_offset = range_start_offset; file_description->range_size = range_size; file_description->is_immutable = is_immutable; + file_description->fs_name = std::move(fs_name); + file_description->mtime = mtime; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz); + global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); } std::filesystem::path _test_dir; @@ -1172,6 +1221,39 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_STRING); } +TEST_F(NewParquetReaderTest, RawByteArrayMappingFollowsV2ScanOption) { + write_unannotated_binary_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto string_reader = create_reader(); + const auto string_init_status = string_reader->init(&state); + ASSERT_TRUE(string_init_status.ok()) << string_init_status; + std::vector string_schema; + ASSERT_TRUE(string_reader->get_schema(&string_schema).ok()); + ASSERT_EQ(string_schema.size(), 1); + EXPECT_EQ(remove_nullable(string_schema[0].type)->get_primitive_type(), TYPE_STRING); + + RuntimeProfile binary_profile("raw_binary_mapping_profile"); + auto binary_reader = + create_reader(0, -1, &binary_profile, false, nullptr, std::nullopt, false, true); + const auto binary_init_status = binary_reader->init(&state); + ASSERT_TRUE(binary_init_status.ok()) << binary_init_status; + std::vector binary_schema; + ASSERT_TRUE(binary_reader->get_schema(&binary_schema).ok()); + ASSERT_EQ(binary_schema.size(), 1); + // The explicit VARBINARY mapping must remain available for scans whose table contract asks for it. + EXPECT_EQ(remove_nullable(binary_schema[0].type)->get_primitive_type(), TYPE_VARBINARY); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back(create_string_in_conjunct(0, {"是"})); + ASSERT_TRUE(binary_reader->open(request).ok()); + // A table-side STRING comparison may be rewritten through this VARBINARY file slot. Native + // dictionary pruning must leave the row group available for the mapping expression. + EXPECT_EQ(binary_profile.get_counter("RowGroupsFilteredByDictionary")->value(), 0); +} + // Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and // file-layer pruning hints. The reader declares those scan-request capabilities by choosing // ParquetColumnMapper itself. @@ -1203,6 +1285,10 @@ TEST_F(NewParquetReaderTest, CountComplexColumnUsesShapeOnlyPath) { EXPECT_EQ(result.count, 4); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); EXPECT_EQ(profile.get_counter("MaterializationTime")->value(), 0); + ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); + ASSERT_NE(profile.get_counter("ParsePageHeaderNum"), nullptr); + EXPECT_GT(profile.get_counter("ParsePageHeaderNum")->value(), 0); } TEST_F(NewParquetReaderTest, CountArrayColumnUsesLevelsOnlyPath) { @@ -1271,6 +1357,238 @@ TEST_F(NewParquetReaderTest, CountStructWithRepeatedChildUsesTopLevelRowBoundari } } +TEST_F(NewParquetReaderTest, NativeComplexColumnsMaterializeDirectlyAcrossBatchChanges) { + write_nullable_complex_parquet_file(_file_path); + RuntimeProfile profile("native_complex_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1), + field_projection(2)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + int batch = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + if (++batch == 1) { + // Adaptive sizing changes only the logical row cap. The persistent native readers and + // their level/string scratch must continue from the same page cursors. + reader->set_batch_size(3); + } + } + + const auto& nullable_map = assert_cast(*output[0]); + ASSERT_EQ(nullable_map.size(), ROW_COUNT); + EXPECT_FALSE(nullable_map.is_null_at(0)); + EXPECT_TRUE(nullable_map.is_null_at(1)); + EXPECT_FALSE(nullable_map.is_null_at(2)); + const auto& map = assert_cast(nullable_map.get_nested_column()); + EXPECT_EQ(map.get_offsets(), ColumnArray::Offsets64({1, 1, 1, 2, 3})); + const auto& map_keys = assert_cast(map.get_keys()); + const auto& key_values = assert_cast(map_keys.get_nested_column()); + ASSERT_EQ(key_values.size(), 3); + EXPECT_EQ(key_values.get_element(0), 10); + EXPECT_EQ(key_values.get_element(2), 30); + const auto& map_values = assert_cast(map.get_values()); + const auto& value_strings = assert_cast(map_values.get_nested_column()); + EXPECT_EQ(value_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(value_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(map_values.is_null_at(2)); + + const auto& nullable_array = assert_cast(*output[1]); + ASSERT_EQ(nullable_array.size(), ROW_COUNT); + EXPECT_TRUE(nullable_array.is_null_at(1)); + const auto& array = assert_cast(nullable_array.get_nested_column()); + EXPECT_EQ(array.get_offsets(), ColumnArray::Offsets64({2, 2, 2, 3, 4})); + const auto& array_values = assert_cast(array.get_data()); + const auto& array_strings = assert_cast(array_values.get_nested_column()); + EXPECT_EQ(array_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(array_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(array_values.is_null_at(2)); + EXPECT_EQ(array_strings.get_data_at(3).size, 4096); + + const auto& nullable_struct = assert_cast(*output[2]); + ASSERT_EQ(nullable_struct.size(), ROW_COUNT); + EXPECT_TRUE(nullable_struct.is_null_at(1)); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& payload = assert_cast(struct_column.get_column(0)); + const auto& payload_strings = assert_cast(payload.get_nested_column()); + EXPECT_EQ(payload_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(payload_strings.get_data_at(2).size, 4096); + EXPECT_TRUE(payload.is_null_at(3)); + EXPECT_EQ(payload_strings.get_data_at(4).size, 4096); + const auto& ids = assert_cast(struct_column.get_column(1)); + const auto& id_values = assert_cast(ids.get_nested_column()); + EXPECT_EQ(id_values.get_element(0), 1); + EXPECT_EQ(id_values.get_element(4), 4); + + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + ASSERT_NE(profile.get_counter("NestedBatches"), nullptr); + EXPECT_GT(profile.get_counter("NestedBatches")->value(), 0); +} + +TEST_F(NewParquetReaderTest, FullComplexChildUnderPartialParentReadsItsWholeSubtree) { + write_nested_complex_under_struct_parquet_file(_file_path); + for (size_t child_index = 0; child_index < 3; ++child_index) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + ASSERT_EQ(schema[0].children.size(), 4); + auto projection = format::LocalColumnIndex::partial_local(0); + projection.children.push_back( + format::LocalColumnIndex::local(schema[0].children[child_index].file_local_id())); + auto request = std::make_shared(); + request->non_predicate_columns = {projection}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + format::ColumnDefinition projected; + ASSERT_TRUE(format::project_column_definition(schema[0], projection, &projected).ok()); + Block block; + block.insert(ColumnWithTypeAndName(projected.type->create_column(), projected.type, + projected.name)); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_GT(rows, 0); + const auto& outer = nullable_nested_column(block, 0); + ASSERT_EQ(outer.tuple_size(), 1); + const auto& nullable_nested = assert_cast(outer.get_column(0)); + if (child_index == 0) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + ASSERT_EQ(nested.tuple_size(), 2); + const auto& payload = assert_cast(nested.get_column(0)); + EXPECT_EQ(assert_cast(payload.get_nested_column()) + .get_data_at(0) + .to_string(), + "small"); + const auto& ids = assert_cast(nested.get_column(1)); + EXPECT_EQ(ids.get_nested_column().get_int(0), 1); + } else if (child_index == 1) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 2); + EXPECT_EQ(nested.get_data().size(), 4); + } else { + const auto& nested = assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 1); + EXPECT_EQ(nested.get_keys().size(), 3); + EXPECT_EQ(nested.get_values().size(), 3); + } + } +} + +TEST_F(NewParquetReaderTest, NativeNestedMapUsesOuterKeyRepetitionShape) { + const char* source_root = std::getenv("ROOT"); + ASSERT_NE(source_root, nullptr); + _file_path = + std::string(source_root) + "/regression-test/data/external_table_p0/tvf/comp.parquet"; + ASSERT_TRUE(std::filesystem::exists(_file_path)); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + for (size_t position = 0; position < schema.size(); ++position) { + request->non_predicate_columns.push_back(field_projection(cast_set(position))); + } + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const Status status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + total_rows += rows; + } + EXPECT_GT(total_rows, 0); +} + +TEST_F(NewParquetReaderTest, NativeDecimalAndFixedBinaryMaterializeDirectly) { + write_decimal_and_fixed_binary_parquet_file(_file_path); + RuntimeProfile profile("native_decimal_fixed_binary_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + reader->set_batch_size(3); + } + + const auto& decimals = assert_cast( + assert_cast(*output[0]).get_nested_column()); + ASSERT_EQ(decimals.size(), ROW_COUNT); + EXPECT_EQ(decimals.get_element(0), Decimal128V3(1234567)); + EXPECT_EQ(decimals.get_element(1), Decimal128V3(-1)); + EXPECT_EQ(decimals.get_element(3), Decimal128V3(-987654321)); + + const auto& fixed_values = assert_cast( + assert_cast(*output[1]).get_nested_column()); + ASSERT_EQ(fixed_values.size(), ROW_COUNT); + EXPECT_EQ(fixed_values.get_data_at(0).to_string(), "ABCD"); + EXPECT_EQ(fixed_values.get_data_at(1).to_string(), std::string("\0x\0y", 4)); + EXPECT_EQ(fixed_values.get_data_at(4).to_string(), std::string("\xff\x00\x7f\x80", 4)); + + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); + EXPECT_EQ(profile.get_counter("ConvertTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { write_struct_filter_parquet_file(_file_path); auto reader = create_reader(); @@ -1297,6 +1615,35 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { EXPECT_TRUE(struct_type->get_element(1)->is_nullable()); } +TEST_F(NewParquetReaderTest, ComplexColumnDoesNotMisreportSiblingPagesAsCrossing) { + write_struct_filter_parquet_file(_file_path); + RuntimeProfile profile("new_parquet_reader_complex_page_fragments"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, 4); + // Each STRUCT child reads one page per Row Group, but no individual leaf crosses a boundary in + // either batch. The aggregate fragment count remains useful without inflating crossing batches. + EXPECT_EQ(profile.get_counter("NativePageFragments")->value(), 8); + EXPECT_EQ(profile.get_counter("PageCrossingBatches")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaMapsInt96ToTimestampTzWhenTimestampTzMappingEnabled) { write_int96_timestamp_parquet_file(_file_path); auto reader = create_reader(0, -1, nullptr, true); @@ -1532,10 +1879,78 @@ TEST_F(NewParquetReaderTest, UnknownMtimeSkipsPageCacheForMutableFile) { ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); ASSERT_NE(profile.get_counter("PageCacheWriteCount"), nullptr); - EXPECT_EQ(profile.get_counter("PageReadCount")->value(), 0); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); EXPECT_EQ(profile.get_counter("PageCacheWriteCount")->value(), 0); } +TEST_F(NewParquetReaderTest, NativeFooterCacheIdentityIncludesFilesystemAndVersion) { + const auto first = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto other_fs = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-b", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto replaced = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 11, 0, 1024, 1024, false); + EXPECT_FALSE(first.empty()); + EXPECT_NE(first, other_fs); + EXPECT_NE(first, replaced); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotCrossFilesystems) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile first_profile("native_footer_cache_nameservice_a"); + auto first = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-a", 1234567); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + RuntimeProfile second_profile("native_footer_cache_nameservice_b"); + auto second = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-b", 1234567); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheSkipsMutableUnknownVersion) { + EXPECT_TRUE(format::parquet::detail::build_native_file_cache_key("hdfs://nameservice-a", + "/warehouse/mutable.parquet", + 0, 0, 1024, 1024, false) + .empty()); + EXPECT_FALSE( + format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/immutable.parquet", 0, 0, 1024, 1024, true) + .empty()); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotReuseMutableUnknownVersion) { + _file_path = (_test_dir / "mutable_footer_cache.parquet").string(); + write_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("native_footer_cache_mutable_first"); + auto first = create_reader(0, -1, &first_profile); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + write_parquet_file(_file_path); + RuntimeProfile second_profile("native_footer_cache_mutable_second"); + auto second = create_reader(0, -1, &second_profile); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterSizeIsBoundedBeforeMetadataAllocation) { + constexpr size_t file_size = 256UL << 20; + constexpr size_t metadata_limit = 100UL << 20; + const auto status = format::parquet::detail::validate_native_footer_size( + static_cast(metadata_limit + 1), file_size, metadata_limit); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("metadata limit"), std::string::npos); +} + TEST_F(NewParquetReaderTest, UnknownMtimeUsesPageCacheForImmutableFile) { _file_path = (_test_dir / "unknown_mtime_page_cache.parquet").string(); write_parquet_file(_file_path); @@ -1645,24 +2060,42 @@ TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); ASSERT_NE(profile.get_counter("ReaderReadRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSelectRows"), nullptr); - ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterHitCache"), nullptr); ASSERT_GT(profile.get_counter("FileReaderCreateTime")->value(), 0); EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 3); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 2); + TRuntimeProfileTree profile_tree; + profile.to_thrift(&profile_tree); + ASSERT_FALSE(profile_tree.nodes.empty()); + const auto parquet_children = + profile_tree.nodes.front().child_counters_map.find("ParquetReader"); + ASSERT_NE(parquet_children, profile_tree.nodes.front().child_counters_map.end()); + EXPECT_TRUE(parquet_children->second.contains("RowsFilteredByConjunct")); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 1); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderReadRows")->value(), ROW_COUNT + 3); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 2); EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 3); - EXPECT_GT(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); EXPECT_GT(profile.get_counter("MaterializationTime")->value(), 0); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + EXPECT_EQ(profile.get_counter("FileFooterReadCalls")->value() + + profile.get_counter("FileFooterHitCache")->value(), + 1); rows = 0; eof = false; @@ -1777,14 +2210,61 @@ TEST_F(NewParquetReaderTest, EmptySelectionUpdatesProfileCounters) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 0); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 0); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 1); } +TEST_F(NewParquetReaderTest, ProfileNestsFormatReaderBelowFileReaderAndRecordsTotalTime) { + RuntimeProfile profile("new_parquet_reader_hierarchy_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + + auto* parquet_total = profile.get_counter("ParquetReader"); + ASSERT_NE(parquet_total, nullptr); + EXPECT_GT(parquet_total->value(), 0); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains(RuntimeProfile::ROOT_COUNTER)); + EXPECT_TRUE(children.at(RuntimeProfile::ROOT_COUNTER).contains("FileScannerV2")); + ASSERT_TRUE(children.contains("FileScannerV2")); + EXPECT_TRUE(children.at("FileScannerV2").contains("TableReader")); + ASSERT_TRUE(children.contains("TableReader")); + EXPECT_TRUE(children.at("TableReader").contains("FileReader")); + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + EXPECT_TRUE(children.at("FileReader").contains("ParquetReader")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("ColumnReadTime")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowGroupsReadNum")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredRowsByGroup")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredBytes")); + EXPECT_TRUE(children.at("ParquetReader").contains("FileNum")); +} + TEST_F(NewParquetReaderTest, ReadMultiPredicateColumnsBeforeExpressionFilter) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); @@ -2010,41 +2490,6 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByStatistics) { TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto value_chunk = row_group->ColumnChunk(1); - ASSERT_NE(value_chunk, nullptr); - ASSERT_TRUE(value_chunk->has_dictionary_page()); - ASSERT_TRUE(value_chunk->statistics() == nullptr || - !value_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 2); - - format::FileScanRequest plan_request; - plan_request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - plan_request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - plan_request, scan_range, false, &plan) - .ok()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 5); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 5); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, 1); - auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); @@ -2080,6 +2525,55 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { EXPECT_EQ(values, std::vector({"lm"})); } +TEST_F(NewParquetReaderTest, DictionaryPruningPublishesColdAndWarmNativePageProfile) { + const double old_cache_threshold = config::parquet_page_cache_decompress_threshold; + config::parquet_page_cache_decompress_threshold = 100.0; + Defer restore_cache_threshold { + [&] { config::parquet_page_cache_decompress_threshold = old_cache_threshold; }}; + write_dictionary_filter_parquet_file(_file_path, ::parquet::Compression::SNAPPY); + + auto open_pruned_reader = [&](RuntimeProfile* profile) { + auto reader = create_reader(0, -1, profile, false, nullptr, std::nullopt, true); + TQueryOptions query_options; + query_options.__set_enable_parquet_file_page_cache(true); + RuntimeState state {query_options, TQueryGlobals()}; + RETURN_IF_ERROR(reader->init(&state)); + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"not-present"})); + use_schema_order_positions(request.get(), schema); + RETURN_IF_ERROR(reader->open(request)); + EXPECT_EQ(profile->get_counter("RowGroupsFilteredByDictionary")->value(), 0); + EXPECT_EQ(profile->get_counter("PageReadCount")->value(), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + // Expensive dictionary probes are current-row-group work now, so advance the scheduler + // once before inspecting page-cache and pruning counters. + return reader->get_block(&block, &rows, &eof); + }; + + RuntimeProfile cold_profile("dictionary_pruning_cold_profile"); + ASSERT_TRUE(open_pruned_reader(&cold_profile).ok()); + for (const auto* counter_name : + {"RowGroupsFilteredByDictionary", "PageReadCount", "PageCacheWriteCount", + "ParsePageHeaderNum", "DecompressCount", "DecodeDictTime"}) { + ASSERT_NE(cold_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(cold_profile.get_counter(counter_name)->value(), 0) << counter_name; + } + + RuntimeProfile warm_profile("dictionary_pruning_warm_profile"); + ASSERT_TRUE(open_pruned_reader(&warm_profile).ok()); + for (const auto* counter_name : {"RowGroupsFilteredByDictionary", "PageReadCount", + "PageCacheHitCount", "ParsePageHeaderNum", "DecodeDictTime"}) { + ASSERT_NE(warm_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(warm_profile.get_counter(counter_name)->value(), 0) << counter_name; + } +} + TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { write_single_row_group_dictionary_filter_parquet_file(_file_path); auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); @@ -2135,7 +2629,7 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { EXPECT_GE(profile.get_counter("ReaderSelectRows")->value(), 8); } -TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) { +TEST_F(NewParquetReaderTest, DictionaryPredicateReaderIsSharedOutsideMergeRangeReader) { write_dictionary_filter_with_trailing_column_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_dictionary_filter_merge_profile"); @@ -2177,8 +2671,14 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) EXPECT_EQ(values, std::vector({"az", "za"})); EXPECT_EQ(payloads, std::vector({20, 50})); EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); + // The native dictionary probe keeps its reader and cursor for predicate data pages. Other + // projected chunks still use the row-group merge reader, so probing never duplicates the + // dictionary read or perturbs the merge range's sequential access order. ASSERT_NE(profile.get_counter("MergedIO"), nullptr); ASSERT_NE(profile.get_counter("MergedBytes"), nullptr); + EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); } TEST_F(NewParquetReaderTest, DictionaryPredicateWorksWithoutRuntimeProfile) { @@ -2258,7 +2758,9 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateSkipsRemainingPredicateColumnsWh // second predicate column is skipped after the selection becomes empty, which verifies the // StarRocks-style round-by-round policy: only rows surviving previous predicates are read. EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 6); - EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 6); + // Five dictionary ids are rejected before the residual predicate; the remaining predicate + // reader then skips all six logical rows once the selection is empty. + EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 11); } TEST_F(NewParquetReaderTest, DictionaryPredicateRunsResidualConjunctOnSurvivors) { @@ -2346,208 +2848,10 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateKeepsNestedOrResidualConjunct) { EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 1); } -TEST_F(NewParquetReaderTest, ScanRangeFiltersRowGroupsBeforeDictionaryPruning) { - write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 2); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByStatistics) { - write_struct_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 2); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByDictionary) { - write_nested_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto name_chunk = row_group->ColumnChunk(1); - ASSERT_NE(name_chunk, nullptr); - ASSERT_TRUE(name_chunk->has_dictionary_page()); - ASSERT_TRUE(name_chunk->statistics() == nullptr || !name_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[1]->name, "name"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 6); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, PlannerNarrowsRowRangesByPageIndex) { - write_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - - format::FileScanRequest request; - request.predicate_columns = {field_projection(0)}; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_GT(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_LT(plan.row_groups[0].selected_ranges.front().length, 128); - auto skip_plan_it = plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(skip_plan_it, plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(skip_plan_it->second.leaf_column_id, 0); - EXPECT_GT(skip_plan_it->second.skipped_ranges.size(), 0); - EXPECT_GT(skip_plan_it->second.skipped_pages.size(), 1); - ASSERT_EQ(skip_plan_it->second.skipped_pages.size(), - skip_plan_it->second.skipped_page_compressed_sizes.size()); - int64_t skipped_compressed_bytes = 0; - for (size_t page_idx = 0; page_idx < skip_plan_it->second.skipped_pages.size(); ++page_idx) { - if (skip_plan_it->second.should_skip_page(page_idx)) { - skipped_compressed_bytes += skip_plan_it->second.skipped_page_compressed_size(page_idx); - } - } - EXPECT_GT(skipped_compressed_bytes, 0); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_GT(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotNarrowRowRangesByPageIndex) { - write_nested_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().length, - parquet_file_reader->metadata()->RowGroup(0)->num_rows()); - EXPECT_TRUE(plan.row_groups[0].page_skip_plans.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - // Scenario: the selected range starts after page-index-pruned rows. The scheduler defers that range // gap for the non-predicate payload reader, then flushes it exactly once before materialization. The -// page skip plan advances the reader without calling Arrow SkipRecords or double-skipping row 64. +// native RowRanges/OffsetIndex plan advances both readers without decoding the rejected pages or +// double-skipping row 64. TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { write_page_index_filter_pair_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_page_skip"); @@ -2571,7 +2875,8 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { bool eof = false; while (!eof) { size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } @@ -2589,7 +2894,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RangeGapSkippedRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); - ASSERT_NE(profile.get_counter("ArrowSkipRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); ASSERT_NE(profile.get_counter("RowGroupFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexReadTime"), nullptr); @@ -2599,7 +2904,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 64); EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 0); - EXPECT_EQ(profile.get_counter("ArrowSkipRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); EXPECT_GT(profile.get_counter("RowGroupFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexReadTime")->value(), 0); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 36cc99ebf106c0..b44b4e6dcdae46 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -23,9 +23,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -44,13 +46,16 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/field.h" +#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_reader.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "gen_cpp/PlanNodes_types.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -58,6 +63,8 @@ #include "storage/index/zone_map/zonemap_eval_context.h" #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/utils.h" +#include "util/coding.h" +#include "util/thrift_util.h" namespace doris { namespace { @@ -73,6 +80,13 @@ const ColumnInt32& int32_data_column(const IColumn& column) { return assert_cast(column); } +const ColumnInt64& int64_data_column(const IColumn& column) { + if (const auto* nullable_column = check_and_get_column(&column)) { + return assert_cast(nullable_column->get_nested_column()); + } + return assert_cast(column); +} + const ColumnString& string_data_column(const IColumn& column) { if (const auto* nullable_column = check_and_get_column(&column)) { return assert_cast(nullable_column->get_nested_column()); @@ -80,6 +94,64 @@ const ColumnString& string_data_column(const IColumn& column) { return assert_cast(column); } +TEST(ParquetScanMetadataSafetyTest, CheckedChunkRangesDrivePrefetchAndSplitAssignment) { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + metadata.__set_schema({root, leaf}); + + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + column.__set_data_page_offset(100); + column.__set_dictionary_page_offset(-1); + column.__set_total_compressed_size(20); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + row_group.__set_columns({chunk}); + metadata.__set_row_groups({row_group}); + + auto schema = std::make_unique(); + schema->local_id = 0; + schema->leaf_column_id = 0; + schema->type = std::make_shared(); + std::vector> file_schema; + file_schema.push_back(std::move(schema)); + std::vector ranges; + ASSERT_TRUE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges[0].offset, 100); + EXPECT_EQ(ranges[0].size, 20); + + std::vector first_rows; + std::vector selected; + format::parquet::ParquetScanRange first_split { + .start_offset = 0, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, first_split, &first_rows, &selected) + .ok()); + EXPECT_TRUE(selected.empty()); + format::parquet::ParquetScanRange second_split { + .start_offset = 100, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, second_split, &first_rows, &selected) + .ok()); + EXPECT_EQ(selected, std::vector({0})); + + metadata.row_groups[0].columns[0].meta_data.__set_data_page_offset(190); + EXPECT_FALSE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); +} + class Int32ZoneMapExpr final : public VExpr { public: enum class Op { GE, GT, LT }; @@ -195,17 +267,179 @@ class Int32PairSumExpr final : public VExpr { const std::string _expr_name = "Int32PairSumExpr"; }; +class Int32DirectGreaterExpr final : public VExpr { +public: + Int32DirectGreaterExpr(int column_id, int32_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int32_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_INT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int32_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int32_t _lower_bound; + const std::string _expr_name = "Int32DirectGreaterExpr"; +}; + +class Int64DirectGreaterExpr final : public VExpr { +public: + Int64DirectGreaterExpr(int column_id, int64_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int64_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_BIGINT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + if (value_width != sizeof(int64_t)) { + return Status::Corruption("BIGINT raw predicate received {}-byte values", value_width); + } + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int64_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int64_t _lower_bound; + const std::string _expr_name = "Int64DirectGreaterExpr"; +}; + +class AlwaysTrueSingleColumnExpr final : public VExpr { +public: + explicit AlwaysTrueSingleColumnExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + const std::string _expr_name = "AlwaysTrueSingleColumnExpr"; +}; + VExprContextSPtr create_int32_zonemap_conjunct(int column_id, Int32ZoneMapExpr::Op op, int32_t value) { return VExprContext::create_shared(std::make_shared(column_id, op, value)); } +VExprContextSPtr create_int32_function_conjunct(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t value) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto result_type = make_nullable(std::make_shared()); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({nullable_int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(result_type->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(result_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(true); + auto root = VectorizedFnCall::create_shared(node); + root->add_child( + VSlotRef::create_shared(column_id, column_id, -1, nullable_int_type, "plain_id")); + root->add_child(VLiteral::create_shared(int_type, Field::create_field(value))); + auto context = VExprContext::create_shared(std::move(root)); + // Direct evaluation does not execute the expression, but a fallback must still fail this test + // instead of silently using an unprepared test-only context. + context->_prepared = true; + context->_opened = true; + return context; +} + VExprContextSPtr create_int32_pair_sum_conjunct(int left_column_id, int right_column_id, int32_t upper_bound) { return VExprContext::create_shared( std::make_shared(left_column_id, right_column_id, upper_bound)); } +VExprContextSPtr create_int32_direct_greater_conjunct(int column_id, int32_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + +VExprContextSPtr create_int64_direct_greater_conjunct(int column_id, int64_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + +VExprContextSPtr create_always_true_single_column_conjunct(int column_id) { + return VExprContext::create_shared(std::make_shared(column_id)); +} + int64_t counter_value(RuntimeProfile& profile, const std::string& name) { auto* counter = profile.get_counter(name); DORIS_CHECK(counter != nullptr); @@ -226,6 +460,14 @@ std::shared_ptr build_int32_array(const std::vector& valu return finish_array(&builder); } +std::shared_ptr build_uint32_array(const std::vector& values) { + arrow::UInt32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -344,6 +586,120 @@ void write_int_pair_parquet_file(const std::string& file_path, int64_t row_group write_table(file_path, table, row_group_size, false, false, enable_statistics); } +void write_uint32_pair_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("id", arrow::uint32(), false), + arrow::field("score", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_uint32_array({1, 2147483648U, 4294957294U}), + build_int32_array({10, 20, 30})}); + write_table(file_path, table, 3, false, false, false); +} + +std::vector serialize_test_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +void write_misdeclared_two_page_parquet_file(const std::string& file_path) { + const std::array first_values {1, 2}; + std::vector first_payload(sizeof(first_values)); + memcpy(first_payload.data(), first_values.data(), first_payload.size()); + tparquet::PageHeader first_header; + first_header.type = tparquet::PageType::DATA_PAGE_V2; + first_header.__set_compressed_page_size(first_payload.size()); + first_header.__set_uncompressed_page_size(first_payload.size()); + first_header.__isset.data_page_header_v2 = true; + first_header.data_page_header_v2.__set_num_values(first_values.size()); + first_header.data_page_header_v2.__set_num_nulls(0); + first_header.data_page_header_v2.__set_num_rows(first_values.size()); + first_header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + first_header.data_page_header_v2.__set_definition_levels_byte_length(0); + first_header.data_page_header_v2.__set_repetition_levels_byte_length(0); + first_header.data_page_header_v2.__set_is_compressed(false); + auto column_bytes = serialize_test_page(first_header, first_payload); + + auto node = ::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32); + ::parquet::ColumnDescriptor descriptor(node, 0, 0); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, &descriptor); + const int32_t second_values[] = {3, 4}; + encoder->Put(second_values, std::size(second_values)); + auto second_buffer = encoder->FlushValues(); + std::vector second_payload(second_buffer->data(), + second_buffer->data() + second_buffer->size()); + tparquet::PageHeader second_header = first_header; + second_header.__set_compressed_page_size(second_payload.size()); + second_header.__set_uncompressed_page_size(second_payload.size()); + second_header.data_page_header_v2.__set_encoding(tparquet::Encoding::DELTA_BINARY_PACKED); + auto second_page = serialize_test_page(second_header, second_payload); + column_bytes.insert(column_bytes.end(), second_page.begin(), second_page.end()); + + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("id"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + // Deliberately omit DELTA_BINARY_PACKED to emulate untrusted/incomplete footer metadata. + column.__set_encodings({tparquet::Encoding::PLAIN, tparquet::Encoding::RLE}); + column.__set_path_in_schema({"id"}); + column.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column.__set_num_values(4); + column.__set_total_uncompressed_size(column_bytes.size()); + column.__set_total_compressed_size(column_bytes.size()); + column.__set_data_page_offset(4); + tparquet::ColumnChunk chunk; + chunk.__set_file_offset(4); + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(column_bytes.size()); + row_group.__set_num_rows(4); + tparquet::FileMetaData metadata; + metadata.__set_version(2); + metadata.__set_schema({root, leaf}); + metadata.__set_num_rows(4); + metadata.__set_row_groups({row_group}); + + std::vector footer; + ThriftSerializer serializer(/*compact=*/true, 1024); + DORIS_CHECK(serializer.serialize(&metadata, &footer).ok()); + std::ofstream output(file_path, std::ios::binary | std::ios::trunc); + output.write("PAR1", 4); + output.write(reinterpret_cast(column_bytes.data()), column_bytes.size()); + output.write(reinterpret_cast(footer.data()), footer.size()); + std::array footer_size {}; + encode_fixed32_le(footer_size.data(), cast_set(footer.size())); + output.write(reinterpret_cast(footer_size.data()), footer_size.size()); + output.write("PAR1", 4); + output.close(); + DORIS_CHECK(output.good()); +} + +void write_long_prefix_parquet_file(const std::string& file_path, size_t rows) { + std::vector ids(rows); + std::vector first(rows); + std::vector second(rows); + std::iota(ids.begin(), ids.end(), 0); + for (size_t row = 0; row < rows; ++row) { + first[row] = static_cast(row + 10); + second[row] = static_cast(row + 20); + } + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("first", arrow::int32(), false), + arrow::field("second", arrow::int32(), false)}); + auto table = arrow::Table::Make( + schema, {build_int32_array(ids), build_int32_array(first), build_int32_array(second)}); + write_table(file_path, table, rows, false, false, false); +} + void write_binary_minmax_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("text", arrow::utf8(), false), @@ -392,15 +748,12 @@ void write_page_index_parquet_file(const std::string& file_path) { write_table(file_path, table, ids.size(), false, true); } -void write_page_index_pair_parquet_file(const std::string& file_path) { - std::vector ids(128); +void write_multi_row_group_page_index_parquet_file(const std::string& file_path) { + std::vector ids(384); std::iota(ids.begin(), ids.end(), 0); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), false), - arrow::field("score", arrow::int32(), false), - }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(ids)}); - write_table(file_path, table, ids.size(), false, true); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + write_table(file_path, table, 128, false, true); } int64_t parquet_column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { @@ -448,24 +801,6 @@ void use_schema_order_positions(format::FileScanRequest* request, } } -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - auto schema_descriptor = reader.metadata()->schema(); - EXPECT_NE(schema_descriptor, nullptr); - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - return file_schema; -} - -int64_t count_range_rows(const std::vector& ranges) { - int64_t rows = 0; - for (const auto& range : ranges) { - rows += range.length; - } - return rows; -} - class ParquetScanTest : public testing::Test { protected: void SetUp() override { @@ -522,87 +857,48 @@ TEST(ParquetScanSelectionTest, CompactFilterShrinksCurrentSelection) { EXPECT_TRUE(selection.verify(selected_rows, 6).ok()); } -TEST_F(ParquetScanTest, PlanRowGroupsAppliesScanRangeBeforeStatistics) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 1); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 1); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 2); +TEST(ParquetScanAdaptivePredicateTest, OrdersByObservedCostPerRejectedRow) { + using format::parquet::detail::AdaptivePredicateStats; + std::unordered_map stats; + stats.emplace(0, AdaptivePredicateStats { + .cost_per_input_row_ns = 10, .survival_ratio = 0.8, .samples = 3}); + stats.emplace(1, AdaptivePredicateStats { + .cost_per_input_row_ns = 20, .survival_ratio = 0.1, .samples = 3}); + stats.emplace(2, AdaptivePredicateStats { + .cost_per_input_row_ns = 50, .survival_ratio = 0.5, .samples = 3}); + + const auto order = format::parquet::detail::order_adaptive_predicates({0, 1, 2}, stats); + EXPECT_EQ(order, std::vector({1, 0, 2})); + const auto prefetched = format::parquet::detail::adaptive_prefetch_prefix(order, stats, 0.25); + EXPECT_EQ(prefetched, std::vector({1})); } -TEST_F(ParquetScanTest, PlanRowGroupsPreservesFirstFileRowAcrossPrunedRowGroups) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.row_groups[0].first_file_row, 4); - EXPECT_EQ(plan.row_groups[0].row_group_rows, 2); - ASSERT_EQ(plan.row_groups[0].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].length, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 2); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 4); -} - -TEST_F(ParquetScanTest, PlanRowGroupsSelectsAllRowGroupsWithoutFilters) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); +TEST(ParquetScanAdaptivePredicateTest, SamplesWarmupThenAtLowFrequency) { + using format::parquet::detail::should_sample_adaptive_predicate; + for (size_t samples = 0; samples < 8; ++samples) { + EXPECT_TRUE(should_sample_adaptive_predicate(samples, samples)); + } + EXPECT_FALSE(should_sample_adaptive_predicate(8, 9)); + EXPECT_TRUE(should_sample_adaptive_predicate(8, 16)); + EXPECT_FALSE(should_sample_adaptive_predicate(9, 17)); + EXPECT_TRUE(should_sample_adaptive_predicate(9, 32)); +} - format::FileScanRequest request; - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); +TEST(ParquetScanSmallFileTest, StagesOnlyBoundedHttpObjects) { + using format::parquet::detail::should_stage_small_http_file; + EXPECT_TRUE(should_stage_small_http_file("http://host/tiny.parquet", 512, 1024)); + EXPECT_TRUE(should_stage_small_http_file("https://host/tiny.parquet", 1024, 1024)); + EXPECT_FALSE(should_stage_small_http_file("https://host/large.parquet", 1025, 1024)); + EXPECT_FALSE(should_stage_small_http_file("/tmp/tiny.parquet", 512, 1024)); + EXPECT_FALSE(should_stage_small_http_file("s3://bucket/tiny.parquet", 512, 1024)); +} - ASSERT_EQ(plan.row_groups.size(), 3); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 3); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 3); - for (size_t row_group_idx = 0; row_group_idx < plan.row_groups.size(); ++row_group_idx) { - EXPECT_EQ(plan.row_groups[row_group_idx].row_group_id, row_group_idx); - EXPECT_EQ(plan.row_groups[row_group_idx].first_file_row, - static_cast(row_group_idx * 2)); - ASSERT_EQ(plan.row_groups[row_group_idx].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].length, 2); - EXPECT_TRUE(plan.row_groups[row_group_idx].page_skip_plans.empty()); - } +TEST(ParquetScanSelectionTest, NativeLazySkipBitmapIsBounded) { + using format::parquet::detail::MAX_NATIVE_LAZY_SKIP_ROWS; + using format::parquet::detail::bounded_native_lazy_skip_rows; + EXPECT_EQ(bounded_native_lazy_skip_rows(1), 1); + EXPECT_EQ(bounded_native_lazy_skip_rows(MAX_NATIVE_LAZY_SKIP_ROWS + 1), + MAX_NATIVE_LAZY_SKIP_ROWS); } TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater) { @@ -612,7 +908,8 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater .first_file_row = ConditionCacheContext::GRANULE_SIZE, .row_group_rows = ConditionCacheContext::GRANULE_SIZE, .selected_ranges = {{.start = 0, .length = ConditionCacheContext::GRANULE_SIZE}}, - .page_skip_plans = {}}); + .page_skip_plans = {}, + .offset_indexes = {}}); format::parquet::ParquetScanScheduler scheduler; scheduler.set_plan(std::move(plan)); @@ -627,131 +924,6 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater EXPECT_EQ(ctx->base_granule, 0); } -TEST_F(ParquetScanTest, PageIndexIntersectsMultipleFiltersAndBuildsSkipPlan) { - write_page_index_pair_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest single_filter_request; - format::FileScanRequestBuilder single_filter_builder(&single_filter_request); - ASSERT_TRUE(single_filter_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - single_filter_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - format::parquet::RowGroupScanPlan single_filter_plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - single_filter_request, scan_range, false, &single_filter_plan) - .ok()); - ASSERT_EQ(single_filter_plan.row_groups.size(), 1); - const int64_t single_filter_rows = - count_range_rows(single_filter_plan.row_groups[0].selected_ranges); - - format::FileScanRequest intersect_request; - format::FileScanRequestBuilder intersect_builder(&intersect_request); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(1)).ok()); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(1, Int32ZoneMapExpr::Op::LT, 96)); - format::parquet::RowGroupScanPlan intersect_plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - intersect_request, scan_range, false, &intersect_plan) - .ok()); - ASSERT_EQ(intersect_plan.row_groups.size(), 1); - ASSERT_FALSE(intersect_plan.row_groups[0].selected_ranges.empty()); - const int64_t intersect_rows = count_range_rows(intersect_plan.row_groups[0].selected_ranges); - EXPECT_GT(single_filter_rows, intersect_rows); - EXPECT_GT(intersect_plan.row_groups[0].selected_ranges.front().start, 0); - const auto& last_range = intersect_plan.row_groups[0].selected_ranges.back(); - EXPECT_LT(last_range.start + last_range.length, 128); - EXPECT_GT(intersect_plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(intersect_plan.pruning_stats.selected_row_ranges, - intersect_plan.row_groups[0].selected_ranges.size()); - - auto id_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(id_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(id_skip_plan->second.leaf_column_id, 0); - EXPECT_FALSE(id_skip_plan->second.empty()); - auto score_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(1); - ASSERT_NE(score_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(score_skip_plan->second.leaf_column_id, 1); - EXPECT_FALSE(score_skip_plan->second.empty()); -} - -TEST_F(ParquetScanTest, PageIndexCanFullyFilterRowGroupAfterRangeIntersection) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::LT, 32)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 1); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 128); -} - -TEST_F(ParquetScanTest, PageIndexFullRangeWhenDisabledOrUnavailable) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); - - const bool old_enable_page_index = config::enable_parquet_page_index; - config::enable_parquet_page_index = false; - std::vector selected_ranges; - std::map page_skip_plans; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - parquet_file_reader.get(), file_schema, request, 0, 128, &selected_ranges, - &page_skip_plans, &pruning_stats) - .ok()); - config::enable_parquet_page_index = old_enable_page_index; - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 128); - EXPECT_TRUE(page_skip_plans.empty()); - EXPECT_EQ(pruning_stats.page_index_read_calls, 0); - - write_int_pair_parquet_file(_file_path, 6); - auto no_index_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto no_index_schema = build_file_schema(*no_index_reader); - format::FileScanRequest no_index_request; - no_index_request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - no_index_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 3)); - selected_ranges.clear(); - page_skip_plans.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - no_index_reader.get(), no_index_schema, no_index_request, 0, 6, - &selected_ranges, &page_skip_plans, &pruning_stats) - .ok()); - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 6); - EXPECT_TRUE(page_skip_plans.empty()); -} - TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); @@ -1081,6 +1253,37 @@ TEST_F(ParquetScanTest, GlobalRowIdUsesFileLocalPositionForScanRange) { EXPECT_EQ(row_ids, std::vector({2, 3})); } +TEST_F(ParquetScanTest, PredicateOnlyGlobalRowIdKeepsSignedFileLocalId) { + write_int_pair_parquet_file(_file_path, 6, false); + format::GlobalRowIdContext context {.version = 7, .backend_id = 123456789, .file_id = 42}; + auto reader = create_reader(0, -1, nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + const auto global_rowid = format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(global_rowid).ok()); + request->predicate_only_columns.push_back(global_rowid); + const auto global_rowid_position = request->local_positions.at(global_rowid); + request->conjuncts.push_back(create_always_true_single_column_conjunct( + cast_set(global_rowid_position.value()))); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(rows, 6); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {1, 2, 3, 4, 5, 6})); +} + TEST_F(ParquetScanTest, EmptyScanPlanReturnsEofWithoutReadingColumns) { write_int_pair_parquet_file(_file_path, 2); auto reader = create_reader(); @@ -1146,7 +1349,8 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { while (!eof) { Block block = build_file_block(schema); size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } @@ -1165,6 +1369,9 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); EXPECT_EQ(counter_value(profile, "SelectedRows"), 2); EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 4); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 1); + EXPECT_GT(counter_value(profile, "PredicateCompactionBytes"), 0); + ASSERT_NE(profile.get_counter("PredicateCompactionTime"), nullptr); EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 10); EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 4); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 2); @@ -1206,6 +1413,151 @@ TEST_F(ParquetScanTest, PredicateColumnsSkipUnreadColumnsWhenFirstPredicateFilte EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 6); } +TEST_F(ParquetScanTest, PredicateOnlyColumnDropsPayloadAfterFiltering) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, PredicateOnlyPlainComparisonUsesPhysicalDirectPath) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "PlainPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "PlainPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, PredicateOnlyUint32FallsBackBeforeRawPlainDecode) { + write_uint32_pair_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_BIGINT); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back( + create_int64_direct_greater_conjunct(0, std::numeric_limits::max())); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows, 2); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {20, 30})); + EXPECT_EQ(counter_value(profile, "PlainPredicateDirectBatches"), 0); +} + +TEST_F(ParquetScanTest, PlainPredicateReportsFooterEncodingMismatchAfterProgress) { + write_misdeclared_two_page_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("encoding"), std::string::npos) << status; +} + +TEST_F(ParquetScanTest, PlainDirectPathKeepsPayloadForMultiColumnResidual) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + request->conjuncts.push_back(create_int32_pair_sum_conjunct(0, 1, 42)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + // A residual expression still consumes id, so raw filtering must leave its payload available. + EXPECT_EQ(counter_value(profile, "PlainPredicateDirectBatches"), 0); +} + // Scenario: every physical batch in every row group is rejected. Predicate readers reach each row // group boundary, while the lazy score reader remains at row 0. The boundary reset must discard that // reader and its pending lag instead of issuing SkipRecords for values that can never be observed. @@ -1236,10 +1588,12 @@ TEST_F(ParquetScanTest, FullyFilteredRowGroupsDropPendingLazyReaders) { } EXPECT_EQ(total_rows, 0); - EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 6); + // The first one-row probe grows after rejection and remains grown across consecutive empty + // row groups, so later two-row groups are consumed in one predicate batch each. + EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 4); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); - ASSERT_NE(profile.get_counter("ArrowSkipRecordsTime"), nullptr); - EXPECT_EQ(profile.get_counter("ArrowSkipRecordsTime")->value(), 0); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); } // Scenario: row group 0 is fully filtered and leaves two pending lazy rows. Reset must discard that @@ -1281,6 +1635,37 @@ TEST_F(ParquetScanTest, PendingLazySkipDoesNotCrossRowGroupReset) { EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); } +TEST_F(ParquetScanTest, LongFilteredPrefixSkipsMultipleLazyColumnsInBoundedChunks) { + constexpr size_t ROWS = 70000; + write_long_prefix_parquet_file(_file_path, ROWS); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(32); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(2)).ok()); + request->conjuncts.push_back( + create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, ROWS - 1)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_element(0), ROWS - 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_element(0), ROWS + 9); + EXPECT_EQ(int32_data_column(*block.get_by_position(2).column).get_element(0), ROWS + 19); + EXPECT_LT(counter_value(profile, "TotalBatches"), 32); +} + // Scenario: a nested lazy column stays behind while id=1 is rejected. Flushing skip(1) must consume // the complete repetition/definition-level span for the first list, then materialize the remaining // two parent rows without corrupting their child boundaries. @@ -1415,5 +1800,30 @@ TEST_F(ParquetScanTest, ProfileCountersReflectPageIndexAndRangeGapPruning) { EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); } +TEST_F(ParquetScanTest, OpenDefersPageIndexProbeToCurrentRowGroup) { + write_multi_row_group_page_index_parquet_file(_file_path); + RuntimeProfile profile("lazy_page_index_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, -1)); + ASSERT_TRUE(reader->open(request).ok()); + + // LIMIT can stop after the first block, so open must not pay remote index I/O for later groups. + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 128); + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 1); +} + } // namespace } // namespace doris diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 9f78735bfb560a..436674bda035f5 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -16,8 +16,8 @@ // under the License. #include -#include +#include #include #include @@ -27,509 +27,382 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" namespace doris::format::parquet { -namespace { - -std::vector> build_fields( - const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - std::vector> fields; - EXPECT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - return fields; -} +TEST(ParquetSchemaTest, NativeMetadataTreePreservesNestedFieldNamesAndIds) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement protocol; + protocol.__set_name("protocol"); + protocol.__set_num_children(2); + protocol.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + protocol.__set_field_id(10); + + tparquet::SchemaElement min_reader; + min_reader.__set_name("minReaderVersion"); + min_reader.__set_type(tparquet::Type::INT32); + min_reader.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + min_reader.__set_field_id(11); + + tparquet::SchemaElement min_writer = min_reader; + min_writer.__set_name("minWriterVersion"); + min_writer.__set_field_id(12); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, protocol, min_reader, min_writer}).ok()); + native_schema.assign_ids(); -Status build_status(const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); std::vector> fields; - return build_parquet_column_schema(descriptor, &fields); -} - -} // namespace - -TEST(ParquetSchemaTest, PrimitiveStateAndFieldIdArePreserved) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("required_i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("optional_i64", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT64, - ::parquet::ConvertedType::NONE, -1, -1, -1, 42), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->local_id, 0); - EXPECT_EQ(fields[0]->name, "required_i32"); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->leaf_column_id, 0); - EXPECT_EQ(fields[0]->nullable_definition_level, 0); - EXPECT_FALSE(fields[0]->type->is_nullable()); - - EXPECT_EQ(fields[1]->local_id, 1); - EXPECT_EQ(fields[1]->parquet_field_id, 42); - EXPECT_EQ(fields[1]->leaf_column_id, 1); - EXPECT_EQ(fields[1]->nullable_definition_level, 1); - EXPECT_TRUE(fields[1]->type->is_nullable()); -} - -TEST(ParquetSchemaTest, PrimitiveTypeDescriptorCoversLogicalConvertedAndPhysicalFallback) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(false, - ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64), - ::parquet::schema::PrimitiveNode::Make("i8", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8), - ::parquet::schema::PrimitiveNode::Make("plain", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - }); - - ASSERT_EQ(fields.size(), 3); - EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(fields[0]->type_descriptor.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - EXPECT_TRUE(fields[0]->type_descriptor.is_timestamp); - EXPECT_FALSE(fields[0]->type_descriptor.timestamp_is_adjusted_to_utc); - - EXPECT_EQ(remove_nullable(fields[1]->type)->get_primitive_type(), TYPE_TINYINT); - EXPECT_EQ(fields[1]->type_descriptor.integer_bit_width, 8); - EXPECT_FALSE(fields[1]->type_descriptor.is_unsigned_integer); - - EXPECT_EQ(remove_nullable(fields[2]->type)->get_primitive_type(), TYPE_DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.physical_type, ::parquet::Type::DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::NONE); -} - -TEST(ParquetSchemaTest, StructMakesDataTypeChildrenNullableAndPropagatesLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::PrimitiveNode::Make("a", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - })}); - + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); ASSERT_EQ(fields.size(), 1); - const auto& struct_schema = *fields[0]; - EXPECT_EQ(struct_schema.kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(struct_schema.nullable_definition_level, 1); - ASSERT_EQ(struct_schema.children.size(), 2); - EXPECT_EQ(struct_schema.children[0]->definition_level, 1); - EXPECT_EQ(struct_schema.children[1]->definition_level, 2); - EXPECT_EQ(struct_schema.max_definition_level, 2); - - const auto& struct_type = - assert_cast(*remove_nullable(struct_schema.type)); - ASSERT_EQ(struct_type.get_elements().size(), 2); - EXPECT_TRUE(struct_type.get_elements()[0]->is_nullable()); - EXPECT_TRUE(struct_type.get_elements()[1]->is_nullable()); -} - -TEST(ParquetSchemaTest, ListCompatibilityRulesAndLevels) { - const auto standard_list = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - const auto structural_array = ::parquet::schema::GroupNode::Make( - "ys", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "array", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({standard_list, structural_array}); - ASSERT_EQ(fields.size(), 2); - - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(xs.definition_level, 2); - EXPECT_EQ(xs.repetition_level, 1); - ASSERT_EQ(xs.children.size(), 1); - EXPECT_EQ(xs.children[0]->name, "element"); - EXPECT_EQ(xs.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_TRUE(xs.children[0]->type->is_nullable()); - const auto& xs_type = assert_cast(*remove_nullable(xs.type)); - EXPECT_TRUE(xs_type.get_nested_type()->is_nullable()); - - const auto& ys = *fields[1]; - EXPECT_EQ(ys.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(ys.children.size(), 1); - EXPECT_EQ(ys.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(remove_nullable(ys.children[0]->type)->get_primitive_type(), TYPE_STRUCT); -} - -TEST(ParquetSchemaTest, LegacyListElementResolutionRulesArePreserved) { - const auto two_level_list = ::parquet::schema::GroupNode::Make( - "two_level", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - const auto tuple_list = ::parquet::schema::GroupNode::Make( - "tuple_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "tuple_list_tuple", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - const auto multi_field_list = ::parquet::schema::GroupNode::Make( - "records", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("name", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::LIST); - const auto fields = build_fields({two_level_list, tuple_list, multi_field_list}); - ASSERT_EQ(fields.size(), 3); - - const auto& two_level = *fields[0]; - EXPECT_EQ(two_level.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(two_level.definition_level, 2); - EXPECT_EQ(two_level.repetition_level, 1); - ASSERT_EQ(two_level.children.size(), 1); - EXPECT_EQ(two_level.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(two_level.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(two_level.children[0]->type)->get_primitive_type(), TYPE_INT); - - const auto& tuple = *fields[1]; - ASSERT_EQ(tuple.children.size(), 1); - EXPECT_EQ(tuple.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(tuple.children[0]->name, "element"); - ASSERT_EQ(tuple.children[0]->children.size(), 1); - EXPECT_EQ(tuple.children[0]->children[0]->name, "value"); - - const auto& multi_field = *fields[2]; - ASSERT_EQ(multi_field.children.size(), 1); - EXPECT_EQ(multi_field.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(multi_field.children[0]->children.size(), 2); - EXPECT_EQ(multi_field.children[0]->children[0]->name, "id"); - EXPECT_EQ(multi_field.children[0]->children[1]->name, "name"); + EXPECT_EQ(fields[0]->name, "protocol"); + EXPECT_EQ(fields[0]->parquet_field_id, 10); + ASSERT_EQ(fields[0]->children.size(), 2); + EXPECT_EQ(fields[0]->children[0]->name, "minReaderVersion"); + EXPECT_EQ(fields[0]->children[0]->leaf_column_id, 0); + EXPECT_EQ(fields[0]->children[1]->name, "minWriterVersion"); + EXPECT_EQ(fields[0]->children[1]->leaf_column_id, 1); + + std::shared_ptr mapping; + ASSERT_TRUE(build_native_schema_node(fields[0]->type, *fields[0], &mapping).ok()); + EXPECT_TRUE(mapping->has_child("minReaderVersion")); + EXPECT_EQ(mapping->file_child_name("minReaderVersion"), "minReaderVersion"); + ASSERT_NE(mapping->child("minReaderVersion"), nullptr); } - -TEST(ParquetSchemaTest, NestedRepeatedInsideListElementIsWrappedOnce) { - const auto list_with_repeated_child = ::parquet::schema::GroupNode::Make( - "outer", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({list_with_repeated_child}); +TEST(ParquetSchemaTest, NativeLogicalUtcTimeIsDeferredToProjectionValidation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement adjusted_time; + adjusted_time.__set_name("time_ms"); + adjusted_time.__set_type(tparquet::Type::INT32); + adjusted_time.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + adjusted_time.__set_logicalType(tparquet::LogicalType()); + adjusted_time.logicalType.__set_TIME(tparquet::TimeType()); + adjusted_time.logicalType.TIME.__set_isAdjustedToUTC(true); + adjusted_time.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + adjusted_time.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, adjusted_time}).ok()); + native_schema.assign_ids(); + std::vector> fields; + const auto status = build_parquet_column_schema(native_schema, &fields); + ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(fields.size(), 1); - const auto& outer = *fields[0]; - EXPECT_EQ(outer.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(outer.children.size(), 1); - const auto& element = *outer.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.children[0]->name, "items"); - ASSERT_EQ(element.children[0]->children.size(), 1); - EXPECT_EQ(element.children[0]->children[0]->name, "element"); + // Preserve the unsupported marker in metadata; request-level validation decides whether the + // leaf is a real projection or an ignorable COUNT(*) placeholder. + EXPECT_EQ(fields[0]->type_descriptor.unsupported_reason, + "Parquet TIME with isAdjustedToUTC=true is not supported"); } -TEST(ParquetSchemaTest, ListWrapperWithLogicalAnnotationIsPreservedAsElement) { - const auto annotated_repeated_group = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - EXPECT_FALSE(build_status({annotated_repeated_group}).ok()); - - const auto nested_list_wrapper = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({nested_list_wrapper}); - ASSERT_EQ(fields.size(), 1); - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(xs.children.size(), 1); - const auto& element = *xs.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.name, "element"); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(element.children[0]->type)->get_primitive_type(), TYPE_INT); +TEST(ParquetSchemaTest, NativeGroupEnumLogicalTypeIsRejected) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement enum_group; + enum_group.__set_name("bad_enum_group"); + enum_group.__set_num_children(1); + enum_group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + enum_group.__set_logicalType(tparquet::LogicalType()); + enum_group.logicalType.__set_ENUM(tparquet::EnumType()); + + tparquet::SchemaElement child; + child.__set_name("value"); + child.__set_type(tparquet::Type::BYTE_ARRAY); + child.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + NativeFieldDescriptor native_schema; + const auto status = native_schema.parse_from_thrift({root, enum_group, child}); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Logical type Enum cannot be applied to group node"), + std::string::npos); } -TEST(ParquetSchemaTest, MapWrapperIsFoldedAndOptionalKeyIsAllowed) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); - - ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_EQ(map_schema.kind, ParquetColumnSchemaKind::MAP); - EXPECT_EQ(map_schema.definition_level, 2); - EXPECT_EQ(map_schema.repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->name, "key"); - EXPECT_EQ(map_schema.children[1]->name, "value"); - EXPECT_TRUE(map_schema.children[0]->type->is_nullable()); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); +TEST(ParquetSchemaTest, NativeStringAnnotationsAndTimeUnitsPreserveLogicalTypes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(8); + + auto binary_leaf = [](const std::string& name) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return leaf; + }; + auto logical_enum = binary_leaf("logical_enum"); + logical_enum.__set_logicalType(tparquet::LogicalType()); + logical_enum.logicalType.__set_ENUM(tparquet::EnumType()); + auto logical_bson = binary_leaf("logical_bson"); + logical_bson.__set_logicalType(tparquet::LogicalType()); + logical_bson.logicalType.__set_BSON(tparquet::BsonType()); + auto converted_enum = binary_leaf("converted_enum"); + converted_enum.__set_converted_type(tparquet::ConvertedType::ENUM); + auto converted_bson = binary_leaf("converted_bson"); + converted_bson.__set_converted_type(tparquet::ConvertedType::BSON); + + auto logical_time = [](const std::string& name, bool millis) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(millis ? tparquet::Type::INT32 : tparquet::Type::INT64); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + leaf.__set_logicalType(tparquet::LogicalType()); + leaf.logicalType.__set_TIME(tparquet::TimeType()); + leaf.logicalType.TIME.__set_isAdjustedToUTC(false); + leaf.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + if (millis) { + leaf.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + } else { + leaf.logicalType.TIME.unit.__set_MICROS(tparquet::MicroSeconds()); + } + return leaf; + }; + auto logical_millis = logical_time("logical_millis", true); + auto logical_micros = logical_time("logical_micros", false); + auto converted_millis = logical_time("converted_millis", true); + converted_millis.__isset.logicalType = false; + converted_millis.__set_converted_type(tparquet::ConvertedType::TIME_MILLIS); + auto converted_micros = logical_time("converted_micros", false); + converted_micros.__isset.logicalType = false; + converted_micros.__set_converted_type(tparquet::ConvertedType::TIME_MICROS); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor + .parse_from_thrift({root, logical_enum, logical_bson, converted_enum, + converted_bson, logical_millis, logical_micros, + converted_millis, converted_micros}) + .ok()); + for (size_t field = 0; field < 4; ++field) { + EXPECT_EQ(remove_nullable(descriptor.get_column(field)->data_type)->get_primitive_type(), + TYPE_STRING); + } + EXPECT_EQ(remove_nullable(descriptor.get_column(4)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(5)->data_type)->get_scale(), 6); + EXPECT_EQ(remove_nullable(descriptor.get_column(6)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(7)->data_type)->get_scale(), 6); } -TEST(ParquetSchemaTest, StandardMapLevelsAndDataTypesAreBuiltFromEntryContext) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::REQUIRED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); - - ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_FALSE(map_schema.type->is_nullable()); - EXPECT_EQ(map_schema.definition_level, 1); - EXPECT_EQ(map_schema.repetition_level, 1); - EXPECT_EQ(map_schema.repeated_repetition_level, 1); - EXPECT_EQ(map_schema.max_definition_level, 2); - EXPECT_EQ(map_schema.max_repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->definition_level, 1); - EXPECT_EQ(map_schema.children[0]->repetition_level, 1); - EXPECT_EQ(map_schema.children[1]->definition_level, 2); - EXPECT_EQ(map_schema.children[1]->nullable_definition_level, 2); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); +TEST(ParquetSchemaTest, NativeSchemaRejectsAmbiguousKindsAndMissingRepetition) { + auto valid_root = []() { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + return root; + }; + auto valid_leaf = []() { + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + return leaf; + }; + + NativeFieldDescriptor descriptor; + auto primitive_root = valid_root(); + primitive_root.__set_type(tparquet::Type::INT32); + EXPECT_FALSE(descriptor.parse_from_thrift({primitive_root, valid_leaf()}).ok()); + + auto repeated_root = valid_root(); + repeated_root.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + EXPECT_FALSE(descriptor.parse_from_thrift({repeated_root, valid_leaf()}).ok()); + + auto missing_repetition = valid_leaf(); + missing_repetition.__isset.repetition_type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_repetition}).ok()); + + auto dual_kind = valid_leaf(); + dual_kind.__set_num_children(1); + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), dual_kind}).ok()); + + auto legacy_zero_children = valid_leaf(); + legacy_zero_children.__set_num_children(0); + EXPECT_TRUE(descriptor.parse_from_thrift({valid_root(), legacy_zero_children}).ok()); + + auto missing_kind = valid_leaf(); + missing_kind.__isset.type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_kind}).ok()); } -TEST(ParquetSchemaTest, BareRepeatedFieldsAreWrappedAsLists) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("items", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32), - ::parquet::schema::GroupNode::Make( - "links", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("url", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("rank", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[0]->children.size(), 1); - EXPECT_EQ(fields[0]->children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->children[0]->name, "element"); - - EXPECT_EQ(fields[1]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[1]->children.size(), 1); - EXPECT_EQ(fields[1]->children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(fields[1]->children[0]->name, "element"); +TEST(ParquetSchemaTest, NativeSchemaRejectsUnboundedChildCountsBeforeAllocation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(std::numeric_limits::max()); + NativeFieldDescriptor descriptor; + EXPECT_FALSE(descriptor.parse_from_thrift({root}).ok()); + + root.__set_num_children(1); + tparquet::SchemaElement nested; + nested.__set_name("nested"); + nested.__set_num_children(std::numeric_limits::max()); + nested.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + EXPECT_FALSE(descriptor.parse_from_thrift({root, nested}).ok()); } -TEST(ParquetSchemaTest, DeepLevelChainPropagatesDefinitionAndRepetitionLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "inner", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})})}); - - ASSERT_EQ(fields.size(), 1); - const auto& s = *fields[0]; - EXPECT_EQ(s.definition_level, 1); - EXPECT_EQ(s.nullable_definition_level, 1); - ASSERT_EQ(s.children.size(), 1); - const auto& inner = *s.children[0]; - EXPECT_EQ(inner.definition_level, 2); - EXPECT_EQ(inner.nullable_definition_level, 2); - ASSERT_EQ(inner.children.size(), 1); - const auto& items = *inner.children[0]; - EXPECT_EQ(items.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(items.definition_level, 3); - EXPECT_EQ(items.repetition_level, 1); - EXPECT_EQ(items.repeated_ancestor_definition_level, 3); - EXPECT_EQ(items.repeated_repetition_level, 1); - EXPECT_EQ(items.max_definition_level, 3); - EXPECT_EQ(items.max_repetition_level, 1); - ASSERT_EQ(items.children.size(), 1); - EXPECT_EQ(items.children[0]->definition_level, 3); - EXPECT_EQ(items.children[0]->repetition_level, 1); +std::vector nested_native_schema(size_t depth) { + std::vector schema; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + schema.push_back(root); + for (size_t level = 0; level < depth; ++level) { + tparquet::SchemaElement group; + group.__set_name("g" + std::to_string(level)); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(group); + } + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(leaf); + return schema; } -TEST(ParquetSchemaTest, BuildEntryValidatesNullPointerAndEmptyRoot) { - auto empty_root = ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, - ::parquet::schema::NodeVector {}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(empty_root); - - EXPECT_FALSE(build_parquet_column_schema(descriptor, nullptr).ok()); - - std::vector> fields; - ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - EXPECT_TRUE(fields.empty()); +TEST(ParquetSchemaTest, NativeSchemaBoundsRecursiveDepth) { + NativeFieldDescriptor accepted; + EXPECT_TRUE(accepted.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH)).ok()); + NativeFieldDescriptor rejected; + EXPECT_FALSE( + rejected.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH + 1)).ok()); } -TEST(ParquetSchemaTest, RejectInvalidListMapAndPreserveUnsupportedTime) { - const auto bad_list = ::parquet::schema::GroupNode::Make( - "bad_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({bad_list}).ok()); - - const auto bad_map = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("entry", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({bad_map}).ok()); - - const auto converted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS); - const auto fields = build_fields({converted_time}); - ASSERT_EQ(fields.size(), 1); - EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_INT); - EXPECT_NE(fields[0]->type_descriptor.unsupported_reason.find( - "Parquet TIME with isAdjustedToUTC=true is not supported"), - std::string::npos); +TEST(ParquetSchemaTest, NativeListTupleCompatibilityRequiresEnclosingListName) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto list = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + schema.__set_converted_type(tparquet::ConvertedType::LIST); + return schema; + }; + auto wrapper = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + return schema; + }; + auto item = []() { + tparquet::SchemaElement schema; + schema.__set_name("item"); + schema.__set_type(tparquet::Type::INT32); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return schema; + }; + + NativeFieldDescriptor mismatched; + ASSERT_TRUE(mismatched.parse_from_thrift({root(), list("xs"), wrapper("other_tuple"), item()}) + .ok()); + const auto* mismatched_element = &mismatched.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(mismatched_element->data_type)->get_primitive_type(), TYPE_INT); + + NativeFieldDescriptor matching; + ASSERT_TRUE(matching.parse_from_thrift({root(), list("xs"), wrapper("xs_tuple"), item()}).ok()); + const auto* matching_element = &matching.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(matching_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(matching_element->children.size(), 1); + EXPECT_EQ(matching_element->children[0].name, "item"); } -TEST(ParquetSchemaTest, RejectAdditionalInvalidListAndMapLayouts) { - const auto zero_child_list = ::parquet::schema::GroupNode::Make( - "zero_child_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make("list", ::parquet::Repetition::REPEATED, - ::parquet::schema::NodeVector {})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({zero_child_list}).ok()); - - const auto repeated_list = ::parquet::schema::GroupNode::Make( - "repeated_list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({repeated_list}).ok()); - - const auto map_with_two_fields = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::GroupNode::Make( - "entry1", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - ::parquet::schema::GroupNode::Make( - "entry2", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_with_two_fields}).ok()); - - const auto non_repeated_map_entry = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({non_repeated_map_entry}).ok()); - - const auto map_entry_with_one_child = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_entry_with_one_child}).ok()); - - const auto repeated_map = ::parquet::schema::GroupNode::Make( - "repeated_map", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({repeated_map}).ok()); +TEST(ParquetSchemaTest, NativeListPreservesRepeatedAndAnnotatedElementWrappers) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto group = [](const std::string& name, tparquet::FieldRepetitionType::type repetition) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(repetition); + return schema; + }; + auto outer_list = group("outer", tparquet::FieldRepetitionType::OPTIONAL); + outer_list.__set_converted_type(tparquet::ConvertedType::LIST); + + auto repeated_wrapper = group("list", tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement repeated_items; + repeated_items.__set_name("items"); + repeated_items.__set_type(tparquet::Type::INT32); + repeated_items.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + NativeFieldDescriptor repeated; + ASSERT_TRUE(repeated.parse_from_thrift({root(), outer_list, repeated_wrapper, repeated_items}) + .ok()); + const auto* repeated_element = &repeated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(repeated_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(repeated_element->children.size(), 1); + EXPECT_EQ(repeated_element->children[0].name, "items"); + EXPECT_EQ(remove_nullable(repeated_element->children[0].data_type)->get_primitive_type(), + TYPE_ARRAY); + + auto annotated_wrapper = repeated_wrapper; + annotated_wrapper.__set_converted_type(tparquet::ConvertedType::LIST); + auto nested_wrapper = repeated_wrapper; + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::INT32); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + NativeFieldDescriptor annotated; + ASSERT_TRUE(annotated + .parse_from_thrift( + {root(), outer_list, annotated_wrapper, nested_wrapper, value}) + .ok()); + const auto* annotated_element = &annotated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(annotated_element->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(annotated_element->children.size(), 1); + EXPECT_EQ(remove_nullable(annotated_element->children[0].data_type)->get_primitive_type(), + TYPE_INT); } -TEST(ParquetSchemaTest, LogicalUtcTimeIsPreservedForProjection) { - const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32); - const auto supported_value = ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64); - const auto row = ::parquet::schema::GroupNode::Make("row", ::parquet::Repetition::OPTIONAL, - {adjusted_time, supported_value}); - const auto fields = build_fields({row}); - ASSERT_EQ(fields.size(), 1); - ASSERT_EQ(fields[0]->children.size(), 2); - EXPECT_EQ(remove_nullable(fields[0]->children[0]->type)->get_primitive_type(), TYPE_INT); - EXPECT_FALSE(fields[0]->children[0]->type_descriptor.unsupported_reason.empty()); - EXPECT_EQ(remove_nullable(fields[0]->children[1]->type)->get_primitive_type(), TYPE_BIGINT); +TEST(ParquetSchemaTest, NativeMetadataRejectsRowGroupChunkCardinalityAndMissingMetadata) { + auto make_metadata = []() { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(2); + tparquet::SchemaElement first; + first.__set_name("a"); + first.__set_type(tparquet::Type::INT32); + first.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement second = first; + second.__set_name("b"); + metadata.__set_schema({root, first, second}); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + return metadata; + }; + + { + NativeParquetMetadata metadata(make_metadata(), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } + { + auto thrift = make_metadata(); + thrift.row_groups[0].columns.resize(2); + tparquet::ColumnMetaData first_meta; + first_meta.__set_type(tparquet::Type::INT32); + thrift.row_groups[0].columns[0].__set_meta_data(first_meta); + NativeParquetMetadata metadata(std::move(thrift), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } } } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp b/be/test/format_v2/parquet/parquet_serde_reader_test.cpp deleted file mode 100644 index c35138e3263723..00000000000000 --- a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp +++ /dev/null @@ -1,459 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_decimal.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_nullable.h" -#include "core/types.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -class ParquetSerdeReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_serde_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "serde.parquet").string(); - write_parquet_file(); - open_file(_file_path); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(0x0000).ok()); - EXPECT_TRUE(builder.Append(0x8000).ok()); - EXPECT_TRUE(builder.Append(0x3E00).ok()); - EXPECT_TRUE(builder.Append(0x7E00).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_binary_array(const std::vector& values) { - arrow::BinaryBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data()), - static_cast(value.size())) - .ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_fixed_binary_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_timestamp_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_decimal_array(const std::shared_ptr& type, - const std::vector& values) { - arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); - } - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, - std::shared_ptr array) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - } - - void write_table(const std::string& file_path, const std::shared_ptr& table, - std::shared_ptr<::parquet::ArrowWriterProperties> arrow_properties = nullptr) { - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - ::parquet::WriterProperties::Builder writer_builder; - writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - writer_builder.compression(::parquet::Compression::UNCOMPRESSED); - if (arrow_properties == nullptr) { - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_properties = arrow_builder.build(); - } - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), *file_result, ROW_COUNT, - writer_builder.build(), std::move(arrow_properties))); - } - - void write_parquet_file() { - add_field(arrow::field("bool_col", arrow::boolean(), false), - build_required_array( - {true, false, true, false, true})); - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50})); - add_field(arrow::field("int64_col", arrow::int64(), false), - build_required_array( - {10000000000L, -9L, 42L, 77L, 123L})); - add_field(arrow::field("uint32_col", arrow::uint32(), false), - build_required_array( - {0U, 1U, 1U << 31, std::numeric_limits::max(), 42U})); - add_field(arrow::field("uint64_col", arrow::uint64(), false), - build_required_array( - {0ULL, 1ULL, 1ULL << 63, std::numeric_limits::max(), 42ULL})); - add_field(arrow::field("float_col", arrow::float32(), false), - build_required_array( - {1.5F, -2.25F, 3.0F, 4.5F, 5.75F})); - add_field(arrow::field("double_col", arrow::float64(), false), - build_required_array({3.5, -4.75, 6.0, 7.25, 8.5})); - add_field(arrow::field("nullable_float16_col", arrow::float16(), true), - build_nullable_float16_array()); - add_field(arrow::field("binary_col", arrow::binary(), false), - build_binary_array({"bin_a", "bin_b", "bin_c", "bin_d", "bin_e"})); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"})); - add_field(arrow::field("fixed_binary_col", arrow::fixed_size_binary(4), false), - build_fixed_binary_array(arrow::fixed_size_binary(4), - {"aaaa", "bbbb", "cccc", "dddd", "eeee"})); - add_field(arrow::field("date_col", arrow::date32(), false), - build_required_array({0, 1, 18628, 18629, 18630})); - add_field(arrow::field("timestamp_millis_col", arrow::timestamp(arrow::TimeUnit::MILLI), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MILLI), - {0, 1234, 1609459200000, 1609459201000, -1})); - add_field(arrow::field("timestamp_micros_col", arrow::timestamp(arrow::TimeUnit::MICRO), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("timestamp_micros_utc_col", - arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("decimal_fixed_binary_9_2_col", arrow::decimal128(9, 2), false), - build_decimal_array(arrow::decimal128(9, 2), {12345, -67, 0, 987, 1000})); - add_field(arrow::field("decimal_fixed_binary_18_6_col", arrow::decimal128(18, 6), false), - build_decimal_array(arrow::decimal128(18, 6), - {1234567, -670000, 0, 9870000, 1000000})); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array()); - - write_table(_file_path, arrow::Table::Make(arrow::schema(_arrow_fields), _arrays)); - } - - void open_file(const std::string& file_path) { - _file_reader = ::parquet::ParquetFileReader::OpenFile(file_path, false); - ASSERT_NE(_file_reader, nullptr); - ASSERT_EQ(_file_reader->metadata()->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet serde test field " << name; - return _fields.size(); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - template - void read_and_validate(const std::string& name, Validator validator) const { - const auto field_idx = find_field_idx(name); - ASSERT_TRUE(supports_record_reader(_fields[field_idx]->type_descriptor)); - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - validator(*_fields[field_idx], *column); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; -}; - -TEST_F(ParquetSerdeReaderTest, ReadAllSupportedPhysicalAndLogicalTypes) { - read_and_validate("bool_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BOOLEAN); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 0); - EXPECT_EQ(values.get_element(4), 1); - }); - read_and_validate("int32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - read_and_validate("int64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10000000000L); - EXPECT_EQ(values.get_element(1), -9L); - }); - read_and_validate("uint32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_BIGINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), 2147483648L); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("uint64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_LARGEINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), static_cast(1) << 63); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("float_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FLOAT); - const auto& values = assert_cast(column); - EXPECT_FLOAT_EQ(values.get_element(0), 1.5F); - EXPECT_FLOAT_EQ(values.get_element(1), -2.25F); - }); - read_and_validate("double_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::DOUBLE); - const auto& values = assert_cast(column); - EXPECT_DOUBLE_EQ(values.get_element(0), 3.5); - EXPECT_DOUBLE_EQ(values.get_element(1), -4.75); - }); - read_and_validate("nullable_float16_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 2); - EXPECT_EQ(schema.type_descriptor.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(schema.type_descriptor.is_string_like); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_FLOAT); - const auto& nullable_column = assert_cast(column); - const auto& values = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FLOAT_EQ(values.get_element(1), 0.0F); - EXPECT_FALSE(std::signbit(values.get_element(1))); - EXPECT_FLOAT_EQ(values.get_element(2), -0.0F); - EXPECT_TRUE(std::signbit(values.get_element(2))); - EXPECT_FLOAT_EQ(values.get_element(3), 1.5F); - EXPECT_TRUE(std::isnan(values.get_element(4))); - }); - read_and_validate("binary_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BYTE_ARRAY); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "bin_a"); - EXPECT_EQ(values.get_data_at(3).to_string(), "bin_d"); - }); - read_and_validate("string_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - read_and_validate("fixed_binary_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 4); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "aaaa"); - EXPECT_EQ(values.get_data_at(2).to_string(), "cccc"); - }); - read_and_validate("date_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATEV2); - EXPECT_EQ(schema.type->to_string(column, 0), "1970-01-01"); - EXPECT_EQ(schema.type->to_string(column, 2), "2021-01-01"); - }); - read_and_validate( - "timestamp_millis_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999"); - }); - read_and_validate( - "timestamp_micros_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("timestamp_micros_utc_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.timestamp_is_adjusted_to_utc); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("decimal_fixed_binary_9_2_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal32(12345)); - EXPECT_EQ(schema.type->to_string(column, 0), "123.45"); - }); - read_and_validate("decimal_fixed_binary_18_6_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal64(1234567)); - EXPECT_EQ(schema.type->to_string(column, 0), "1.234567"); - }); - read_and_validate( - "nullable_int_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); -} - -TEST_F(ParquetSerdeReaderTest, ReadInt96TimestampAsDateTimeV2) { - const auto file_path = (_test_dir / "int96_timestamp.parquet").string(); - auto field = arrow::field("col_datetime", arrow::timestamp(arrow::TimeUnit::MICRO), false); - auto array = build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1}); - auto table = arrow::Table::Make(arrow::schema({field}), {array}); - - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_builder.enable_force_write_int96_timestamps(); - _fields.clear(); - _file_reader.reset(); - _row_group.reset(); - write_table(file_path, table, arrow_builder.build()); - open_file(file_path); - - ASSERT_EQ(_fields.size(), 1); - EXPECT_EQ(_fields[0]->type_descriptor.physical_type, ::parquet::Type::INT96); - EXPECT_EQ(_fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - ASSERT_TRUE(supports_record_reader(_fields[0]->type_descriptor)); - ASSERT_EQ(remove_nullable(_fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - - auto reader = create_reader(0); - ASSERT_NE(reader, nullptr); - auto column = _fields[0]->type->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(ROW_COUNT, column, &rows_read).ok()); - ASSERT_EQ(rows_read, ROW_COUNT); - EXPECT_EQ(_fields[0]->type->to_string(*column, 0), "1970-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 2), "2021-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 4), "1969-12-31 23:59:59.999999"); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index dd2138279b11d1..7ba9a3cde19746 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -17,29 +17,22 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include #include -#include -#include -#include -#include -#include #include #include #include +#include #include #include #include #include +#include #include #include #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_timestamptz.h" #include "core/field.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr.h" @@ -47,293 +40,125 @@ #include "exprs/vslot_ref.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "storage/index/bloom_filter/block_split_bloom_filter.h" -#include "storage/index/zone_map/zonemap_eval_context.h" -#include "storage/index/zone_map/zonemap_filter_result.h" - +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" +#include "io/fs/file_reader.h" +#include "util/thrift_util.h" namespace doris { namespace { -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -std::shared_ptr int32_array(const std::vector>& values) { - arrow::Int32Builder builder; - for (const auto& value : values) { - if (value.has_value()) { - EXPECT_TRUE(builder.Append(*value).ok()); - } else { - EXPECT_TRUE(builder.AppendNull().ok()); - } - } - return finish_array(&builder); -} - -std::shared_ptr uint32_array(const std::vector& values) { - arrow::UInt32Builder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr float_array(const std::vector& values) { - arrow::FloatBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr double_array(const std::vector& values) { - arrow::DoubleBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -template -std::string encoded_value(const NativeType& value) { - return {reinterpret_cast(&value), sizeof(value)}; -} - -std::shared_ptr string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr timestamp_array(const std::vector& values) { - arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::unique_ptr<::parquet::ParquetFileReader> make_reader( - const std::shared_ptr& table, int64_t row_group_size, bool enable_dictionary, - bool enable_statistics) { - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.compression(::parquet::Compression::UNCOMPRESSED); - if (enable_dictionary) { - builder.enable_dictionary(); - } else { - builder.disable_dictionary(); - } - if (!enable_statistics) { - builder.disable_statistics(); - } - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - row_group_size, builder.build()) - .ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - return ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); -} - -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*reader.metadata()->schema(), &file_schema) - .ok()); - return file_schema; -} - -template -class TestColumnIndex final : public ::parquet::TypedColumnIndex { +class StatisticsMemoryFileReader final : public io::FileReader { public: - using NativeType = typename ParquetDType::c_type; - - TestColumnIndex(NativeType min_value, NativeType max_value) - : TestColumnIndex(std::vector {min_value}, - std::vector {max_value}) {} + explicit StatisticsMemoryFileReader(std::vector bytes) + : _bytes(std::move(bytes)), _path("native-bloom-filter.parquet") {} - TestColumnIndex(std::vector min_values, std::vector max_values) - : _null_pages(min_values.size(), false), - _null_counts(min_values.size(), 0), - _min_values(std::move(min_values)), - _max_values(std::move(max_values)) { - EXPECT_EQ(_min_values.size(), _max_values.size()); - for (size_t page_idx = 0; page_idx < _min_values.size(); ++page_idx) { - _non_null_page_indices.push_back(static_cast(page_idx)); - } - } - - const std::vector& null_pages() const override { return _null_pages; } - const std::vector& encoded_min_values() const override { return _encoded_values; } - const std::vector& encoded_max_values() const override { return _encoded_values; } - ::parquet::BoundaryOrder::type boundary_order() const override { - return ::parquet::BoundaryOrder::Unordered; + Status close() override { + _closed = true; + return Status::OK(); } - bool has_null_counts() const override { return true; } - const std::vector& null_counts() const override { return _null_counts; } - const std::vector& non_null_page_indices() const override { - return _non_null_page_indices; + const io::Path& path() const override { return _path; } + size_t size() const override { return _bytes.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 1; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext*) override { + if (offset > _bytes.size() || result.size > _bytes.size() - offset) { + return Status::IOError("native Bloom test read exceeds memory file"); + } + memcpy(result.data, _bytes.data() + offset, result.size); + *bytes_read = result.size; + return Status::OK(); } - const std::vector& min_values() const override { return _min_values; } - const std::vector& max_values() const override { return _max_values; } private: - const std::vector _null_pages; - const std::vector _encoded_values; - const std::vector _null_counts; - std::vector _non_null_page_indices; - const std::vector _min_values; - const std::vector _max_values; + std::vector _bytes; + io::Path _path; + bool _closed = false; }; - -class Int32ZoneMapExpr final : public VExpr { +class BloomInExpr final : public VExpr { public: - enum class Op { GE, GT, IS_NULL, IS_NOT_NULL }; - - Int32ZoneMapExpr(int column_id, Op op, int32_t value = 0) + BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) : VExpr(std::make_shared(), false), - _column_id(column_id), - _op(op), - _value(value) {} + _slot(VSlotRef::create_shared(0, column_id, -1, std::move(data_type), "c0")), + _values(std::move(values)) {} const std::string& expr_name() const override { return _expr_name; } Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError("Int32ZoneMapExpr is only used by parquet statistics tests"); + return Status::InternalError("BloomInExpr is only used by parquet statistics tests"); } - bool can_evaluate_zonemap_filter() const override { return true; } + bool can_evaluate_bloom_filter() const override { return true; } - void collect_slot_column_ids(std::set& column_ids) const override { - column_ids.insert(_column_id); + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { + return expr_zonemap::eval_in_bloom_filter(ctx, _slot, false, _values); } - ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { - auto zone_map = ctx.zone_map(_column_id); - if (zone_map == nullptr) { - return unsupported_zonemap_filter(ctx); - } - if (_op == Op::IS_NULL) { - return zone_map->has_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (_op == Op::IS_NOT_NULL) { - return zone_map->has_not_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (!zone_map->has_not_null) { - return ZoneMapFilterResult::kNoMatch; - } - const auto literal = Field::create_field(_value); - if (_op == Op::GE) { - return zone_map->max_value < literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; - } - return zone_map->max_value <= literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; + void collect_slot_column_ids(std::set& column_ids) const override { + _slot->collect_slot_column_ids(column_ids); } private: - int _column_id; - Op _op; - int32_t _value; - const std::string _expr_name = "Int32ZoneMapExpr"; + VExprSPtr _slot; + std::vector _values; + const std::string _expr_name = "BloomInExpr"; }; -class StringDictionaryInExpr final : public VExpr { +class DictionaryStringInExpr final : public VExpr { public: - StringDictionaryInExpr(int column_id, std::vector values) - : VExpr(std::make_shared(), false), - _slot(VSlotRef::create_shared(0, column_id, -1, std::make_shared(), - "c0")) { - _values.reserve(values.size()); - for (auto& value : values) { - _values.emplace_back(Field::create_field(std::move(value))); - } - } + DictionaryStringInExpr() : VExpr(std::make_shared(), false) {} const std::string& expr_name() const override { return _expr_name; } Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError( - "StringDictionaryInExpr is only used by parquet statistics tests"); + return Status::InternalError("DictionaryStringInExpr is metadata-only"); } bool can_evaluate_dictionary_filter() const override { return true; } - ZoneMapFilterResult evaluate_dictionary_filter( - const DictionaryEvalContext& ctx) const override { - return expr_zonemap::eval_in_dictionary(ctx, _slot, false, _values); + ZoneMapFilterResult evaluate_dictionary_filter(const DictionaryEvalContext&) const override { + return ZoneMapFilterResult::kNoMatch; } - void collect_slot_column_ids(std::set& column_ids) const override { - _slot->collect_slot_column_ids(column_ids); - } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } private: - VExprSPtr _slot; - std::vector _values; - const std::string _expr_name = "StringDictionaryInExpr"; + const std::string _expr_name = "DictionaryStringInExpr"; }; -class BloomInExpr final : public VExpr { +class MetadataInt32GreaterThanExpr final : public VExpr { public: - BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) - : VExpr(std::make_shared(), false), - _slot(VSlotRef::create_shared(0, column_id, -1, std::move(data_type), "c0")), - _values(std::move(values)) {} + explicit MetadataInt32GreaterThanExpr(int32_t value) + : VExpr(std::make_shared(), false), _value(value) {} const std::string& expr_name() const override { return _expr_name; } - Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError("BloomInExpr is only used by parquet statistics tests"); + return Status::InternalError("MetadataInt32GreaterThanExpr is metadata-only"); } - - bool can_evaluate_bloom_filter() const override { return true; } - - ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { - return expr_zonemap::eval_in_bloom_filter(ctx, _slot, false, _values); - } - - void collect_slot_column_ids(std::set& column_ids) const override { - _slot->collect_slot_column_ids(column_ids); + bool can_evaluate_zonemap_filter() const override { return true; } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(0); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + return zone_map->max_value <= Field::create_field(_value) + ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; } private: - VExprSPtr _slot; - std::vector _values; - const std::string _expr_name = "BloomInExpr"; + int32_t _value; + const std::string _expr_name = "MetadataInt32GreaterThanExpr"; }; - -format::FileScanRequest request_with_zonemap_conjunct(std::shared_ptr expr) { - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared(std::move(expr))); - return request; -} - -format::FileScanRequest request_with_dictionary_conjunct(std::vector values) { - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared( - std::make_shared(0, std::move(values)))); - return request; -} - VExprContextSPtrs bloom_conjuncts(DataTypePtr data_type, std::vector values) { return {VExprContext::create_shared( std::make_shared(0, std::move(data_type), std::move(values)))}; @@ -346,690 +171,295 @@ format::FileScanRequest request_with_bloom_conjunct(DataTypePtr data_type, request.conjuncts = bloom_conjuncts(std::move(data_type), std::move(values)); return request; } - -void add_bloom_field(segment_v2::BlockSplitBloomFilter* bloom_filter, const Field& value, - PrimitiveType type) { - DORIS_CHECK(bloom_filter != nullptr); - switch (type) { - case TYPE_BOOLEAN: { - const bool typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_INT: { - const int32_t typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_STRING: { - const auto& typed_value = value.get(); - bloom_filter->add_bytes(typed_value.data(), typed_value.size()); - break; - } - default: - DORIS_CHECK(false); - } -} - -std::unique_ptr bloom_filter_for_fields( - const std::vector& values, PrimitiveType type) { - auto bloom_filter = std::make_unique(); - EXPECT_TRUE(bloom_filter->init(segment_v2::BloomFilter::MINIMUM_BYTES).ok()); - for (const auto& value : values) { - add_bloom_field(bloom_filter.get(), value, type); - } - return bloom_filter; -} - -BloomFilterEvalContext bloom_context(const DataTypePtr& data_type, - const segment_v2::BloomFilter* bloom_filter) { - BloomFilterEvalContext ctx; - ctx.slots.emplace(0, BloomFilterEvalContext::SlotBloomFilter {.data_type = data_type, - .bloom_filter = bloom_filter}); - return ctx; -} - -std::unique_ptr<::parquet::BlockSplitBloomFilter> parquet_bloom_filter() { - auto bloom_filter = std::make_unique<::parquet::BlockSplitBloomFilter>(); - bloom_filter->Init(::parquet::BlockSplitBloomFilter::kMinimumBloomFilterBytes); - return bloom_filter; -} - format::parquet::ParquetColumnSchema uint32_parquet_bloom_schema() { format::parquet::ParquetColumnSchema column_schema; column_schema.type = std::make_shared(); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::INT32; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; column_schema.type_descriptor.integer_bit_width = 32; column_schema.type_descriptor.is_unsigned_integer = true; return column_schema; } +TEST(ParquetBloomFilterPruningTest, NativeUint32BloomUsesPhysicalInt32Hash) { + const auto column_schema = uint32_parquet_bloom_schema(); + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) + .ok()); -format::parquet::ParquetColumnSchema fixed_len_string_parquet_bloom_schema(int fixed_length) { - format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); - column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = fixed_length; - column_schema.type_descriptor.is_string_like = true; - return column_schema; -} - -format::parquet::ParquetColumnSchema float16_parquet_bloom_schema() { - format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); - column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = 2; - column_schema.type_descriptor.extra_type_info = format::parquet::ParquetExtraTypeInfo::FLOAT16; - return column_schema; -} - -TEST(ParquetStatisticsTransformTest, ConvertsMinMaxNullCountUnsignedStringAndTimestamp) { - auto table = arrow::Table::Make( - arrow::schema({ - arrow::field("i", arrow::int32(), true), - arrow::field("u", arrow::uint32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - }), - {int32_array({1, std::nullopt, 5}), uint32_array({7, 9, 11}), - string_array({"alpha", "beta", "omega"}), timestamp_array({1000, 2000, 3000})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto row_group = reader->metadata()->RowGroup(0); - - const auto int_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], row_group->ColumnChunk(0)->statistics()); - EXPECT_TRUE(int_stats.has_min_max); - EXPECT_TRUE(int_stats.has_null_count); - EXPECT_TRUE(int_stats.has_null); - EXPECT_TRUE(int_stats.has_not_null); - EXPECT_EQ(int_stats.min_value.get(), 1); - EXPECT_EQ(int_stats.max_value.get(), 5); - - const auto uint_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[1], row_group->ColumnChunk(1)->statistics()); - EXPECT_TRUE(uint_stats.has_min_max); - EXPECT_EQ(uint_stats.min_value.get(), 7); - EXPECT_EQ(uint_stats.max_value.get(), 11); - - const auto string_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], row_group->ColumnChunk(2)->statistics()); - EXPECT_TRUE(string_stats.has_min_max); - EXPECT_EQ(string_stats.min_value.get(), "alpha"); - EXPECT_EQ(string_stats.max_value.get(), "omega"); - - auto utc = cctz::utc_time_zone(); - const auto timestamp_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[3], row_group->ColumnChunk(3)->statistics(), &utc); - EXPECT_TRUE(timestamp_stats.has_min_max); - EXPECT_EQ(timestamp_stats.min_value.get_type(), TYPE_DATETIMEV2); - EXPECT_EQ(timestamp_stats.max_value.get_type(), TYPE_DATETIMEV2); - EXPECT_LT(timestamp_stats.min_value, timestamp_stats.max_value); -} - -TEST(ParquetStatisticsTransformTest, DisablesUtcTimestampMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - // America/New_York moved from UTC-04:00 to UTC-05:00 at 2021-11-07 06:00:00 UTC. - // Both UTC endpoints below map to 01:30 local time, while values inside the interval cover - // 01:00 through 01:59. Endpoint conversion therefore cannot represent the true local range. - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636263900 * MICROS_PER_SECOND, - 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto local_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(local_stats.has_not_null); - EXPECT_FALSE(local_stats.has_min_max); - - auto utc = cctz::utc_time_zone(); - const auto utc_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &utc); - EXPECT_TRUE(utc_stats.has_min_max); - EXPECT_LT(utc_stats.min_value, utc_stats.max_value); -} - -TEST(ParquetStatisticsTransformTest, KeepsTimestampTzMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - // This is the effective type produced by enable_mapping_timestamp_tz. The physical timestamp - // flags intentionally remain adjusted-to-UTC so decoding can preserve the source semantics. - schema[0]->type = std::make_shared(6); - schema[0]->type_descriptor.doris_type = schema[0]->type; - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto timestamp_tz_stats = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(timestamp_tz_stats.has_min_max); - EXPECT_EQ(timestamp_tz_stats.min_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_EQ(timestamp_tz_stats.max_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_LT(timestamp_tz_stats.min_value, timestamp_tz_stats.max_value); -} - -TEST(ParquetStatisticsTransformTest, HandlesMissingStatisticsAndAllNullChunks) { - auto no_stats_table = arrow::Table::Make( - arrow::schema({arrow::field("i", arrow::int32(), true)}), {int32_array({1, 2, 3})}); - auto no_stats_reader = make_reader(no_stats_table, 3, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - auto no_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *no_stats_schema[0], - no_stats_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_FALSE(no_stats.has_min_max); - - auto all_null_table = - arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt})}); - auto all_null_reader = make_reader(all_null_table, 2, false, true); - auto all_null_schema = build_file_schema(*all_null_reader); - auto all_null_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *all_null_schema[0], - all_null_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_TRUE(all_null_stats.has_null_count); - EXPECT_TRUE(all_null_stats.has_null); - EXPECT_FALSE(all_null_stats.has_not_null); - EXPECT_FALSE(all_null_stats.has_min_max); -} - -TEST(ParquetStatisticsTransformTest, MissingNullCountConservativelyReportsPossibleNulls) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({1, std::nullopt, 3})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto file_statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - auto statistics_without_null_count = ::parquet::MakeStatistics<::parquet::Int32Type>( - reader->metadata()->schema()->Column(0), file_statistics->EncodeMin(), - file_statistics->EncodeMax(), file_statistics->num_values(), 0, 0, true, false, false); - - const auto statistics = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics_without_null_count); - EXPECT_FALSE(statistics.has_null_count); - EXPECT_TRUE(statistics.has_null); - EXPECT_TRUE(statistics.has_not_null); - EXPECT_TRUE(statistics.has_min_max); - EXPECT_EQ(statistics.min_value.get(), 1); - EXPECT_EQ(statistics.max_value.get(), 3); -} - -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const float float_nan = std::numeric_limits::quiet_NaN(); - const float float_max = 2.0F; - auto float_stats = ::parquet::MakeStatistics<::parquet::FloatType>( - schema[0]->descriptor, encoded_value(float_nan), encoded_value(float_max), 2, 0, 0, - true, true, false); - const auto converted_float = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], float_stats); - EXPECT_FALSE(converted_float.has_min_max); - EXPECT_TRUE(converted_float.has_not_null); - - const double double_nan = std::numeric_limits::quiet_NaN(); - const double double_min = 1.0; - auto double_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_nan), 2, 0, 0, - true, true, false); - const auto converted_double = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - double_stats); - EXPECT_FALSE(converted_double.has_min_max); - EXPECT_TRUE(converted_double.has_not_null); - - const double double_max = 2.0; - auto finite_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_max), 2, 0, 0, - true, true, false); - const auto converted_finite = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - finite_stats); - EXPECT_TRUE(converted_finite.has_min_max); -} - -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleColumnIndexMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - auto float_index = std::make_shared>( - 1.0F, std::numeric_limits::quiet_NaN()); - format::parquet::ParquetColumnStatistics float_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - float_index, *schema[0], 0, &float_page_stats)); - EXPECT_FALSE(float_page_stats.has_min_max); - EXPECT_TRUE(float_page_stats.has_not_null); - - auto double_index = std::make_shared>( - std::numeric_limits::quiet_NaN(), 2.0); - format::parquet::ParquetColumnStatistics double_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - double_index, *schema[1], 0, &double_page_stats)); - EXPECT_FALSE(double_page_stats.has_min_max); - EXPECT_TRUE(double_page_stats.has_not_null); - - auto finite_index = std::make_shared>(1.0, 2.0); - format::parquet::ParquetColumnStatistics finite_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - finite_index, *schema[1], 0, &finite_page_stats)); - EXPECT_TRUE(finite_page_stats.has_min_max); - - auto mixed_index = std::make_shared>( - std::vector {std::numeric_limits::quiet_NaN(), 1.0}, - std::vector {std::numeric_limits::quiet_NaN(), 2.0}); - format::parquet::ParquetColumnStatistics nan_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 0, &nan_page_stats)); - EXPECT_FALSE(nan_page_stats.has_min_max); - - format::parquet::ParquetColumnStatistics following_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 1, &following_page_stats)); - EXPECT_TRUE(following_page_stats.has_min_max); - EXPECT_EQ(following_page_stats.min_value.get(), 1.0); - EXPECT_EQ(following_page_stats.max_value.get(), 2.0); -} - -TEST(ParquetStatisticsTransformTest, IgnoresInvertedFooterAndColumnIndexMinMax) { - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("i", arrow::int32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {int32_array({1, 2}), string_array({"a", "z"}), timestamp_array({1000000, 2000000})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const int32_t inverted_min = 10; - const int32_t inverted_max = 1; - auto int_stats = ::parquet::MakeStatistics<::parquet::Int32Type>( - schema[0]->descriptor, encoded_value(inverted_min), encoded_value(inverted_max), 2, 0, - 0, true, true, false); - const auto converted_int = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], int_stats); - EXPECT_TRUE(converted_int.has_not_null); - EXPECT_FALSE(converted_int.has_min_max); - - const std::string inverted_string_min = "z"; - const std::string inverted_string_max = "a"; - auto string_stats = ::parquet::MakeStatistics<::parquet::ByteArrayType>( - schema[1]->descriptor, inverted_string_min, inverted_string_max, 2, 0, 0, true, true, - false); - const auto converted_string = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - string_stats); - EXPECT_TRUE(converted_string.has_not_null); - EXPECT_FALSE(converted_string.has_min_max); - - auto int_index = - std::make_shared>(inverted_min, inverted_max); - format::parquet::ParquetColumnStatistics page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - int_index, *schema[0], 0, &page_stats)); - EXPECT_TRUE(page_stats.has_not_null); - EXPECT_FALSE(page_stats.has_min_max); - - // These endpoints are inverted within one second. Whole-second validation alone would miss - // the corruption, and TIMESTAMPTZ must reject the same raw inversion before its UTC shortcut. - constexpr int64_t timestamp_min = 1500000; - constexpr int64_t timestamp_max = 1000000; - auto timestamp_stats = ::parquet::MakeStatistics<::parquet::Int64Type>( - schema[2]->descriptor, encoded_value(timestamp_min), encoded_value(timestamp_max), 2, 0, - 0, true, true, false); - auto utc = cctz::utc_time_zone(); - const auto converted_timestamp = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp.has_min_max); - - schema[2]->type = std::make_shared(6); - schema[2]->type_descriptor.doris_type = schema[2]->type; - const auto converted_timestamp_tz = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp_tz.has_min_max); -} - -TEST(ParquetStatisticsTransformTest, PreservesNullCountWhenNaNInvalidatesMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float64(), false)}), - {double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const double nan = std::numeric_limits::quiet_NaN(); - const double max_value = 2.0; - auto footer_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[0]->descriptor, encoded_value(nan), encoded_value(max_value), 2, 0, 0, true, - true, false); - const auto converted_footer = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[0], - footer_stats); - auto footer_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(converted_footer); - ASSERT_NE(footer_zone_map, nullptr); - EXPECT_TRUE(footer_zone_map->pass_all); - EXPECT_FALSE(footer_zone_map->has_null); - EXPECT_TRUE(footer_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*footer_zone_map, schema[0]->type)); - - ZoneMapEvalContext footer_ctx; - footer_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = footer_zone_map}); - Int32ZoneMapExpr is_null_expr(0, Int32ZoneMapExpr::Op::IS_NULL); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(footer_ctx), ZoneMapFilterResult::kNoMatch); - - auto column_index = std::make_shared>(nan, max_value); - format::parquet::ParquetColumnStatistics page_stats; - ASSERT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *schema[0], 0, &page_stats)); - auto page_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(page_stats); - ASSERT_NE(page_zone_map, nullptr); - EXPECT_TRUE(page_zone_map->pass_all); - EXPECT_FALSE(page_zone_map->has_null); - EXPECT_TRUE(page_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*page_zone_map, schema[0]->type)); + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); - ZoneMapEvalContext page_ctx; - page_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = page_zone_map}); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(page_ctx), ZoneMapFilterResult::kNoMatch); + EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field( + static_cast(present_value))}), + bloom_filter)); + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), + bloom_filter)); } -TEST(ParquetStatisticsPruningTest, ExprZonemapPredicatesAndNullPredicatesPruneRowGroups) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt, 3, 4, 5, 6})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct( - std::make_shared(0, Int32ZoneMapExpr::Op::GE, 5)), - nullptr, &selected, false, &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({2})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 2); +TEST(ParquetBloomFilterPruningTest, NativeRowGroupKeepsPresentUint32AboveInt32Max) { + auto column_schema = + std::make_unique(uint32_parquet_bloom_schema()); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; - selected.clear(); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NOT_NULL)), - nullptr, &selected, false, &pruning_stats) + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) .ok()); - EXPECT_EQ(selected, std::vector({1, 2})); - - selected.clear(); + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); + + tparquet::BloomFilterAlgorithm algorithm; + algorithm.__set_BLOCK(tparquet::SplitBlockAlgorithm()); + tparquet::BloomFilterHash hash; + hash.__set_XXHASH(tparquet::XxHash()); + tparquet::BloomFilterCompression compression; + compression.__set_UNCOMPRESSED(tparquet::Uncompressed()); + tparquet::BloomFilterHeader bloom_header; + bloom_header.__set_numBytes(static_cast(bloom_filter.size())); + bloom_header.__set_algorithm(algorithm); + bloom_header.__set_hash(hash); + bloom_header.__set_compression(compression); + std::vector bloom_bytes; + ThriftSerializer serializer(/*compact=*/true, 64); + ASSERT_TRUE(serializer.serialize(&bloom_header, &bloom_bytes).ok()); + bloom_bytes.insert(bloom_bytes.end(), bloom_filter.data(), + bloom_filter.data() + bloom_filter.size()); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(1); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_bloom_filter_offset(0); + column_metadata.__set_bloom_filter_length(static_cast(bloom_bytes.size())); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_version(1); + metadata.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + + format::parquet::ParquetFileContext file_context; + file_context.native_file = std::make_shared(std::move(bloom_bytes)); + auto request = request_with_bloom_conjunct( + column_schema->type, + {Field::create_field(static_cast(present_value))}); + std::vector> schema; + schema.push_back(std::move(column_schema)); + std::vector selected_row_groups; + format::parquet::ParquetPruningStats pruning_stats; ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NULL)), - nullptr, &selected, false, &pruning_stats) + metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) .ok()); - EXPECT_EQ(selected, std::vector({0})); + EXPECT_EQ(selected_row_groups, std::vector({0})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); } -TEST(ParquetStatisticsPruningTest, DictionaryPruningHandlesExcludeIncludeAndUnsupportedPaths) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, false); - auto schema = build_file_schema(*reader); +TEST(NativeParquetStatisticsTest, EmptyDictionaryRowGroupIsSkippedBeforeMetadataProbes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::BYTE_ARRAY); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(0); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_dictionary_page_offset(0); + column_metadata.__set_encodings({tparquet::Encoding::RLE_DICTIONARY}); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(0); + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root, leaf}); + thrift_metadata.__set_num_rows(0); + thrift_metadata.__set_row_groups({row_group}); + + format::parquet::NativeParquetMetadata native_metadata(thrift_metadata, 0); + ASSERT_TRUE(native_metadata.init_schema(false, false).ok()); + format::parquet::ParquetFileContext file_context; + file_context.native_file = + std::make_shared(std::vector {}); + file_context.native_metadata = &native_metadata; + + auto column_schema = std::make_unique(); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::BYTE_ARRAY; + column_schema->type_descriptor.is_string_like = true; + std::vector> schema; + schema.push_back(std::move(column_schema)); - std::vector selected; + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = {VExprContext::create_shared(std::make_shared())}; + std::vector selected_row_groups; format::parquet::ParquetPruningStats pruning_stats; ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"gamma"}), nullptr, &selected, false, - &pruning_stats) + thrift_metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 1); - - auto plain_reader = make_reader(table, 2, false, false); - auto plain_schema = build_file_schema(*plain_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *plain_reader->metadata(), plain_reader.get(), plain_schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 0); + EXPECT_TRUE(selected_row_groups.empty()); } -TEST(ParquetStatisticsPruningTest, VExprUsesDictionaryAndMissingBloomKeepsRows) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, true); - auto schema = build_file_schema(*reader); +TEST(NativeParquetStatisticsTest, InvalidCandidateRowGroupReturnsCorruption) { + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); + format::FileScanRequest request; + const std::vector> schema; + const std::vector candidates {1}; + std::vector selected_row_groups; + + const auto status = format::parquet::select_row_groups_by_metadata( + metadata, schema, request, &candidates, &selected_row_groups, false, nullptr, nullptr, + nullptr, nullptr); + EXPECT_TRUE(status.is()) << status; +} +TEST(NativeParquetStatisticsTest, LegacyBinaryFooterBoundsRequireComparableOrdering) { + format::parquet::ParquetTypeDescriptor binary_type; + binary_type.physical_type = tparquet::Type::BYTE_ARRAY; + + tparquet::Statistics max_only; + max_only.__set_max("III"); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(binary_type, max_only, false)); + + tparquet::Statistics legacy_different; + legacy_different.__set_min("III"); + legacy_different.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, + legacy_different, false)); + + tparquet::Statistics legacy_equal; + legacy_equal.__set_min("same"); + legacy_equal.__set_max("same"); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, legacy_equal, + false)); + + tparquet::Statistics type_defined; + type_defined.__set_min_value("III"); + type_defined.__set_max_value("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + false)); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + true)); + + tparquet::Statistics mixed_fields; + mixed_fields.__set_min_value("III"); + mixed_fields.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, mixed_fields, + true)); +} + +TEST(NativeParquetStatisticsTest, TypeDefinedBoundsRequireSupportedColumnOrder) { + auto encode_int32 = [](int32_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + + auto column_schema = std::make_unique(); + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::INT32; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::Statistics statistics; + statistics.__set_min_value(encode_int32(1)); + statistics.__set_max_value(encode_int32(2)); + statistics.__set_null_count(0); + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_num_values(1); + column_metadata.__set_statistics(statistics); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = { + VExprContext::create_shared(std::make_shared(100))}; + std::vector selected_row_groups; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - auto no_stats_reader = make_reader(table, 2, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *no_stats_reader->metadata(), no_stats_reader.get(), no_stats_schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) + EXPECT_EQ(selected_row_groups, std::vector({0})); + + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_min_values({encode_int32(1)}); + page_index.column_index.__set_max_values({encode_int32(2)}); + page_index.column_index.__set_null_pages({false}); + page_index.column_index.__set_null_counts({0}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, page_index); + std::vector selected_ranges; + std::map skip_plans; + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); -} - -TEST(ParquetStatisticsPruningTest, BloomFilterCacheIsScopedToRowGroupAndColumn) { - auto input = arrow::io::ReadableFile::Open( - "./be/test/exec/test_data/parquet_scanner/multi_row_group_bloom_filter.parquet"); - ASSERT_TRUE(input.ok()); - auto reader = ::parquet::ParquetFileReader::Open(*input); - ASSERT_EQ(reader->metadata()->num_row_groups(), 2); - auto& bloom_filter_reader = reader->GetBloomFilterReader(); - for (int row_group_idx = 0; row_group_idx < 2; ++row_group_idx) { - auto row_group_reader = bloom_filter_reader.RowGroup(row_group_idx); - ASSERT_NE(row_group_reader, nullptr); - ASSERT_NE(row_group_reader->GetColumnBloomFilter(0), nullptr); - } - auto schema = build_file_schema(*reader); - - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - auto request = request_with_bloom_conjunct(std::make_shared(), - {Field::create_field(12345)}); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - false, &pruning_stats) + EXPECT_EQ(selected_ranges.size(), 1); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + metadata.__set_column_orders({order}); + selected_row_groups.clear(); + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - true, &pruning_stats) + EXPECT_TRUE(selected_row_groups.empty()); + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 1); -} - -TEST(ParquetBloomFilterPruningTest, VExprEqPrunesAbsentIntValue) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2)}), ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(3)}), ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprInPrunesOnlyWhenAllValuesAreAbsent) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(4)}), - ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(3)}), - ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprBoolAndStringUseSlotBloomFilter) { - auto bool_type = std::make_shared(); - auto bool_filter = - bloom_filter_for_fields({Field::create_field(true)}, TYPE_BOOLEAN); - auto bool_ctx = bloom_context(bool_type, bool_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(false)}), - bool_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(true)}), - bool_ctx), - ZoneMapFilterResult::kMayMatch); - - auto string_type = std::make_shared(); - auto string_filter = bloom_filter_for_fields( - {Field::create_field("alpha"), Field::create_field("omega")}, - TYPE_STRING); - auto string_ctx = bloom_context(string_type, string_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("beta")}), - string_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("alpha")}), - string_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, MissingOrUnsupportedBloomContextKeepsRowGroup) { - auto int_type = std::make_shared(); - BloomFilterEvalContext missing_ctx; - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(int_type, {Field::create_field(2)}), missing_ctx), - ZoneMapFilterResult::kMayMatch); - - auto smallint_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields({Field::create_field(1)}, TYPE_INT); - auto unsupported_ctx = bloom_context(smallint_type, bloom_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(smallint_type, {Field::create_field(2)}), - unsupported_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, ParquetUint32BloomUsesPhysicalInt32Hash) { - const auto column_schema = uint32_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - const uint32_t present_value = 4000000000U; - int32_t physical_value; - memcpy(&physical_value, &present_value, sizeof(physical_value)); - bloom_filter->InsertHash(bloom_filter->Hash(physical_value)); - - // UINT32 is exposed to VExpr as Doris BIGINT, but Parquet stores and hashes it as a physical - // INT32 carrier. A present value above INT32_MAX must therefore be narrowed to the physical - // bit pattern before probing the file bloom filter. - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field( - static_cast(present_value))}), - *bloom_filter)); - - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts( - column_schema.type, - {Field::create_field( - static_cast(std::numeric_limits::max()) + 1)}), - *bloom_filter)); -} - -TEST(ParquetBloomFilterPruningTest, ParquetFixedLenByteArrayBloomUsesFlbaHash) { - const auto column_schema = fixed_len_string_parquet_bloom_schema(4); - auto bloom_filter = parquet_bloom_filter(); - - const std::string present_value = "abcd"; - ::parquet::FLBA physical_value(reinterpret_cast(present_value.data())); - bloom_filter->InsertHash( - bloom_filter->Hash(&physical_value, column_schema.type_descriptor.fixed_length)); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(present_value)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field("abc")}), - *bloom_filter)); -} - -TEST(ParquetBloomFilterPruningTest, ParquetFloat16BloomDoesNotUseFloatHash) { - const auto column_schema = float16_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(1.0F)}), - *bloom_filter)); + EXPECT_TRUE(selected_ranges.empty()); } } // namespace diff --git a/be/test/format_v2/parquet/parquet_type_test.cpp b/be/test/format_v2/parquet/parquet_type_test.cpp index 4bca77c1803b49..0680cc828a9f7a 100644 --- a/be/test/format_v2/parquet/parquet_type_test.cpp +++ b/be/test/format_v2/parquet/parquet_type_test.cpp @@ -17,478 +17,49 @@ #include "format_v2/parquet/parquet_type.h" -#include -#include #include -#include -#include -#include - -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/primitive_type.h" namespace doris::format::parquet { -namespace { - -::parquet::SchemaDescriptor make_descriptor(const ::parquet::schema::NodePtr& node) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, {node}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - return descriptor; -} - -ParquetTypeDescriptor resolve_node(const ::parquet::schema::NodePtr& node) { - auto descriptor = make_descriptor(node); - return resolve_parquet_type(descriptor.Column(0)); -} - -PrimitiveType primitive_type(const DataTypePtr& type) { - return remove_nullable(type)->get_primitive_type(); -} - -int scale_of(const DataTypePtr& type) { - return remove_nullable(type)->get_scale(); -} - -std::shared_ptr make_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.Append(0x3E00).ok()); - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetTypeDescriptor resolve_arrow_float16_type() { - const auto schema = arrow::schema({arrow::field("f16", arrow::float16(), true)}); - const auto table = arrow::Table::Make(schema, {make_float16_array()}); - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1).ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - - auto reader = ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); - return resolve_parquet_type(reader->metadata()->schema()->Column(0)); -} - -} // namespace - -TEST(ParquetTypeTest, ResolveLogicalIntegerMappings) { - struct Case { - int bit_width; - bool is_signed; - PrimitiveType expected_type; - bool expected_unsigned; - }; - const std::vector cases = { - {8, true, TYPE_TINYINT, false}, {8, false, TYPE_SMALLINT, true}, - {16, true, TYPE_SMALLINT, false}, {16, false, TYPE_INT, true}, - {32, true, TYPE_INT, false}, {32, false, TYPE_BIGINT, true}, - {64, true, TYPE_BIGINT, false}, {64, false, TYPE_LARGEINT, true}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.bit_width); - const auto node = ::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Int(test_case.bit_width, test_case.is_signed), - test_case.bit_width == 64 ? ::parquet::Type::INT64 : ::parquet::Type::INT32); - const auto type = resolve_node(node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_TRUE(type.supports_record_reader); - } -} - -TEST(ParquetTypeTest, ResolveLogicalTimeAndTimestampMappings) { - const auto time_millis = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - ASSERT_NE(time_millis.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_millis.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_millis.time_unit, ParquetTimeUnit::MILLIS); - EXPECT_EQ(time_millis.extra_type_info, ParquetExtraTypeInfo::UNIT_MS); - - const auto time_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_us", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64)); - ASSERT_NE(time_micros.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_micros.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(time_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - - const auto adjusted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_adjusted", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - EXPECT_EQ(adjusted_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_time.supports_record_reader); - EXPECT_FALSE(adjusted_time.unsupported_reason.empty()); - - const auto timestamp_nanos = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ns", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(timestamp_nanos.doris_type, nullptr); - EXPECT_TRUE(timestamp_nanos.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(timestamp_nanos.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(timestamp_nanos.is_timestamp); - EXPECT_TRUE(timestamp_nanos.timestamp_is_adjusted_to_utc); - EXPECT_EQ(timestamp_nanos.time_unit, ParquetTimeUnit::NANOS); - EXPECT_EQ(timestamp_nanos.extra_type_info, ParquetExtraTypeInfo::UNIT_NS); -} - -TEST(ParquetTypeTest, ResolveLogicalTimestampMatrix) { - struct Case { - ::parquet::LogicalType::TimeUnit::unit parquet_unit; - bool adjusted_to_utc; - ParquetTimeUnit expected_unit; - ParquetExtraTypeInfo expected_extra; - int expected_scale; - }; - const std::vector cases = { - {::parquet::LogicalType::TimeUnit::MILLIS, true, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MILLIS, false, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MICROS, true, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::MICROS, false, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, true, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, false, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_scale); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(test_case.adjusted_to_utc, - test_case.parquet_unit), - ::parquet::Type::INT64)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(type.doris_type), test_case.expected_scale); - EXPECT_TRUE(type.is_timestamp); - EXPECT_EQ(type.timestamp_is_adjusted_to_utc, test_case.adjusted_to_utc); - EXPECT_EQ(type.time_unit, test_case.expected_unit); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ConvertedTimeIsRejectedButConvertedTimestampIsSupported) { - const auto converted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS)); - EXPECT_EQ(converted_time.doris_type, nullptr); - EXPECT_FALSE(converted_time.supports_record_reader); - EXPECT_FALSE(converted_time.unsupported_reason.empty()); - - const auto converted_timestamp = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MILLIS)); - ASSERT_NE(converted_timestamp.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_timestamp.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(converted_timestamp.is_timestamp); - EXPECT_TRUE(converted_timestamp.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp.time_unit, ParquetTimeUnit::MILLIS); - const auto converted_timestamp_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_us", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MICROS)); - ASSERT_NE(converted_timestamp_micros.doris_type, nullptr); - EXPECT_TRUE(converted_timestamp_micros.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(converted_timestamp_micros.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(converted_timestamp_micros.doris_type), 6); - EXPECT_TRUE(converted_timestamp_micros.is_timestamp); - EXPECT_TRUE(converted_timestamp_micros.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(converted_timestamp_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); -} - -TEST(ParquetTypeTest, ResolveConvertedIntegerMappingsAndDecodedKinds) { - struct Case { - ::parquet::ConvertedType::type converted_type; - ::parquet::Type::type physical_type; - PrimitiveType expected_type; - int bit_width; - bool expected_unsigned; - DecodedValueKind expected_value_kind; - }; - const std::vector cases = { - {::parquet::ConvertedType::INT_8, ::parquet::Type::INT32, TYPE_TINYINT, 8, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_8, ::parquet::Type::INT32, TYPE_SMALLINT, 8, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_16, ::parquet::Type::INT32, TYPE_SMALLINT, 16, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_16, ::parquet::Type::INT32, TYPE_INT, 16, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_32, ::parquet::Type::INT32, TYPE_INT, 32, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_32, ::parquet::Type::INT32, TYPE_BIGINT, 32, true, - DecodedValueKind::UINT32}, - {::parquet::ConvertedType::INT_64, ::parquet::Type::INT64, TYPE_BIGINT, 64, false, - DecodedValueKind::INT64}, - {::parquet::ConvertedType::UINT_64, ::parquet::Type::INT64, TYPE_LARGEINT, 64, true, - DecodedValueKind::UINT64}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.converted_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, test_case.physical_type, - test_case.converted_type)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_value_kind); - } -} - -TEST(ParquetTypeTest, ResolveConvertedDecimalCarriers) { - struct Case { - ::parquet::Type::type physical_type; - int type_length; - int precision; - int scale; - PrimitiveType expected_type; - ParquetExtraTypeInfo expected_extra; - }; - const std::vector cases = { - {::parquet::Type::INT32, -1, 9, 2, TYPE_DECIMAL32, ParquetExtraTypeInfo::DECIMAL_INT32}, - {::parquet::Type::INT64, -1, 18, 6, TYPE_DECIMAL64, - ParquetExtraTypeInfo::DECIMAL_INT64}, - {::parquet::Type::BYTE_ARRAY, -1, 20, 5, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16, 38, 6, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20, 39, 6, TYPE_DECIMAL256, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.physical_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, test_case.physical_type, - ::parquet::ConvertedType::DECIMAL, test_case.type_length, test_case.precision, - test_case.scale)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_TRUE(type.is_decimal); - EXPECT_FALSE(type.is_string_like); - EXPECT_EQ(type.decimal_precision, test_case.precision); - EXPECT_EQ(type.decimal_scale, test_case.scale); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ResolveLogicalStringDateAndDecimalMappings) { - const std::vector> string_like_logical_types = { - ::parquet::LogicalType::String(), ::parquet::LogicalType::Enum(), - ::parquet::LogicalType::JSON(), ::parquet::LogicalType::BSON()}; - for (const auto& logical_type : string_like_logical_types) { - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::OPTIONAL, logical_type, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_STRING); - EXPECT_TRUE(type.is_string_like); - } - - const auto uuid = resolve_node(::parquet::schema::PrimitiveNode::Make( - "uuid", ::parquet::Repetition::OPTIONAL, ::parquet::LogicalType::UUID(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(uuid.doris_type, nullptr); - EXPECT_TRUE(uuid.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(uuid.doris_type), TYPE_STRING); - EXPECT_TRUE(uuid.is_string_like); - - const auto date = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Date(), - ::parquet::Type::INT32)); - ASSERT_NE(date.doris_type, nullptr); - EXPECT_EQ(primitive_type(date.doris_type), TYPE_DATEV2); - - const auto decimal64 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d64", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(18, 6), - ::parquet::Type::INT64)); - ASSERT_NE(decimal64.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal64.doris_type), TYPE_DECIMAL64); - EXPECT_TRUE(decimal64.is_decimal); - EXPECT_EQ(decimal64.decimal_precision, 18); - EXPECT_EQ(decimal64.decimal_scale, 6); - EXPECT_EQ(decimal64.extra_type_info, ParquetExtraTypeInfo::DECIMAL_INT64); - - const auto decimal128 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d128", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(38, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(decimal128.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal128.doris_type), TYPE_DECIMAL128I); - EXPECT_TRUE(decimal128.is_decimal); - EXPECT_EQ(decimal128.decimal_precision, 38); - EXPECT_EQ(decimal128.decimal_scale, 6); - EXPECT_EQ(decimal128.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d256", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(39, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - EXPECT_FALSE(decimal256.is_string_like); -} - -TEST(ParquetTypeTest, LogicalConvertedAndPhysicalFallbackLevelsAreDistinct) { - const auto logical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Int(8, true), - ::parquet::Type::INT32)); - ASSERT_NE(logical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(logical_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(logical_type.integer_bit_width, 8); - - const auto converted_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8)); - ASSERT_NE(converted_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(converted_type.integer_bit_width, 8); - - const auto physical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32)); - ASSERT_NE(physical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(physical_type.doris_type), TYPE_INT); - EXPECT_EQ(physical_type.integer_bit_width, -1); -} - -TEST(ParquetTypeTest, ResolveDecimalStringLikeFloat16AndPhysicalFallback) { - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::DECIMAL, 20, 39, 6)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_FALSE(decimal256.is_string_like); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto plain_binary = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::REQUIRED, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(plain_binary.doris_type, nullptr); - EXPECT_EQ(primitive_type(plain_binary.doris_type), TYPE_STRING); - EXPECT_TRUE(plain_binary.is_string_like); - - const auto float16 = resolve_arrow_float16_type(); - ASSERT_NE(float16.doris_type, nullptr); - EXPECT_TRUE(float16.doris_type->is_nullable()); - EXPECT_EQ(float16.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(float16.fixed_length, 2); - EXPECT_EQ(primitive_type(float16.doris_type), TYPE_FLOAT); - EXPECT_EQ(float16.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(float16.is_string_like); - EXPECT_EQ(decoded_value_kind(float16), DecodedValueKind::FIXED_BINARY); -} - -TEST(ParquetTypeTest, ResolveNullDescriptorAndPhysicalFallback) { - const auto null_type = resolve_parquet_type(nullptr); - EXPECT_EQ(null_type.doris_type, nullptr); - EXPECT_EQ(null_type.physical_type, ::parquet::Type::UNDEFINED); - EXPECT_TRUE(null_type.supports_record_reader); - - const auto int96 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT96)); - ASSERT_NE(int96.doris_type, nullptr); - EXPECT_EQ(primitive_type(int96.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(int96.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - EXPECT_EQ(decoded_value_kind(int96), DecodedValueKind::INT96); -} - -TEST(ParquetTypeTest, ResolveEveryPhysicalFallback) { - struct Case { - ::parquet::schema::NodePtr node; - PrimitiveType expected_type; - DecodedValueKind expected_kind; - bool expected_string_like = false; - }; - const std::vector cases = { - {::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BOOLEAN), - TYPE_BOOLEAN, DecodedValueKind::BOOL}, - {::parquet::schema::PrimitiveNode::Make("i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - TYPE_INT, DecodedValueKind::INT32}, - {::parquet::schema::PrimitiveNode::Make("i64", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT64), - TYPE_BIGINT, DecodedValueKind::INT64}, - {::parquet::schema::PrimitiveNode::Make("f", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FLOAT), - TYPE_FLOAT, DecodedValueKind::FLOAT}, - {::parquet::schema::PrimitiveNode::Make("d", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - TYPE_DOUBLE, DecodedValueKind::DOUBLE}, - {::parquet::schema::PrimitiveNode::Make("s", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY), - TYPE_STRING, DecodedValueKind::BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("fs", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::NONE, 4), - TYPE_STRING, DecodedValueKind::FIXED_BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("ts", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT96), - TYPE_DATETIMEV2, DecodedValueKind::INT96}, +TEST(ParquetTypeTest, DecodedValueKindUsesNativePhysicalTypes) { + ParquetTypeDescriptor descriptor; + + const std::pair cases[] = { + {tparquet::Type::BOOLEAN, DecodedValueKind::BOOL}, + {tparquet::Type::INT32, DecodedValueKind::INT32}, + {tparquet::Type::INT64, DecodedValueKind::INT64}, + {tparquet::Type::INT96, DecodedValueKind::INT96}, + {tparquet::Type::FLOAT, DecodedValueKind::FLOAT}, + {tparquet::Type::DOUBLE, DecodedValueKind::DOUBLE}, + {tparquet::Type::BYTE_ARRAY, DecodedValueKind::BINARY}, + {tparquet::Type::FIXED_LEN_BYTE_ARRAY, DecodedValueKind::FIXED_BINARY}, }; - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_type); - const auto type = resolve_node(test_case.node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_kind); - EXPECT_EQ(type.is_string_like, test_case.expected_string_like); - EXPECT_TRUE(type.supports_record_reader); + for (const auto& [physical_type, expected] : cases) { + descriptor.physical_type = physical_type; + descriptor.is_unsigned_integer = false; + descriptor.integer_bit_width = -1; + EXPECT_EQ(decoded_value_kind(descriptor), expected); } } -TEST(ParquetTypeTest, InvalidLogicalAnnotationsFallBackOrRejectAsSpecified) { - EXPECT_THROW(::parquet::LogicalType::Int(24, true), ::parquet::ParquetException); +TEST(ParquetTypeTest, DecodedValueKindPreservesUnsignedWidth) { + ParquetTypeDescriptor descriptor; + descriptor.is_unsigned_integer = true; - const auto nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(nanos_time.doris_type, nullptr); - EXPECT_EQ(primitive_type(nanos_time.doris_type), TYPE_BIGINT); - EXPECT_TRUE(nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 32; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT32); - const auto adjusted_nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns_utc", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - EXPECT_EQ(adjusted_nanos_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_nanos_time.supports_record_reader); - EXPECT_FALSE(adjusted_nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT64; + descriptor.integer_bit_width = 64; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT64); - EXPECT_THROW(::parquet::schema::PrimitiveNode::Make("f16_bad", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Float16(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 4), - ::parquet::ParquetException); + // Narrow unsigned logical integers still use the signed physical decoder; conversion + // happens after decoding so the on-disk bit width remains the single source of truth. + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 16; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::INT32); } } // namespace doris::format::parquet diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index 2dd001469d8a66..6f9bea89ffe943 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -23,10 +23,12 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -122,6 +124,23 @@ TTableFormatFileDesc hudi_table_format_desc(std::optional schema_id) { return table_format_params; } +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + // Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical // file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id // mapping, including renamed nested children. @@ -300,5 +319,47 @@ TEST(HudiHybridReaderTest, NativeCountStarReportsMetadataRowsThroughHybridReader std::filesystem::remove_all(test_dir); } +TEST(HudiHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileFormatType::type thrift_format) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range.__set_format_type(thrift_format); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + verify_first_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp new file mode 100644 index 00000000000000..a219ba37f3dd8d --- /dev/null +++ b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" + +#include + +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::iceberg { +namespace { + +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_delete_files({delete_file}); + TTableFormatFileDesc table_format_desc; + table_format_desc.__set_iceberg_params(std::move(iceberg_desc)); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_desc)); + return range; +} + +TEST(IcebergPositionDeleteSysTableV2ProfileTest, UsesDistinctProfileForNestedPositionReader) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("position_delete_system_table_profile"); + TFileScanRangeParams params; + std::vector file_slot_descs; + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + delete_file.__set_path("/not-opened-during-prepare.parquet"); + + IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._scan_params = ¶ms; + reader._file_slot_descs = &file_slot_descs; + reader._current_range = range_with_delete_file(delete_file); + ASSERT_TRUE(reader._init_split().ok()); + + ASSERT_NE(reader._position_reader_profile, nullptr); + EXPECT_NE(reader._position_reader_profile, &profile); + EXPECT_EQ(profile.get_child("IcebergPositionDeleteFileReader"), + reader._position_reader_profile); +} + +} // namespace +} // namespace doris::format::iceberg diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 71dc1ea1db7c69..7d8572101e342e 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -25,12 +25,15 @@ #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -732,6 +735,12 @@ class ScopedDebugPoint { DebugPoints::instance()->add(_name); } + ScopedDebugPoint(std::string name, std::function callback) + : _name(std::move(name)), _enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add_with_callback(_name, std::move(callback)); + } + ~ScopedDebugPoint() { DebugPoints::instance()->remove(_name); config::enable_debug_points = _enable_debug_points; @@ -2934,7 +2943,25 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteFileIsReusedAcrossSplits) { first_split.cache = &cache; first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); - ASSERT_TRUE(reader.prepare_split(first_split).ok()); + const auto* total_timer = profile.get_counter("TableReader"); + const auto* prepare_timer = profile.get_counter("PrepareSplitTime"); + ASSERT_NE(total_timer, nullptr); + ASSERT_NE(prepare_timer, nullptr); + const int64_t total_before = total_timer->value(); + const int64_t prepare_before = prepare_timer->value(); + constexpr int64_t DELETE_FILE_DELAY_NS = 8'000'000; + { + ScopedDebugPoint delay_delete_file_scan( + "IcebergTableReader.prepare_split.before_delete_file_scan", + [] { std::this_thread::sleep_for(std::chrono::milliseconds(8)); }); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + } + const int64_t total_delta = total_timer->value() - total_before; + const int64_t prepare_delta = prepare_timer->value() - prepare_before; + // A cache-miss delete-file scan is derived prepare work; both common parent timers must + // contain it so the expensive miss cannot disappear between profile levels. + EXPECT_GE(prepare_delta, DELETE_FILE_DELAY_NS); + EXPECT_GE(total_delta, DELETE_FILE_DELAY_NS); EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); // The cached delete file contains entries for every referenced data file, so another split can diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index fbb3c79d5619b6..3c85eb4632f86c 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -23,11 +23,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include "core/assert_cast.h" @@ -60,6 +62,23 @@ namespace doris::format { namespace { +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + DataTypePtr table_type(const DataTypePtr& type) { return type->is_nullable() ? type : make_nullable(type); } @@ -763,6 +782,49 @@ TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) { ASSERT_TRUE(reader.close().ok()); } +TEST(PaimonHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileRangeDesc range) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range = std::move(range); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, + make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + verify_first_split(FileFormat::JNI, make_paimon_jni_range()); +} + TEST(PaimonJniReaderTest, BuildScannerParamsKeepsExplicitIOManagerTempDir) { auto scan_params = make_paimon_jni_scan_params(); scan_params.__set_paimon_options({ diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index d0096990357935..3e05a8dbf91264 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -327,9 +327,50 @@ class NullableArrayBigintDefaultExpr final : public VExpr { class TableReaderMaterializeTestHelper final : public TableReader { public: + using TableReader::_materialize_mapping_column; using TableReader::_materialize_map_mapping_column; }; +TEST(TableReaderTest, LastProjectionDetachesNestedMapWithoutCopyingStrings) { + auto keys = ColumnString::create(); + keys->insert_data(std::string(1UL << 20, 'k').data(), 1UL << 20); + auto values = ColumnString::create(); + values->insert_data(std::string(1UL << 20, 'v').data(), 1UL << 20); + const auto* original_value_bytes = values->get_chars().data(); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(1); + auto map = ColumnMap::create(std::move(keys), std::move(values), std::move(offsets)); + auto null_map = ColumnUInt8::create(1, 0); + ColumnPtr source = ColumnNullable::create(std::move(map), std::move(null_map)); + + const auto string_type = std::make_shared(); + const auto map_type = make_nullable(std::make_shared(string_type, string_type)); + Block block; + block.insert({source, map_type, "large_map"}); + source.reset(); + + ColumnMapping mapping; + mapping.global_index = GlobalIndex(0); + mapping.table_column_name = "large_map"; + mapping.file_column_name = "large_map"; + mapping.file_local_id = 0; + mapping.file_type = map_type; + mapping.table_type = map_type; + mapping.is_trivial = true; + mapping.projection = + VExprContext::create_shared(VSlotRef::create_shared(0, 0, -1, map_type, "large_map")); + TableReaderMaterializeTestHelper reader; + ColumnPtr detached; + ASSERT_TRUE(reader._materialize_mapping_column(mapping, &block, 1, &detached, + /*take_projection_result=*/true) + .ok()); + EXPECT_EQ(block.get_by_position(0).column->size(), 0); + const auto& detached_nullable = assert_cast(*detached); + const auto& detached_map = assert_cast(detached_nullable.get_nested_column()); + const auto& detached_values = assert_cast(detached_map.get_values()); + EXPECT_EQ(detached_values.get_chars().data(), original_value_bytes); +} + VExprSPtr table_int32_sum_expr(int left_slot_id, int left_column_id, int right_slot_id, int right_column_id) { const auto int_type = std::make_shared(); @@ -4027,6 +4068,7 @@ TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) EXPECT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); EXPECT_EQ(projection_ids(file_request.non_predicate_columns), std::vector({1})); + EXPECT_TRUE(file_request.predicate_only_columns.empty()); ASSERT_EQ(file_request.local_positions.size(), 2); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(0)).value(), 1); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(1)).value(), 0); diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index c2aea1cd825253..975f1e8eddd88b 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -54,6 +54,16 @@ io::FileCacheStatistics make_file_cache_stats(int64_t multiplier) { stats.inverted_index_io_timer = multiplier * 28; stats.remote_only_on_miss_triggered = multiplier * 29; stats.remote_only_on_miss_threshold_bytes = multiplier * 30; + stats.num_cross_cg_peer_io_total = multiplier * 31; + stats.bytes_read_from_cross_cg_peer = multiplier * 32; + stats.cross_cg_peer_io_timer = multiplier * 33; + stats.num_same_cg_peer_io_total = multiplier * 34; + stats.bytes_read_from_same_cg_peer = multiplier * 35; + stats.same_cg_peer_io_timer = multiplier * 36; + stats.num_peer_race_peer_win = multiplier * 37; + stats.num_peer_race_s3_win = multiplier * 38; + stats.num_peer_lazy_fetch = multiplier * 39; + stats.peer_lazy_fetch_timer = multiplier * 40; return stats; } @@ -94,6 +104,16 @@ void expect_file_cache_stats_eq(const io::FileCacheStatistics& actual, EXPECT_EQ(actual.remote_only_on_miss_triggered, expected.remote_only_on_miss_triggered); EXPECT_EQ(actual.remote_only_on_miss_threshold_bytes, expected.remote_only_on_miss_threshold_bytes); + EXPECT_EQ(actual.num_cross_cg_peer_io_total, expected.num_cross_cg_peer_io_total); + EXPECT_EQ(actual.bytes_read_from_cross_cg_peer, expected.bytes_read_from_cross_cg_peer); + EXPECT_EQ(actual.cross_cg_peer_io_timer, expected.cross_cg_peer_io_timer); + EXPECT_EQ(actual.num_same_cg_peer_io_total, expected.num_same_cg_peer_io_total); + EXPECT_EQ(actual.bytes_read_from_same_cg_peer, expected.bytes_read_from_same_cg_peer); + EXPECT_EQ(actual.same_cg_peer_io_timer, expected.same_cg_peer_io_timer); + EXPECT_EQ(actual.num_peer_race_peer_win, expected.num_peer_race_peer_win); + EXPECT_EQ(actual.num_peer_race_s3_win, expected.num_peer_race_s3_win); + EXPECT_EQ(actual.num_peer_lazy_fetch, expected.num_peer_lazy_fetch); + EXPECT_EQ(actual.peer_lazy_fetch_timer, expected.peer_lazy_fetch_timer); } } // namespace @@ -139,6 +159,10 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot EXPECT_EQ(profile->get_counter("CacheGetOrSetTimer")->value(), after_second_report.cache_get_or_set_timer); EXPECT_EQ(profile->get_counter("LockWaitTimer")->value(), after_second_report.lock_wait_timer); + EXPECT_EQ(profile->get_counter("CrossCGPeerIOTime")->value(), + after_second_report.cross_cg_peer_io_timer); + EXPECT_EQ(profile->get_counter("PeerLazyFetchTime")->value(), + after_second_report.peer_lazy_fetch_timer); } } // namespace doris diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index b05ea798ee46b0..82a3ba03667753 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -99,14 +99,135 @@ format-specific checklist when reviewing Parquet or ORC. - Keep index construction, predicate translation, cache lookup, and virtual-column setup out of per-row and repeated batch paths unless the work is inherently row-local. Avoid repeated schema traversal, expression cloning, metadata parsing, allocation, and conversion. +- Keep Profile counters in the visible `FileScannerV2 -> TableReader -> FileReader -> IO` hierarchy. + Format-specific readers, such as `ParquetReader`, belong below `FileReader`; lifecycle, metadata, + index, predicate, decode, materialization, and physical I/O paths must all have timers at the layer + that owns the work. Flush recursively aggregated child-reader statistics at every batch boundary, + including empty-selection and error exits, so a slow in-progress scan is diagnosable before close. - Require format readers to populate the common `ReaderStatistics` accurately where applicable: filtered/read row groups, Bloom and min/max pruning, filtered group/page/lazy rows, read rows and bytes, metadata/footer/cache timing, page-index work, predicate time, dictionary rewrite, and Bloom read time. +- Reject dead or ambiguous counters. In particular, `FilteredBytes` counts compressed bytes of + projected physical chunks avoided by pruning, not every child in the Row Group; footer read, + footer parse, lazy page-index materialization, and page-index predicate evaluation need distinct + timers. Raw I/O counters stay under `IO` even when a format reader initiates the request. - Evaluate performance with representative format versions, writers, data ordering, predicate selectivity, nested width, remote storage, batch sizes, and warm/cold caches. Report both the optimization overhead and the avoided work; a low pruning ratio alone is not a defect. +## Parquet Native Decode Boundary + +- V2 must instantiate only readers and decoders under `be/src/format_v2/parquet/`; calls into the + v1 `ParquetColumnReader` or edits under `be/src/format/parquet/` are review blockers. +- Footer parsing, schema-ID assignment, and the cached native metadata tree must also be v2-owned. + Reusing a stable base file identity is allowed, but require a v2 cache type discriminator so + v1/v2 metadata objects can never be cross-cast. Production planning must not retain or rebuild an + Arrow `FileMetaData` tree from the serialized footer. +- Trace the hot path as `ColumnReader -> Decoder span/cursor API -> DataTypeSerDe -> Doris Column`. + Decoder must not accept a Doris column or target type, and the path must not create Arrow arrays, + builders, `DecodedColumnView`, or another decoded leaf batch. +- Verify physical/logical metadata is immutable per leaf reader and complete for signed integers, + decimal precision/scale, date/time/timestamp units and UTC adjustment, INT96, UUID, FLOAT16, and + fixed-width binary. Unsupported combinations return explicit errors before plausible output. +- Treat legacy Parquet `TIMESTAMP_MILLIS` and `TIMESTAMP_MICROS` converted types as UTC-adjusted. + Do not give them the local/unspecified semantics of an unannotated INT64 timestamp; data decode, + statistics conversion, and min/max pruning must use the same timezone rule. +- Route plain, dictionary, and decoded timestamp inputs through one checked conversion contract. + Validate INT96 nanos-of-day before widened Julian-day arithmetic, reject unit scaling overflow, + and enforce Doris year 0001-9999 before materialization. Conversion failures must follow the same + strict/non-strict and dictionary-ID propagation rules as other direct types. +- Verify schema-change routing separately from physical decode. The reader must emit the projected + file type, while `ColumnMapper`/`TableReader` perform file-to-table casts after file predicates. + Any different requested type at the native reader boundary must fail as an invariant violation; + do not add a reader-local conversion column or decoder-facing conversion ABI. +- Dictionary review must separate dictionary-entry IDs from logical rows and non-null payload + ordinals. Materialize the typed dictionary once per generation through the same SerDe, validate + every index before access, and invalidate cached dictionary state at Row Group/file/type changes. + Dictionary-entry predicate evaluation and later row-value flattening must reuse that same typed + generation rather than serializing, parsing, or converting the dictionary twice. +- Check direct materialization for PLAIN, RLE/dictionary, DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Filtering must advance encoded + values without allocating output; null runs must append defaults without advancing payload. +- For predicate-only fixed-width PLAIN primitives, allow direct comparison only after proving the + whole Column Chunk uses compatible PLAIN value pages and every Expr advertises raw fixed-value + evaluation with identical Doris comparison semantics, including NaN ordering. The fallback + decision must precede definition-level consumption. Disable the direct path when a residual or + delete conjunct still references the hidden slot, because it needs the materialized payload. + Verify sparse input selection, interleaved NULLs, reversed literal comparisons, multiple ANDed + comparisons, mixed-encoding fallback, residual slot reuse, and a stable row-shaped placeholder. +- For a filtered scalar page fragment, require one SerDe entry and one batch-level selected-decode + dispatch. Nullable selections must first map logical rows to selected non-NULL physical ranges, + decode those ranges once, and restore NULL slots in place; falling back per NULL run is a review + blocker for a scalar destination that supports in-place expansion. Selection ranges belong to + persistent reader scratch; per-range virtual SerDe/decoder calls in the hot path are a review + blocker. Fixed PLAIN should + bulk-gather spans, BYTE_ARRAY PLAIN should scan lengths once, dictionary decode should validate + every ID before gathering selected IDs, and stateful encodings should batch-decode/reconstruct and + compact. Any remaining NULL-interleaving fallback must preserve logical output order without a + decoded intermediate column and be counted by `HybridSelectionNullFallbackBatches`. +- Review complex types as a level/shape problem around scalar leaf materialization. Parent offsets, + null maps, sibling alignment, page-spanning rows, and child payload counts must remain correct + without materializing an intermediate complex column. +- For MAP, the non-null key leaf owns the outer entry shape. Validate the materialized key/value + entry counts, but do not compare their raw repetition vectors: a nested value legitimately has + deeper levels. For STRUCT, compare each sibling only at the current parent boundary and ignore + repetition owned by a deeper child collection. +- Require a bounded high-water policy for persistent definition/repetition, null, selection, + conversion, dictionary-index, and decoder-owned scratch. Distinguish active bytes from retained + capacity: never release an oversized buffer while the current batch still needs it, and require + three ordinary/idle batches before releasing capacity above the high-water limit. Test both + outlier release and steady-state reuse so the policy does not create allocation thrash. +- Review all decoder read and skip paths as equally exposed corruption boundaries. Check requested + counts against remaining/declared values before pointer arithmetic or narrowing; use checked + addition/multiplication for byte extents; bound BYTE_ARRAY dictionary entry counts and IDs before + allocation/indexing; and require DELTA_BYTE_ARRAY prefixes to fit the previous reconstructed + value. BOOLEAN RLE and DELTA skip paths must consume bounded chunks and fail on short streams. +- Validate every decoded definition/repetition level against the schema maximum in batch, run, and + single-value cursor APIs. Page value counts must not drive eager nested scratch allocation; + reserve from the requested parent-row frontier and cover tiny-payload huge-count Page V1/V2 files. +- Before Snappy decompression, inspect the encoded uncompressed length and validate destination + capacity. Page V1, Page V2, and dictionary pages must produce exactly the declared decoded size; + an UNCOMPRESSED dictionary page must also declare equal compressed and uncompressed sizes. +- For a STRUCT whose projected children are all missing after schema evolution, require a + levels-only physical reference leaf. It must advance and validate encoded payload cursors while + deriving the synthetic child count, without constructing a discarded string/complex column. +- `CountColumnReader` must use the native levels-only reader and must not decode payload or call + Arrow `ReadRecords`. Require profiles that distinguish page I/O, decompression, level decode, + value decode, SerDe materialization, hybrid selection batches/ranges/NULL fallback, + filtered-value skips, and page fragmentation. +- Validate every signed Column Chunk offset/length before converting it to `size_t`. The dictionary + offset is usable only when it is non-negative and precedes the data offset; the complete range + must fit the file. Apply the PARQUET-816 tail padding only to affected parquet-mr versions, cap it + at 100 bytes, and keep it inside the file. Scalar and levels-only COUNT readers share this helper. +- Treat OffsetIndex as one optional, all-or-nothing navigation structure. Require first row zero, + the first physical location to equal the owning ColumnMetaData `data_page_offset`, strictly + increasing row ordinals and physical offsets, positive sizes, non-overlapping page ranges, and + containment in the owning Column Chunk. Discard a malformed index before selecting the indexed + reader. +- Page iteration skips `INDEX_PAGE` and unknown auxiliary pages before initializing a data decoder. + Dictionary pages retain their special first-page handling; a later dictionary page is corrupt. +- Derive writer workarounds once from `created_by` and pass them through scalar, nested, page-cache, + and COUNT paths. Pre-Arrow-3 parquet-cpp Data Page V2 payloads remain compressed despite the + historical `is_compressed=false` flag. +- Preserve nullable conversion semantics in direct native materialization. Numeric, DATE, + DATETIME, TIME, and DECIMAL failures insert a default nested value and mark the corresponding NULL + only in non-strict mode; strict or non-nullable reads return the error. Dictionary failures follow + the selected dictionary IDs to output rows. +- Keep Parquet decimals in a source-width or wider intermediate until exact scaling, target + precision, and overflow checks succeed. Scale-down with a non-zero remainder is a conversion + failure; plain and dictionary integer/binary paths must narrow only afterward. +- For cold small-file tests, separate footer I/O/Thrift parse from native schema and index planning. + V2 must not retain serialized footer bytes after the native metadata tree is initialized; v1 opens + remain independent. +- For HTTP Parquet objects at or below `in_memory_file_size`, v2 stages the complete object from + byte zero before native page access. Verify both cold and footer-cache-hit scans: some Range + servers accept the capability probe but return HTTP 200 for a near-EOF overlong range, so warm + scans must not depend on the footer read having populated an incidental transport buffer. +- Identical fixed-width POD values append with one bulk copy. FIXED_LEN_BYTE_ARRAY strings copy the + dense byte span once and synthesize offsets; validate this execution contract without flaky + wall-clock assertions. + ## Parquet Multi-Level Filtering - Use [FileScannerV2 Parquet Scan Design](file-scanner-v2-parquet-scan-design.md) as the detailed @@ -129,12 +250,37 @@ format-specific checklist when reviewing Parquet or ORC. - Verify lazy materialization avoids reading and decoding non-predicate columns for rejected rows while advancing all readers correctly. Predicate columns should be read/prefetched first; output prefetch should wait for survivors when filtering is active. +- For safe staged single-column predicates, review the observed cost/rejection ordering and its + cold-start behavior. Reordering is allowed only after every candidate has a sample; prefetch may + stop at a low-probability reach prefix, while output-column prefetch may start early only after a + learned high survival ratio. Static conjunct schedules and position maps belong to the scanner + lifecycle; after eight warm-up samples, per-predicate clocks should be sampled only periodically. + Cache the batch SelectionVector's dense bitmap by generation so a wide lazy projection does not + rebuild the same O(batch) filter for every column. +- Single-column predicate rounds may keep previously read columns in their original row mappings. + Require one alignment compaction before multi-column, delete, or output boundaries, and expose + its time, bytes, and count instead of charging repeated movement invisibly to predicate time. + Reused compaction masks must be fully cleared when coordinate-space sizes shrink. +- PLAIN BYTE_ARRAY must not parse lengths in both decoder and destination column. Review the direct + payload-offset/cumulative-offset contract, uint32 overflow checks, surviving-span coverage, and + the legacy consumer fallback for non-string logical types. +- Production PageIndex planning must consume native Compact Thrift ColumnIndex/OffsetIndex objects. + Coalesce adjacent serialized index ranges and transfer validated OffsetIndexes into execution so + the Row Group does not read them twice. Arrow PageIndexReader is a test oracle; a production Arrow + metadata adapter or rebuilt `FileMetaData` tree is a review blocker. +- Fixed-width conversion fast paths may remove row branches only when the source domain provably + fits the target domain. Narrowing, timestamp, decimal scaling, strict rollback, and non-strict + NULL marking must retain corrupt-value tests. +- Recursive reader-profile publication and retained-scratch inspection should be amortized, but Row + Group reset/EOF/close must force the final profile delta before reader destruction. +- Accumulated lazy-column skips must not allocate one dense byte per rejected prefix row. Require a + fixed chunk bound (including multiple lazy columns) or a level-only/all-filtered cursor contract. - Register Parquet Page Cache ranges only for surviving projected Column Chunks, require a stable file-version key, and assess FileCache, MergeRange, prefetch, requests, and read amplification together. - Require counters for Statistics/Dictionary/Bloom pruning, Page Index selected ranges and skipped - rows/pages, raw and filtered rows, dictionary-row filtering, lazy-read savings, cache sources, and - remote I/O. + rows/pages, raw and filtered rows, dictionary-row filtering, PLAIN direct-predicate batches/rows, + lazy-read savings, cache sources, and remote I/O. - Differential tests must cover absent/invalid statistics, missing or partial Page Index, mixed dictionary/plain encoding, Bloom false positives, NULL/NaN/type conversion, cross-Page batches, nested/repeated columns, multiple Row Groups/Splits, and all/none filtered. diff --git a/docs/file-scanner-v2-design.md b/docs/file-scanner-v2-design.md index 66b96de1204d14..b2fe43ae76f6b0 100644 --- a/docs/file-scanner-v2-design.md +++ b/docs/file-scanner-v2-design.md @@ -282,6 +282,23 @@ Scan optimization remains maintainable only when costs are visible, sources are and failure semantics are explicit. V2 provides three complementary views: Query Profile, query resource context, and global metrics. +Query Profile uses one stable ownership tree: + +```text +FileScannerV2 +└── TableReader + └── FileReader + ├── format-specific reader (ParquetReader, OrcReader, ...) + └── IO +``` + +Scanner lifecycle and Split scheduling are charged to `FileScannerV2`; table-schema restoration, +delete handling, and reader lifecycle are charged to `TableReader`; metadata, index, decode, and +materialization are charged to `FileReader` and its format subtree; physical reads, bytes, cache +waits, and remote/local attribution remain under `IO`. Every executable lifecycle path needs a +timer, and cumulative child-reader statistics must be published at batch boundaries as well as +close, so an active slow query never presents an unexplained timing gap. + ```mermaid flowchart LR R[FileReader and FileCache Raw Statistics] --> P[Query Profile] diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 63f63819670dd2..70c041a89ccd48 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -25,8 +25,9 @@ predicate columns for surviving ranges, and finally defer output-column reads un - **Layered caches:** File-block cache, Parquet page cache, condition-result cache, and merged small I/O solve different problems and are not interchangeable. -**Scope:** This document focuses on the FileScannerV2 Parquet Reader design and core pipeline. It -does not cover Arrow decoder internals, complex-type reconstruction, or expression implementation. +**Scope:** This document covers the FileScannerV2 Parquet Reader pipeline, the native page/decode +kernel, selection-aware materialization, and complex-type reconstruction contracts. Expression +implementation details remain outside its scope. ## 2. Overall Architecture @@ -40,9 +41,10 @@ flowchart TB B --> C[TableReader
Schema Mapping, Partition/Default Values, Predicate Localization] C --> D[ParquetReader
Footer/Schema and Row Group Scan Planning] D --> E[ParquetScanScheduler
Row Group Lifecycle and Batch Reads] - E --> F[ParquetColumnReader
Page Skipping, Decompression, Decoding, Materialization] - F --> G[ParquetFileContext / Arrow RandomAccessFile
Page Cache, MergeRange, Prefetch] - G --> H[Doris FileReader / FileCache / Remote FS] + E --> F[ParquetColumnReader
Persistent Column State and Complex Reconstruction] + F --> G[Native ColumnChunk / Page / Encoding Decoders
Selection-aware Doris Materialization] + G --> H[ParquetFileContext / Stream Adapter
Page Cache, MergeRange, Prefetch] + H --> I[Doris FileReader / FileCache / Remote FS] ``` | Layer | Core responsibilities | Responsibilities intentionally excluded | @@ -58,6 +60,64 @@ flowchart TB > layer can use footer, page index, dictionary, and other format knowledge while upper layers retain > uniform scan semantics. +### 2.1 Native Reader Target and Migration State + +The target execution path does not use Arrow builders or `ReadRecords` to decode data pages. Doris +owns the Column Chunk, page, level, encoding, selection, conversion, and materialization state, and +writes directly into Doris columns. This follows the same broad separation used by DuckDB: metadata +planning is distinct from a persistent per-column reader, and page/encoding decoders expose narrow +cursor-based contracts rather than an Arrow array as an intermediate result. + +V1 remains the differential baseline, but v2 owns an independent reader implementation. Its page +and encoding algorithms started from proven Doris behavior and are maintained under the v2 tree; +the production v2 path never instantiates the v1 `ParquetColumnReader`: + +| Stage | State and boundary | +| --- | --- | +| Native encoding kernel | V2 owns the Column Chunk, page, level, and encoding decoders under `be/src/format_v2/parquet/reader/native/`. Decoders expose raw fixed/binary spans, validated dictionary indices, and skip operations; they never write Doris columns. | +| Logical materialization | `DataTypeSerDe` interprets Parquet physical/logical metadata and writes decoded spans directly into the final Doris column. Typed dictionaries and scratch are persistent per leaf reader. | +| Native column reader | `NativeColumnReader` is persistent for one top-level column and Row Group. It owns selection/filter/dictionary scratch and drives the native decoder directly into the final Doris column. | +| Complex reconstruction | Choose a Dremel shape-owning leaf per requested parent-row range; derive parent offsets/nulls once from it and keep sibling leaf streams aligned to the same parent rows. | +| Metadata and planning | Replace Arrow footer/schema/Row Group metadata dependencies with native Thrift-derived objects while preserving the existing planner, index, cache, and split contracts. | +| Compatibility removal | Production v2 has no Arrow data-read or metadata adapter and never falls back to the v1 reader; unsupported combinations return explicit errors. Arrow remains only in test oracles and fixture writers. | + +Data-page value scans and levels-only aggregate scans no longer use Arrow `RecordReader`, arrays, or +builders. V2 parses and owns the Thrift footer, schema parser, field-ID assignment, physical schema, +statistics, dictionary metadata, bloom filters, and page indexes. Production planning never builds +an Arrow metadata tree; Arrow is limited to fixture writers and differential test oracles. + +All new integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the correctness +and performance control. Compatibility is demonstrated through differential tests and the explicit +type/encoding matrix, not through a runtime dependency on v1 code. + +### 2.2 Native Interface Ownership + +```mermaid +flowchart LR + A[Native Footer / Physical Schema] --> B[RowGroupReadPlan] + B --> C[Persistent ParquetColumnReader] + C --> D[Persistent ColumnChunkReader] + D --> E[Page Reader + Level Decoders] + E --> F[Encoding Decoder] + F --> G[ColumnSelectVector] + G --> H[Doris MutableColumn] + C --> I[Shared Complex Level Plan] + I --> G +``` + +- **Physical schema contract:** Immutable physical type, fixed length, maximum definition and + repetition levels, and repeated-parent definition threshold. It owns no Arrow descriptor and + borrows no temporary table-schema object. +- **Persistent column state:** Page/decompress buffers, dictionary decoder, SerDe/conversion state, + null maps, selection ranges, binary-value references, and builder capacity live with the column + reader and are logically reset rather than recreated for each batch. +- **Decoder contract:** Consume a known number of logical level entries and encoded payload values, + then materialize only selected rows. Decoders do not decide table projection, predicate meaning, + or parent complex offsets. +- **Complex plan contract:** The shape-owning leaf's definition/repetition levels are parsed once + into parent-row boundaries and child-presence/null decisions. Sibling physical streams consume + the same parent-row range and validate their counts so offsets and null maps cannot drift. + ## 3. From File Open to Scan Plan After a reader receives a Split, it opens the file and builds the scan plan. This phase determines @@ -75,7 +135,7 @@ sequenceDiagram FS->>TR: prepare/open split TR->>TR: Map schema and localize predicates TR->>PR: FileScanRequest - PR->>FC: Open FileReader + PR->>FC: Open FileReader / native stream adapter FC->>META: Read footer and schema META-->>PR: Row Group / Column Chunk metadata PR->>PLAN: Candidate Row Groups and local predicates @@ -94,8 +154,17 @@ sequenceDiagram delete conjuncts, and local column-position mappings. - **RowGroupReadPlan:** Records the Row Group, its file-global starting row, `selected_ranges` produced by page-index pruning, and the `page_skip_plan` for each leaf column. -- **ParquetFileContext:** Adapts Doris FileReader to Arrow RandomAccessFile and owns Page Cache, - FileCache prefetch, and MergeRange routing. +- **ParquetFileContext:** Adapts Doris FileReader to the native metadata/data stream interface and + owns Page Cache, FileCache prefetch, and MergeRange routing. Its immutable footer cache entry owns + the V2 `NativeFieldDescriptor` tree directly; repeated V2 opens neither re-read the footer nor + serialize and parse another metadata representation. + +For every selected Row Group, native dictionary/index probes finish on the cached metadata tree first. The +scheduler then computes projected physical Column Chunk ranges and installs one shared native +`MergeRangeFileReader` when their average size is below v1's small-I/O threshold. All predicate and +lazy output readers share that wrapper; their internal `BufferedFileStreamReader` prefetch is +disabled in this mode, avoiding duplicate buffers. Large chunks and in-memory files use the base +reader, while remote FileCache prefetch remains the non-MergeRange path. > Planning intentionally proceeds from cheap to expensive. Split and metadata pruning reduce the > candidate set before finer indexes are read for surviving Row Groups, avoiding index I/O for data @@ -279,6 +348,258 @@ flowchart LR > output columns must decode and copy. This is the main benefit of lazy materialization in a > columnar format. +### 7.1 Selection and Cursor Contract + +The native decoder receives a sorted selection over logical rows. Logical positions include nulls; +they are not offsets into the encoded non-null payload. Definition levels and selection are merged +as two ordered streams into four run types: + +| Run | Consume logical level entries | Consume encoded payload | Append output | +| --- | --- | --- | --- | +| Selected non-null (`CONTENT`) | Yes | Yes | Value | +| Selected null (`NULL_DATA`) | Yes | No | Default plus null-map bit | +| Filtered non-null (`FILTERED_CONTENT`) | Yes | Yes or decoder-native skip | No | +| Filtered null (`FILTERED_NULL`) | Yes | No | No | + +The contract keeps three counts explicit: logical level entries consumed, encoded non-null values +consumed, and Doris output values appended. The page reader may cross page boundaries while +satisfying one batch, but all three counts must be aligned when it returns. A dense identity +selection, an empty selection, and an arbitrary fragmented selection use the same decoder API. +Selection inputs are borrowed only for the duration of the decode call. + +For a flat leaf, the fast path is valid only when the maximum repetition level is zero. It decodes +definition-level runs and builds the four-way selection plan in persistent scratch whose capacity +survives adaptive batch changes. The plan is normalized to sorted physical non-NULL ranges and +enters `DataTypeSerDe` and the encoding decoder once. If selected NULLs are interleaved, the reader +separately records their logical positions, decodes the compact physical payload, and expands the +final scalar column backwards in place. This is the hybrid selection path: it keeps the dense +direct-materialization path, but moves sparse range traversal inside the concrete decoder instead +of repeatedly constructing a SerDe consumer for every selected or NULL run. + +The encoding chooses the cheapest inner loop. PLAIN fixed-width values gather ranges with bulk +copies; PLAIN BYTE_ARRAY scans every length once and publishes compact source offsets, cumulative +output offsets, and coalesced surviving spans directly to the SerDe; dictionary encoding validates +IDs in bounded batches and either gathers repeated/literal runs directly or builds one compact +selected-ID batch when a sparse lookup would exceed the L2 cache; BOOLEAN, +DELTA, and BYTE_STREAM_SPLIT batch-decode or reconstruct their stateful stream and compact selected +values. This follows DuckDB's vector-at-a-time principle while preserving Doris's separate +Decoder/SerDe ownership boundary. A filtered non-null value always advances and validates encoding +state even when it is not copied, preventing the next batch from decoding shifted values. + +Scalar destinations with in-place expansion no longer use the per-run NULL fallback. The fallback +remains only for unsupported destination shapes and is visible through +`HybridSelectionNullFallbackBatches`; it is a correctness boundary, not an Arrow fallback. + +### 7.2 Page and Encoding Kernel + +The native page reader parses Page V1 and Page V2 headers, obtains level streams and payload bytes, +decompresses only the required region, and installs a decoder selected from physical type plus data +encoding. Page V1 and V2 differ in where levels are stored and which bytes are compressed; after +that parsing step both feed the same level and value-decoder contracts. + +| Encoding family | Native responsibility | +| --- | --- | +| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; identical POD types append by bulk copy, dense fixed-length strings use one byte-span copy plus offset synthesis, sparse fixed-width ranges consume contiguous spans directly, and variable strings reuse decoder-produced offsets without a `StringRef[]` staging pass | +| RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, validate IDs in bounded batches, fuse RLE runs with direct gather for cache-resident dictionaries, and use one compact selected-ID gather for large sparse dictionaries | +| RLE / BIT_PACKED levels | Decode definition/repetition levels and preserve runs across page and batch boundaries | +| DELTA_BINARY_PACKED | Preserve block/mini-block state, decode one page-fragment batch, and compact selected values in-place | +| DELTA_LENGTH_BYTE_ARRAY | Decode lengths and byte payload in lockstep, then retain selected references only | +| DELTA_BYTE_ARRAY | Reconstruct prefix/suffix values with persistent previous-value state, then retain selected references only | +| BYTE_STREAM_SPLIT | Reassemble selected primitive lane ranges directly into one compact batch without an Arrow intermediate | + +Unsupported physical-type/encoding combinations return an explicit error. They never fall back to +Arrow and never produce a plausible result through a decoder selected only by logical Doris type. +Dictionary-to-plain transitions, multiple data pages, Page V1/V2, truncated +payloads, integer overflow, and invalid lengths/IDs are part of the unit-test matrix. + +Read and skip are the same trust boundary. Every decoder validates the requested count against its +declared/remaining values before pointer advancement, allocation, multiplication, or integer +narrowing. BYTE_ARRAY dictionaries bound the entry count by the available four-byte length prefixes +before reserving storage and validate every decoded ID. DELTA_BYTE_ARRAY requires each prefix to fit +the preceding reconstructed value and checks the reconstructed and aggregate byte lengths. BOOLEAN +RLE and DELTA skip paths operate in bounded chunks and reject short streams instead of advancing a +partially consumed cursor as if the request succeeded. + +Level bit width is only a storage bound: for a schema maximum of 2, two bits can still encode the +invalid value 3. Batch, run, and single-value level cursors therefore reject every decoded value +above the schema maximum. Nested page traversal reserves only the requested parent-row frontier; +an untrusted Page V1/V2 `num_values` cannot trigger an eager page-sized allocation before the level +payload proves those entries exist. + +Compression has an exact-size contract. Snappy's encoded uncompressed length is checked against the +destination capacity before decompression; Page V1, Page V2, and dictionary decode must then produce +exactly the size declared in the page header. For the UNCOMPRESSED codec, dictionary compressed and +uncompressed sizes must be equal. These rules turn malformed size metadata into corruption instead +of a buffer overrun, silent truncation, or plausible shifted values. + +Before a stream is created, scalar and levels-only readers call one signed, file-bounded Column +Chunk range validator. Writer compatibility derived from `created_by` may add at most 100 bytes of +file-bounded PARQUET-816 padding, or override the pre-Arrow-3 Data Page V2 compressed flag. Page +iteration ignores auxiliary `INDEX_PAGE`/unknown pages without consuming a logical data-page +ordinal. OffsetIndex is published only when its rows and non-overlapping physical page ranges are +strictly increasing, its first location equals the owning `data_page_offset`, and every location +remains inside that validated Column Chunk. The metadata anchor rejects uniformly shifted indexes +that row monotonicity alone cannot detect; otherwise sequential traversal preserves correctness. + +### 7.3 Direct Materialization and Scratch Reuse + +Encoding decoders expose contiguous physical spans and advance encoded-stream cursors. The selected +`DataTypeSerDe` consumes those spans and writes directly into the Doris mutable column. Fixed-width +types append contiguous runs. PLAIN BYTE_ARRAY publishes payload offsets, cumulative destination +offsets, and coalesced survivor spans so string columns perform larger range copies without a +`StringRef[]` staging array. Other binary encodings may use persistent references that remain valid +only while the page or dictionary buffer is pinned by the persistent leaf reader. + +The direct path materializes the projected file type. Table-schema changes, including numeric +widening, decimal precision/scale changes, and string/date conversions, are represented by +`ColumnMapper` and executed by `TableReader` after file-local predicates finish. The native reader +rejects a different requested type because accepting it would move schema evolution into physical +decode and could change predicate semantics. No intermediate physical-value batch is introduced. + +`DecodedColumnView` is not the native Parquet decoder output ABI. It describes already decoded +physical values and is useful to generic format conversion code, but routing every Parquet value +through it would recreate an intermediate materialization layer. The v2 native path does not use it: +ColumnReader handles levels/selection, Decoder parses encoding streams, and DataTypeSerDe performs +physical/logical conversion while appending to the final column. + +Decimal and FIXED_LEN_BYTE_ARRAY direct paths validate the physical byte width, decode big-endian +two's-complement values with correct sign extension, and apply precision/scale conversion exactly +once. Decimal integer and binary inputs stay in a 256-bit intermediate through exact scale-down, +scale-up overflow, and target-precision checks; discarded non-zero digits are conversion failures, +and narrowing happens only after success. Date, timestamp, INT96, unsigned annotations, +CHAR/VARCHAR, and timezone conversions retain +the same semantic checks as the general conversion path. A fast path is enabled only when those +checks prove the result is equivalent. In particular, legacy converted `TIMESTAMP_MILLIS` and +`TIMESTAMP_MICROS` are UTC-adjusted, while an unannotated INT64 timestamp has distinct +local/unspecified semantics; data decode and statistics pruning share this interpretation. +Timestamp conversion validates millisecond scaling before multiplication, validates INT96 +nanos-of-day before widened Julian-day arithmetic, and rejects values outside Doris years +0001-9999. Plain, dictionary, and decoded-value inputs share these bounds. + +Direct conversion also preserves load-mode error semantics. A nullable target in non-strict mode +stores the nested default and marks the exact output row NULL for numeric, date/time, timestamp, and +decimal conversion failures. Strict mode and non-nullable targets return the error. Typed +dictionary materialization records failing dictionary entries once and propagates those failures +through decoded dictionary IDs, so dictionary and plain pages have identical behavior. + +The typed dictionary is materialized through the logical SerDe once per decoder dictionary +generation. Dictionary-entry predicate evaluation, dictionary-ID row filtering, and surviving +row-value flattening reuse that same Doris column. A Row Group, file, type, or dictionary-generation +change invalidates it, and every ID is checked before access. Cache-resident or dense reads stream +validated RLE runs directly into the destination column without a page-sized ID vector. Sparse +reads whose dictionary exceeds L2 retain only selected IDs, then perform one gather; malformed late +IDs roll back the destination to its pre-batch size. + +The persistent leaf reader owns reusable conversion objects, null map, selection ranges, +definition/repetition levels, binary references, dictionary state, decompression buffers, and Doris +column capacity. Logical sizes are reset at batch boundaries and normal-size capacity is retained. +After the top-level complex reader has consumed the level plan, retained capacity above the 4 MiB +high-water limit becomes eligible for release. The reader accounts separately for active and +retained bytes, including decoder-owned buffers: active oversized scratch is never released, and +eligible capacity is released only after three consecutive ordinary/idle batches. This hysteresis +prevents a repeated-value outlier from being pinned for the Row Group without turning a legitimate +large steady-state batch into allocate/free thrash. The native path does not create an Arrow +builder or Arrow array. + +### 7.4 Complex Types and Parent Shape Plans + +Repeated Parquet leaves cannot interpret a requested parent-row count as a leaf-value count. One +parent row may contain zero, one, or many level entries, and its final entry can reside on the next +page. The complex reader therefore builds a parent shape plan from one owning leaf with: + +- parent-row start/end boundaries derived from repetition levels; +- ancestor-null, collection-null, empty-collection, element-null, and present-value decisions from + definition thresholds; +- child payload positions and selected parent rows; +- cross-page continuation state for an unfinished parent row. + +ARRAY and MAP offsets and null maps are derived from that plan. Parquet stores separate level +streams for separate physical leaves, so sibling readers still consume their own streams; they must +advance over the same parent-row range and validate their payload counts instead of redefining the +parent shape. STRUCT uses a representative present leaf for its null map and parent count. MAP uses +the key leaf as the entry-shape owner, requires the materialized key and value columns to match that +outer entry count, and validates Parquet's non-null key requirement rather than repairing it. Raw +key/value repetition vectors are intentionally not compared because a nested MAP value owns +additional repetition levels. STRUCT siblings similarly normalize repetition to the current parent +boundary; deeper child collection repetition cannot redefine or invalidate the sibling shape. + +Level scratch grows with level entries that were actually decoded, while its initial reservation is +bounded by the requested parent-row frontier rather than the page header's untrusted value count. +Long repeated rows can still grow incrementally beyond that frontier, and long null/non-null runs +are split into representable internal runs without introducing a new row boundary. Tests cover null +ancestors, empty collections, null elements/values, nested +STRUCT-in-ARRAY and ARRAY-in-STRUCT shapes, sibling page misalignment, and rows spanning pages and +batches. + +#### Complex-reader interface and materialization cost + +The original v2 prototype placed an Arrow-decoded leaf batch and separate load/build/consume phases +between encoded pages and Doris columns. That design required implicit phase ordering, duplicated +level traversal in ARRAY/MAP/STRUCT wrappers, and retained decoded binary payload even when a caller +needed only nullability. The prototype hierarchy and its batch container have been removed. + +The production boundary is now the same compact cursor contract used for scalar columns: +`NativeColumnReader::read/select/skip`. Its persistent native reader owns page, decoder, level, +conversion, and complex-column state and materializes the complete result directly into the caller's +Doris column. No Arrow value reader, intermediate decoded leaf container, temporary nested Doris +column, or public load-before-build protocol participates in predicate or output scans. + +Internally, complex decoding still has to solve the parent-shape problem. For example, levels +representing `[["a", "b"], NULL, []]` produce entry counts `[2, 0, 0]`, parent nulls `[0, 1, 0]`, +and string payload ordinals `[0, 1]`. MAP uses the key leaf as the entry-shape owner and validates +the value leaf against it; STRUCT children advance in parent-row lockstep. This state belongs behind +the native reader boundary rather than in caller-visible phases. + +When schema evolution makes every projected STRUCT child missing, the native reader retains one +physical reference leaf solely for shape. Its levels-only interface advances definition, +repetition, and encoded payload cursors (including dictionary-index validation), counts STRUCT +instances from the parent thresholds, and discards the payload without allocating a temporary +string or nested Doris column. + +`CountColumnReader` selects one representative leaf (the key for MAP) and uses the v2 native +`LevelReader` to consume definition/repetition levels without decoding payload values. It exposes +neither decoded values nor the ordinary scan-reader API, so COUNT pushdown cannot become a value +fallback and large complex payloads never enter aggregate state. + +Doris v1 remains the behavior/performance baseline: `read_column_data()` owns physical decode and +the collection reader consumes persistent level buffers. DuckDB provides the same useful design +principle through its single `Read(input, vector)` boundary and reusable child vectors. See DuckDB's +official [LIST reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/list_column_reader.cpp), +[STRUCT reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/struct_column_reader.cpp), +and [base column reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/column_reader.cpp). + +### 7.5 Index Coordinate Domains and Composition + +Index correctness depends on keeping its coordinate systems separate: + +| Identity | Scope | Must not be confused with | +| --- | --- | --- | +| Table/local column ID | Current file request and Doris block | Physical Parquet leaf ordinal | +| Physical leaf-column ID | Footer Row Group Column Chunk array | Logical parent STRUCT/ARRAY/MAP ordinal | +| Row Group ID | File metadata | Split ordinal or batch ordinal | +| Data-page ordinal | One Column Chunk, excluding dictionary pages | PageHeader sequence including dictionary page | +| OffsetIndex row ordinal | Logical row inside a Row Group | Encoded non-null value position | +| Selection index | Logical row inside the current batch, nulls included | Dictionary ID or compact output position | +| Dictionary-entry ID | One Column Chunk dictionary | Row ordinal, global dictionary ID, or the next Row Group's dictionary | + +Row Group statistics, dictionary pruning, Bloom, ColumnIndex/OffsetIndex, page skip plans, cache +ranges, row selection, and lazy predicate/output readers are composed in that order. Each stage may +only remove candidates already expressed in the same Row Group logical-row domain. Page ordinals +from ColumnIndex and OffsetIndex are validated together before they become `selected_ranges` and a +per-leaf physical page-skip plan. + +Dictionary row filtering starts only when metadata proves every data page in the Column Chunk uses +a dictionary encoding. The predicate is evaluated against the current dictionary to produce an +entry bitmap; decoded IDs are checked against both dictionary length and bitmap length. A mixed +dictionary/plain transition falls back before consuming data. Once selected dictionary reading has +advanced a page cursor, loss of dictionary output is corruption rather than a retry through another +path with shifted state. + +Missing optional indexes or unsupported predicate/type combinations retain rows. Malformed +offsets, inconsistent page counts, out-of-range dictionary IDs, overlapping/unsorted invalid ranges, +or an impossible cursor relationship are reported as corruption. Cache hits and misses do not +change any page, level, value, or dictionary cursor. + ## 8. Supported Indexes and Their Boundaries V2 uses native Parquet metadata and encoding information. It does not construct Doris-internal @@ -315,12 +636,12 @@ flowchart TD ## 9. Cache and I/O Optimization -Parquet V2 has four complementary cache and I/O paths: cache remote file blocks, cache serialized -Parquet ranges, cache predicate results, and merge small random reads. +Parquet V2 has five complementary cache and I/O paths: cache footer/parsed metadata, cache remote +file blocks, cache serialized Parquet ranges, cache predicate results, and merge small random reads. ```mermaid flowchart TB - A[Parquet Column Reader ReadAt] --> B{Parquet Page Cache Hit?} + A[Parquet Column Reader Range Read] --> B{Parquet Page Cache Hit?} B -- "Yes" --> C[Return Cached Serialized Range Bytes] B -- "No" --> D{MergeRange Active?} D -- "Yes" --> E[MergeRangeFileReader
Merge Adjacent Small I/O] @@ -334,11 +655,46 @@ flowchart TB | Mechanism | Cached or optimized object | Lifecycle and key | Problem addressed | | --- | --- | --- | --- | +| Footer metadata cache | V2-owned immutable Thrift metadata and native physical schema tree | Stable base file identity plus a v2 type discriminator and schema-affecting mapping options | Avoid repeated footer I/O, Thrift parsing, schema construction, and unsafe cross-version cache casts | +| Small HTTP object staging | Complete object bytes for files at or below `in_memory_file_size` | Per-reader v2 wrapper; loaded once from byte zero and released with the file context | Collapse cold small-file requests and keep footer-cache-hit scans independent of server-specific near-EOF Range behavior | | FileCache | Remote file blocks | Related to filesystem/path and file version; may hit locally or through a peer | Avoid repeated object-storage access and support background prefetch | | Parquet Page Cache | Serialized bytes within registered Column Chunk ranges | Stable file key depends on path, mtime/version, and file size; disabled when mtime is unreliable | Reduce repeated page reads and support exact/subrange coverage | | Condition Cache | Condition-surviving granule bitmap | Managed by condition and file-range context | Reuse filtering results before reading columns | | MergeRangeFileReader | Not a cache; merges small ranges into larger slices | Installed temporarily for projected chunks of the current Row Group | Reduce remote small-I/O count and request overhead | +### Footer Cache Parity with V1 + +V2 reuses the common stable file identity policy, but owns its footer parser, schema lifecycle, +metadata payload type, and cache-key suffix entirely under `format_v2`. A v2 cache entry can be +shared by v2 readers but can never be cast as a v1 metadata object. Mutable Row Group, selection, +decoder, and scratch state remains per reader. Path-only keys are insufficient. Parse failures, +short files, unsupported metadata, and schema-affecting option changes cannot populate or reuse a +successful entry. No change to the v1 parser or metadata cache behavior is required. + +On a v2 cold miss, the parser deserializes the footer once into the V2-owned Thrift object and builds +the native schema tree directly from `SchemaElement` into V2's `NativeFieldDescriptor`. Row-group statistics, +dictionary and Bloom metadata planning consume the same native tree; no Arrow `FileMetaData` tree +or serialized-footer copy is retained. Production ColumnIndex/OffsetIndex planning reads and parses +Compact Thrift indexes natively; adjacent per-leaf serialized ranges are coalesced, and validated +OffsetIndex objects are transferred into Row Group execution instead of being fetched a second +time. Arrow metadata and PageIndexReader paths remain only as test oracles. + +Before the footer lookup, v2 wraps bounded HTTP Parquet objects in an in-memory reader. The wrapper +loads from byte zero on its first physical access, whether that access is a cold footer read or a +warm data-page read after a footer-cache hit. This preserves identical cold/warm behavior for HTTP +servers that advertise Range support but answer an overlong near-EOF range with HTTP 200, and keeps +the compatibility policy out of the v1 reader. + +### Page Cache Parity with V1 + +The native path uses v1's page-cache semantics as its minimum contract: the same stable file +identity, Column Chunk/page byte ranges, compressed versus decompressed entry distinction, checksum +and decompression ownership, subrange coverage, invalidation, admission, and fallback I/O rules. +Mutable decoder and dictionary state is never cached. A hit returns immutable bytes and advances the +reader exactly as the corresponding base read would; a miss performs normal I/O before optional +admission. An alternative policy is accepted only with correctness coverage plus warm/cold, +selective/dense, local/remote benchmarks showing it is no worse than v1. + ### Why Page Cache registers only surviving chunks The footer is read before Row Group planning and before Page Cache ranges are registered, so @@ -349,8 +705,10 @@ Chunks from surviving Row Groups are registered, limiting pollution and key coun - When the base reader is CachedRemoteFileReader, predicate/output ranges for the current Row Group may be prefetched into FileCache. -- When average projected chunks are small and the reader is not in-memory, install - MergeRangeFileReader so subsequent Arrow `ReadAt` calls actually use merged reads. +- After dictionary probing, small projected chunks share one Row-Group-scoped + MergeRangeFileReader on the native data-page path. Large chunks and in-memory files keep the base + reader. Native metadata/index reads use explicit immutable ranges and never own a decoder stream + cursor. - With row-level filters, prefetch predicate columns first. Prefetch non-predicate columns only after at least one row survives, avoiding unnecessary bandwidth. @@ -396,6 +754,14 @@ flowchart LR E --> F[Bound by batch_size and Selected Range] ``` +Changing the requested row cap changes only the amount of work performed by the persistent reader. +It does not recreate Column Readers, Column Chunk readers, dictionaries, SerDe/conversion objects, +builders, or scratch. The probe is charged as a normal batch and estimates completed Doris rows and +bytes after table-level materialization, matching v1's measurement point. Empty or highly filtered +probes do not permanently collapse the batch size; the scheduler retains enough evidence before +updating its estimate and respects page/range boundaries without turning each fragment into a new +reader lifecycle. + ### 10.3 Aggregate Pushdown When TableReader proves that no filter or delete semantics can change the result, COUNT / MIN / MAX @@ -408,6 +774,34 @@ Without row-level filtering, output columns may be warmed together. With filteri columns first and defer non-predicate columns until at least one row survives, aligning network bandwidth with lazy materialization. +For safe single-column conjuncts, the scheduler records an exponentially weighted cost per input +row and survival ratio. Cold batches preserve declaration order; after every candidate has a sample, +the next batch orders predicates by cost per rejected row. The same learned order limits predicate +prefetch to the prefix with a meaningful probability of being reached, and a sustained high overall +survival ratio estimate may warm output chunks early. This retains correctness because only +independently safe conjuncts move and all readers still advance over the same logical batch. + +Predicate-only `INT`, `BIGINT`, `FLOAT`, and `DOUBLE` comparisons can bypass Doris-column +materialization when every value page in the Column Chunk is fixed-width PLAIN and the expression +is an exact `=`, `!=`, `<`, `<=`, `>`, or `>=` comparison with a literal. The scheduler compiles +the conjunction into physical-value predicates, the decoder compares selected non-NULL spans in +place, and the reader remaps the compact match bytes to logical nullable rows. NULL comparisons are +false. Mixed encodings, projected predicate columns, casts, logical conversions, compound +expressions, and unsupported types make the decision before either the definition-level or value +cursor advances, so they safely retain ordinary expression evaluation. An exact direct result +leaves only a row-shaped placeholder for the hidden predicate slot and avoids both predicate-column +materialization and later predicate compaction. + +Selection state is scheduler-owned across adaptive batches. Its dense filter bitmap is cached by +selection generation and batch shape, so every lazily materialized output reader consumes the same +bitmap without rebuilding an O(batch-size) array. Logical sizes reset per batch while ordinary +capacity remains reusable. + +Fully rejected batches accumulate a logical lag for lazy columns. When a later batch survives, each +lazy reader consumes that lag in at most 65,535-row dense-filter chunks. This keeps the scheduler's +single logical skip while bounding adapter-owned bitmap capacity independently of prefix length and +lazy projection width. + ## 11. Correctness, Fallback, and Capability Boundaries V2 follows a prove-before-skip rule. Missing indexes, unsupported types, expressions that cannot be @@ -442,6 +836,15 @@ split safely, or read anomalies must never change query semantics. Troubleshoot in this order: verify planning effectiveness, row filtering, lazy materialization, and then I/O/cache health. Total ScanTime alone does not identify the cause. +The visible timer hierarchy is `FileScannerV2 -> TableReader -> FileReader -> IO`; the +format-specific `ParquetReader` subtree belongs to `FileReader`. Scanner lifecycle and Split work, +table-semantic restoration, format metadata/index/decode/materialization, and physical I/O are +charged to their owning layer. Native child-reader statistics accumulate in plain integers and are +published every 16 batches. Row Group reset, EOF, and reader close force the final delta before +destroying the reader tree, so short files retain complete attribution without paying recursive +profile publication on every tiny batch. Retained-scratch inspection uses the same amortized cadence +and a Row Group remains its hard lifetime bound. + ```mermaid flowchart TD A[Slow Scan] --> B{Many Row Groups Pruned?} @@ -465,10 +868,18 @@ flowchart TD | Page index pruning | How many indexes were checked, pages/rows were pruned, ranges selected, and pages skipped? | | Dictionary row filter | How often were predicates rewritten, dictionaries read, bitmaps built, and attempts successful or rejected? | | Predicate / raw rows | How many rows were read and rejected, and was lazy materialization worthwhile? | +| Predicate compaction | Did selection-first evaluation avoid repeated movement? Inspect `PredicateCompactionTime/Bytes/Count`; single-column rounds retain row mappings and compact at multi-column/delete/output boundaries. | +| PLAIN direct predicate | How many eligible predicate-only physical batches and input rows bypassed Doris-column materialization? Inspect `PlainPredicateDirectBatches/Rows`. | +| Avoided projected I/O | How many compressed bytes from projected physical chunks were avoided? `FilteredBytes` deliberately excludes unprojected nested children. | +| Metadata lifecycle | How much time was spent reading/parsing the native footer and schema tree, natively reading/parsing page indexes, and evaluating row-group/page-index predicates? | | Parquet Page Cache | What were hit/miss/write counts and compressed/decompressed hit shapes? | | FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | | Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | | Condition Cache | How many rows were skipped early after a cache hit? | +| Native decode | How much time is spent in page parsing, decompression, levels, encoding, selection, conversion, string/fixed-binary materialization, and scratch growth? Compare `HybridSelectionBatches`, `HybridSelectionRanges`, and `HybridSelectionNullFallbackBatches` to distinguish batched sparse decode from NULL-interleaving fallback. | +| Batch fragmentation | How does `TotalBatches` divide into adaptive probes, dense, selected, empty, page-crossing, and nested/fragmented batches? | +| Index decisions | How often were statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skips attempted, accepted, conservatively rejected, or rejected as corrupt? | +| Cache lifecycle | For footer/page/file/condition caches, what were request, hit, miss, bypass, admission/write, byte, wait, and underlying-I/O counts using v1-compatible meanings? | > Interpret pruning ratios in the context of write layout. Unsorted data produces wide min/max > ranges, so Row Group/Page pruning may be ineffective even when the reader and indexes work @@ -476,14 +887,16 @@ flowchart TD ## 13. Summary -The FileScannerV2 Parquet scan pipeline has three primary threads: +The FileScannerV2 Parquet scan pipeline has four primary threads: 1. **Semantic thread:** TableReader maps table schema and predicates into stable file-local semantics, preserving schema evolution, partition columns, and missing columns. 2. **Pruning thread:** Split → Row Group → Page → Row progressively applies Runtime Filters, Statistics, Dictionary, Bloom, Page Index, and actual-value filters. -3. **I/O thread:** Predicate-first reads, SelectionVector, lazy materialization, adaptive batches, - FileCache/Page Cache/Condition Cache, and MergeRange reduce read amplification together. +3. **Decode thread:** Persistent native page/encoding readers merge Dremel levels with Selection, + reuse scratch, reconstruct complex values from shared plans, and materialize directly to Doris. +4. **I/O thread:** Predicate-first reads, adaptive batches, Footer/File/Page/Condition caches, and + MergeRange reduce read amplification together. ```mermaid flowchart LR diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out index 4fe42d7fcdcc77..bb74c79d5a6907 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out @@ -808,10 +808,10 @@ true -- !test_98 -- \N \N \N -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 -- !test_100 -- 1317017856 1 18752152 809291 1089176 19951117 3-MEDIUM 0 40 4801000 16034243 9 4368910 72015 3 19951228 RAIL Customer#018752152 q4gN2btSpiKXdN,6 ALGERIA 1 ALGERIA AFRICA 10-753-996-8708 MACHINERY Supplier#001089176 ROidEL1L6yeFsJqnUjD EGYPT 5 EGYPT MIDDLE EAST 14-807-108-7869 blanched gainsboro MFGR#4 MFGR#43 MFGR#433 brown MEDIUM BRUSHED STEEL 42 MED BAG @@ -866,10 +866,10 @@ abcDeFGhijkLmnOp \N 1212 -- !test_107 -- \N \N \N -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -- !test_107_desc -- decimal_flba decimal(5,2) Yes false \N NONE diff --git a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy index 98f03c46e48959..981cd0e3b98676 100644 --- a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy +++ b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy @@ -177,25 +177,25 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("2", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("2", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("1", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("1", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -208,25 +208,25 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -239,25 +239,25 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -271,26 +271,26 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("8", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("8", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("7", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("7", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("6", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("6", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("9", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("9", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) }